build notifications for container breaches

This commit is contained in:
Alex Holliday committed 2026-09-23 10:54:35 -07:00
1 parent a75bb5fae6
commit 4ea6539a7d
6 files changed
+122 -21

No files matched your search

+47
View File
@@ -0,0 +1,47 @@
import type { DockerContainerInfo } from "@/domain/docker/docker.type.js";
import type { Monitor } from "@/domain/monitors/monitor.type.js";
export const DockerAlertReasons = ["stopped", "unhealthy"] as const;
export type DockerAlertReason = (typeof DockerAlertReasons)[number];
export interface DockerContainerBreach {
name: string;
reason: DockerAlertReason;
exitCode?: number;
}
export const isContainerStopped = (container: DockerContainerInfo): boolean => {
// Container with exit code 0 is probably a one shot container don't report stopped
return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0);
};
export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => {
return container.health === "unhealthy";
};
export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => {
const breaches: DockerContainerBreach[] = [];
for (const container of containers) {
if (monitor.dockerAlertOnStopped && isContainerStopped(container)) {
breaches.push({
name: container.name,
reason: "stopped",
exitCode: container.exitCode,
});
} else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) {
breaches.push({
name: container.name,
reason: "unhealthy",
});
}
}
return breaches;
};
export const describeContainerBreach = (breach: DockerContainerBreach): string => {
if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`;
if (breach.exitCode === undefined) {
return `${breach.name}: stopped`;
}
return `${breach.name} stopped (exit code ${breach.exitCode})`;
};
@@ -1,5 +1,5 @@
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
import type {
NotificationMessage,
@@ -8,6 +8,7 @@ import type {
ThresholdBreach,
NotificationContent,
} from "@/domain/notifications/notification.type.js";
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
export interface INotificationMessageBuilder {
buildMessage(
@@ -59,32 +60,29 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
return "monitor_down";
}
// Threshold breach (only if not down)
if (decision.notificationReason === "threshold_breach") {
return "threshold_breach";
const isDocker = monitor.type === "docker";
switch (decision.notificationReason) {
case "threshold_breach":
return isDocker ? "container_breach" : "threshold_breach";
case "threshold_resolved":
return isDocker ? "container_resolved" : "threshold_resolved";
default:
return "monitor_up";
}
// Recovery from threshold breach (only for hardware monitors)
if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") {
return "threshold_resolved";
}
// Standard recovery (up)
if (monitor.status === "up") {
return "monitor_up";
}
// Default to monitor_up for any other case
return "monitor_up";
}
private determineSeverity(type: NotificationType): NotificationSeverity {
switch (type) {
case "monitor_up":
return "success";
case "monitor_down":
return "critical";
case "container_breach":
return "warning";
case "threshold_breach":
return "warning";
case "monitor_up":
case "container_resolved":
return "success";
case "threshold_resolved":
return "success";
case "test":
@@ -104,6 +102,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<HardwareStatusPayload>);
case "threshold_resolved":
return this.buildThresholdResolvedContent(monitor);
case "container_breach":
return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<DockerStatusPayload>);
case "container_resolved":
return this.buildContainersRecoveredContent(monitor);
default:
return this.buildDefaultContent(monitor);
}
@@ -174,6 +176,23 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}
private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<DockerStatusPayload>): NotificationContent {
const payload = monitorStatusResponse.payload;
const containers = payload && typeof payload !== "string" ? payload.containers : [];
const breaches = findContainerBreaches(monitor, containers);
const title = `Container Alert: ${monitor.name}`;
const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`;
const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)];
return { title, summary, details, timestamp: new Date() };
}
private buildContainersRecoveredContent(monitor: Monitor): NotificationContent {
const title = `Containers Recovered: ${monitor.name}`;
const summary = `All containers on "${monitor.name}" are back to normal.`;
const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`];
return { title, summary, details, timestamp: new Date() };
}
private buildDefaultContent(monitor: Monitor): NotificationContent {
return {
title: `Monitor: ${monitor.name}`,
@@ -71,7 +71,8 @@ export interface AlertDiscordPayload {
* Part of notification system unification effort
*/
export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test";
export type NotificationType =
"monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test";
export type NotificationSeverity = "critical" | "warning" | "info" | "success";