build notifications for container breaches

server/src/domain/docker/docker-alert.ts

@@ -0,0 +1,47 @@
+import type { DockerContainerInfo } from "@/domain/docker/docker.type.js";
+import type { Monitor } from "@/domain/monitors/monitor.type.js";
+
+export const DockerAlertReasons = ["stopped", "unhealthy"] as const;
+export type DockerAlertReason = (typeof DockerAlertReasons)[number];
+
+export interface DockerContainerBreach {
+	name: string;
+	reason: DockerAlertReason;
+	exitCode?: number;
+}
+
+export const isContainerStopped = (container: DockerContainerInfo): boolean => {
+	// Container with exit code 0 is probably a one shot container don't report stopped
+	return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0);
+};
+
+export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => {
+	return container.health === "unhealthy";
+};
+
+export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => {
+	const breaches: DockerContainerBreach[] = [];
+	for (const container of containers) {
+		if (monitor.dockerAlertOnStopped && isContainerStopped(container)) {
+			breaches.push({
+				name: container.name,
+				reason: "stopped",
+				exitCode: container.exitCode,
+			});
+		} else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) {
+			breaches.push({
+				name: container.name,
+				reason: "unhealthy",
+			});
+		}
+	}
+	return breaches;
+};
+
+export const describeContainerBreach = (breach: DockerContainerBreach): string => {
+	if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`;
+	if (breach.exitCode === undefined) {
+		return `${breach.name}: stopped`;
+	}
+	return `${breach.name} stopped (exit code ${breach.exitCode})`;
+};

server/src/domain/notifications/notification.message-builder.ts

@@ -1,5 +1,5 @@
 import type { Monitor } from "@/domain/monitors/monitor.type.js";
-import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
+import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
 import type { MonitorActionDecision } from "@/worker/worker.interface.js";
 import type {
 	NotificationMessage,
@@ -8,6 +8,7 @@ import type {
 	ThresholdBreach,
 	NotificationContent,
 } from "@/domain/notifications/notification.type.js";
+import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
 
 export interface INotificationMessageBuilder {
 	buildMessage(
@@ -59,32 +60,29 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
 			return "monitor_down";
 		}
 
-		// Threshold breach (only if not down)
-		if (decision.notificationReason === "threshold_breach") {
-			return "threshold_breach";
+		const isDocker = monitor.type === "docker";
+		switch (decision.notificationReason) {
+			case "threshold_breach":
+				return isDocker ? "container_breach" : "threshold_breach";
+			case "threshold_resolved":
+				return isDocker ? "container_resolved" : "threshold_resolved";
+			default:
+				return "monitor_up";
 		}
-
-		// Recovery from threshold breach (only for hardware monitors)
-		if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") {
-			return "threshold_resolved";
-		}
-
-		// Standard recovery (up)
-		if (monitor.status === "up") {
-			return "monitor_up";
-		}
-
-		// Default to monitor_up for any other case
-		return "monitor_up";
 	}
 
 	private determineSeverity(type: NotificationType): NotificationSeverity {
 		switch (type) {
+			case "monitor_up":
+				return "success";
 			case "monitor_down":
 				return "critical";
+			case "container_breach":
+				return "warning";
 			case "threshold_breach":
 				return "warning";
-			case "monitor_up":
+			case "container_resolved":
+				return "success";
 			case "threshold_resolved":
 				return "success";
 			case "test":
@@ -104,6 +102,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
 				return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<HardwareStatusPayload>);
 			case "threshold_resolved":
 				return this.buildThresholdResolvedContent(monitor);
+			case "container_breach":
+				return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<DockerStatusPayload>);
+			case "container_resolved":
+				return this.buildContainersRecoveredContent(monitor);
 			default:
 				return this.buildDefaultContent(monitor);
 		}
@@ -174,6 +176,23 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
 		};
 	}
 
+	private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<DockerStatusPayload>): NotificationContent {
+		const payload = monitorStatusResponse.payload;
+		const containers = payload && typeof payload !== "string" ? payload.containers : [];
+		const breaches = findContainerBreaches(monitor, containers);
+		const title = `Container Alert: ${monitor.name}`;
+		const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`;
+		const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)];
+		return { title, summary, details, timestamp: new Date() };
+	}
+
+	private buildContainersRecoveredContent(monitor: Monitor): NotificationContent {
+		const title = `Containers Recovered: ${monitor.name}`;
+		const summary = `All containers on "${monitor.name}" are back to normal.`;
+		const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`];
+		return { title, summary, details, timestamp: new Date() };
+	}
+
 	private buildDefaultContent(monitor: Monitor): NotificationContent {
 		return {
 			title: `Monitor: ${monitor.name}`,

server/src/domain/notifications/notification.type.ts

@@ -71,7 +71,8 @@ export interface AlertDiscordPayload {
  * Part of notification system unification effort
  */
 
-export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test";
+export type NotificationType =
+	"monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test";
 
 export type NotificationSeverity = "critical" | "warning" | "info" | "success";
 

server/src/service/statusService.ts

@@ -19,6 +19,7 @@ import { ILogger } from "@/utils/logger.js";
 import type { HardwareStatusMetrics } from "@/types/network.js";
 import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js";
 import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js";
+import { findContainerBreaches } from "@/domain/docker/docker-alert.js";
 
 const SERVICE_NAME = "StatusService";
 const HARDWARE_ALERT_COUNTER_START = 5;
@@ -162,6 +163,21 @@ export class StatusService implements IStatusService {
 		return { nextStatus, transitioned, breaches, nextCounters };
 	};
 
+	private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => {
+		const { currentStatus, reachabilityDown, breaching } = params;
+		let nextStatus: MonitorStatus = currentStatus;
+		let transitioned = false;
+
+		if (!reachabilityDown) {
+			if (breaching && currentStatus !== "breached") {
+				nextStatus = "breached";
+				transitioned = true;
+			} else if (!breaching && currentStatus === "breached") nextStatus = "up";
+			transitioned = true;
+		}
+		return { nextStatus, transitioned };
+	};
+
 	updateMonitorStatus = async (
 		statusResponse: MonitorStatusResponse<
 			| PingStatusPayload
@@ -268,6 +284,20 @@ export class StatusService implements IStatusService {
 				}
 			}
 
+			const dockerPayload = statusResponse.payload as DockerStatusPayload | undefined;
+			if (monitor.type === "docker" && dockerPayload?.containers) {
+				const breaches = findContainerBreaches(monitor, dockerPayload.containers);
+				const docker = this.computeDockerStatus({
+					currentStatus: newStatus,
+					reachabilityDown: newStatus === "down",
+					breaching: breaches.length > 0,
+				});
+				if (docker.transitioned) {
+					newStatus = docker.nextStatus;
+					statusChanged = true;
+				}
+			}
+
 			patch.status = newStatus;
 
 			// Single atomic write: push arrays + set status/counters

server/src/worker/worker.interface.ts

@@ -12,7 +12,7 @@ export interface MonitorActionDecision {
 	shouldResolveIncident: boolean;
 	shouldSendNotification: boolean;
 	incidentReason: "status_down" | "threshold_breach" | null;
-	notificationReason: "status_change" | "threshold_breach" | null;
+	notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null;
 	thresholdBreaches?: {
 		cpu?: boolean;
 		memory?: boolean;

server/src/worker/worker.pipeline.ts

@@ -326,7 +326,11 @@ export class WorkerPipeline implements IWorkerPipeline {
 			decision.shouldSendNotification = true;
 			decision.incidentReason = "threshold_breach";
 			decision.notificationReason = "threshold_breach";
-		} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
+		} else if (monitor.status === "up" && prevStatus === "breached") {
+			decision.shouldResolveIncident = true;
+			decision.shouldSendNotification = true;
+			decision.notificationReason = "threshold_resolved";
+		} else if (monitor.status === "up" && prevStatus === "down") {
 			decision.shouldResolveIncident = true;
 			decision.shouldSendNotification = true;
 			decision.notificationReason = "status_change";