diff --git a/server/src/domain/docker/docker-alert.ts b/server/src/domain/docker/docker-alert.ts new file mode 100644 index 000000000..45b8075ba --- /dev/null +++ b/server/src/domain/docker/docker-alert.ts @@ -0,0 +1,47 @@ +import type { DockerContainerInfo } from "@/domain/docker/docker.type.js"; +import type { Monitor } from "@/domain/monitors/monitor.type.js"; + +export const DockerAlertReasons = ["stopped", "unhealthy"] as const; +export type DockerAlertReason = (typeof DockerAlertReasons)[number]; + +export interface DockerContainerBreach { + name: string; + reason: DockerAlertReason; + exitCode?: number; +} + +export const isContainerStopped = (container: DockerContainerInfo): boolean => { + // Container with exit code 0 is probably a one shot container don't report stopped + return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0); +}; + +export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => { + return container.health === "unhealthy"; +}; + +export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => { + const breaches: DockerContainerBreach[] = []; + for (const container of containers) { + if (monitor.dockerAlertOnStopped && isContainerStopped(container)) { + breaches.push({ + name: container.name, + reason: "stopped", + exitCode: container.exitCode, + }); + } else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) { + breaches.push({ + name: container.name, + reason: "unhealthy", + }); + } + } + return breaches; +}; + +export const describeContainerBreach = (breach: DockerContainerBreach): string => { + if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`; + if (breach.exitCode === undefined) { + return `${breach.name}: stopped`; + } + return `${breach.name} stopped (exit code ${breach.exitCode})`; +}; diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 641d9527c..5cd78e9d9 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -1,5 +1,5 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; -import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; +import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { NotificationMessage, @@ -8,6 +8,7 @@ import type { ThresholdBreach, NotificationContent, } from "@/domain/notifications/notification.type.js"; +import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; export interface INotificationMessageBuilder { buildMessage( @@ -59,32 +60,29 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return "monitor_down"; } - // Threshold breach (only if not down) - if (decision.notificationReason === "threshold_breach") { - return "threshold_breach"; + const isDocker = monitor.type === "docker"; + switch (decision.notificationReason) { + case "threshold_breach": + return isDocker ? "container_breach" : "threshold_breach"; + case "threshold_resolved": + return isDocker ? "container_resolved" : "threshold_resolved"; + default: + return "monitor_up"; } - - // Recovery from threshold breach (only for hardware monitors) - if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") { - return "threshold_resolved"; - } - - // Standard recovery (up) - if (monitor.status === "up") { - return "monitor_up"; - } - - // Default to monitor_up for any other case - return "monitor_up"; } private determineSeverity(type: NotificationType): NotificationSeverity { switch (type) { + case "monitor_up": + return "success"; case "monitor_down": return "critical"; + case "container_breach": + return "warning"; case "threshold_breach": return "warning"; - case "monitor_up": + case "container_resolved": + return "success"; case "threshold_resolved": return "success"; case "test": @@ -104,6 +102,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); case "threshold_resolved": return this.buildThresholdResolvedContent(monitor); + case "container_breach": + return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); + case "container_resolved": + return this.buildContainersRecoveredContent(monitor); default: return this.buildDefaultContent(monitor); } @@ -174,6 +176,23 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } + private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + const payload = monitorStatusResponse.payload; + const containers = payload && typeof payload !== "string" ? payload.containers : []; + const breaches = findContainerBreaches(monitor, containers); + const title = `Container Alert: ${monitor.name}`; + const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`; + const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)]; + return { title, summary, details, timestamp: new Date() }; + } + + private buildContainersRecoveredContent(monitor: Monitor): NotificationContent { + const title = `Containers Recovered: ${monitor.name}`; + const summary = `All containers on "${monitor.name}" are back to normal.`; + const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`]; + return { title, summary, details, timestamp: new Date() }; + } + private buildDefaultContent(monitor: Monitor): NotificationContent { return { title: `Monitor: ${monitor.name}`, diff --git a/server/src/domain/notifications/notification.type.ts b/server/src/domain/notifications/notification.type.ts index 628633d56..7f2c7df1e 100644 --- a/server/src/domain/notifications/notification.type.ts +++ b/server/src/domain/notifications/notification.type.ts @@ -71,7 +71,8 @@ export interface AlertDiscordPayload { * Part of notification system unification effort */ -export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test"; +export type NotificationType = + "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test"; export type NotificationSeverity = "critical" | "warning" | "info" | "success"; diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 07ddee2a0..cb093c479 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -19,6 +19,7 @@ import { ILogger } from "@/utils/logger.js"; import type { HardwareStatusMetrics } from "@/types/network.js"; import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js"; import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js"; +import { findContainerBreaches } from "@/domain/docker/docker-alert.js"; const SERVICE_NAME = "StatusService"; const HARDWARE_ALERT_COUNTER_START = 5; @@ -162,6 +163,21 @@ export class StatusService implements IStatusService { return { nextStatus, transitioned, breaches, nextCounters }; }; + private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => { + const { currentStatus, reachabilityDown, breaching } = params; + let nextStatus: MonitorStatus = currentStatus; + let transitioned = false; + + if (!reachabilityDown) { + if (breaching && currentStatus !== "breached") { + nextStatus = "breached"; + transitioned = true; + } else if (!breaching && currentStatus === "breached") nextStatus = "up"; + transitioned = true; + } + return { nextStatus, transitioned }; + }; + updateMonitorStatus = async ( statusResponse: MonitorStatusResponse< | PingStatusPayload @@ -268,6 +284,20 @@ export class StatusService implements IStatusService { } } + const dockerPayload = statusResponse.payload as DockerStatusPayload | undefined; + if (monitor.type === "docker" && dockerPayload?.containers) { + const breaches = findContainerBreaches(monitor, dockerPayload.containers); + const docker = this.computeDockerStatus({ + currentStatus: newStatus, + reachabilityDown: newStatus === "down", + breaching: breaches.length > 0, + }); + if (docker.transitioned) { + newStatus = docker.nextStatus; + statusChanged = true; + } + } + patch.status = newStatus; // Single atomic write: push arrays + set status/counters diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 379040010..d29b95fbb 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -12,7 +12,7 @@ export interface MonitorActionDecision { shouldResolveIncident: boolean; shouldSendNotification: boolean; incidentReason: "status_down" | "threshold_breach" | null; - notificationReason: "status_change" | "threshold_breach" | null; + notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null; thresholdBreaches?: { cpu?: boolean; memory?: boolean; diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index bdcb86c00..658737826 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -326,7 +326,11 @@ export class WorkerPipeline implements IWorkerPipeline { decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; decision.notificationReason = "threshold_breach"; - } else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) { + } else if (monitor.status === "up" && prevStatus === "breached") { + decision.shouldResolveIncident = true; + decision.shouldSendNotification = true; + decision.notificationReason = "threshold_resolved"; + } else if (monitor.status === "up" && prevStatus === "down") { decision.shouldResolveIncident = true; decision.shouldSendNotification = true; decision.notificationReason = "status_change";