diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index c487f8d81..1010a94ed 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -85,6 +85,8 @@ export const getMonitorDefaults = ( dockerTlsKey: "", dockerTlsKeySet: data?.dockerTlsKeySet ?? false, dockerLogsEnabled: data?.dockerLogsEnabled ?? false, + dockerAlertOnStopped: data?.dockerAlertOnStopped ?? false, + dockerAlertOnUnhealthy: data?.dockerAlertOnUnhealthy ?? false, }; break; case "game": diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 1de12d265..d936d5496 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -617,7 +617,6 @@ const CreateMonitorPage = () => { } /> )} - {showStep(0) && ( { } /> )} - {showStep(0) && watchedType === "docker" && isDockerTlsUrl(watchedUrl) && ( { } /> )} - {showStep(1) && ( { } /> )} - {/* Alert Thresholds - only for hardware type */} {watchedType === "hardware" && showStep(1) && ( { } /> )} - {showStep(1) && ( { } /> )} - {showStep(1) && ( { } /> )} - {showStep(1) && ( { } /> )} - {showStep(1) && watchedType === "docker" && !isCaptureDocker && ( { } /> )} - + {showStep(1) && watchedType === "docker" && ( + + + + + } + /> + )} {showStep(2) && (watchedType === "http" || watchedType === "grpc" || @@ -989,7 +1002,6 @@ const CreateMonitorPage = () => { } /> )} - {showStep(2) && watchedType === "http" && ( { } /> )} - {showStep(2) && supportsGeoCheck(watchedType) && ( { } /> )} - { controller={c} bulkActions showTypeFilter={false} + summaryProps={{ showBreached: true }} > { websocket: "uptime", dns: "uptime", unknown: "uptime", - docker: "docker", + docker: "docker/host", hardware: "infrastructure", pagespeed: "pagespeed", }; diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 682c76a7e..f774ceae3 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -152,6 +152,8 @@ const dockerSchema = baseSchema.extend({ // Read only mirror of the server flag, stripped on submit dockerTlsKeySet: z.boolean(), dockerLogsEnabled: z.boolean().register(monitorStepRegistry, { step: 1 }), + dockerAlertOnStopped: z.boolean().register(monitorStepRegistry, { step: 1 }), + dockerAlertOnUnhealthy: z.boolean().register(monitorStepRegistry, { step: 1 }), }); // Game server monitor schema diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 1b1337dd2..4c0161001 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -557,7 +557,7 @@ }, "status": { "down": "down", - "breached": "threshold exceeded", + "breached": "alert", "initializing": "initializing", "maintenance": "maintenance", "paused": "paused", @@ -827,6 +827,18 @@ } } }, + "dockerAlerts": { + "title": "Container alerts", + "description": "Mark this host as breached while any container on it is stopped or failing its health check, and notify on the change. Containers that exit with code 0 do not count as stopped. Capture does not report exit codes, so on Capture hosts every stopped container counts.", + "option": { + "onStopped": { + "label": "Alert when a container is stopped" + }, + "onUnhealthy": { + "label": "Alert when a container is unhealthy" + } + } + }, "stats": { "title": "Monitor history", "description": "Clear all monitoring history and statistics for your team. This action is irreversible.", @@ -1685,7 +1697,7 @@ "status": { "up": "Operational", "down": "Down", - "breached": "Threshold exceeded", + "breached": "Alert", "maintenance": "Maintenance", "paused": "Paused", "initializing": "Initializing" @@ -1725,13 +1737,13 @@ "monitoringSummary": "Monitoring {{count}} service · Updates every 10 seconds", "monitoringSummary_other": "Monitoring {{count}} services · Updates every 10 seconds", "allUp": "All systems operational", - "allBreached": "All systems have exceeded thresholds", + "allBreached": "All systems have active alerts", "allMaintenance": "All systems are under maintenance", "allDown": "All systems are down", "allPaused": "All systems are paused", "allInitializing": "All systems are initializing", - "breachedAndDown": "Some systems have exceeded thresholds and are down", - "breached": "Some systems have exceeded thresholds", + "breachedAndDown": "Some systems have active alerts and some are down", + "breached": "Some systems have active alerts", "degraded": "Some systems are experiencing issues", "maintenanceAndDown": "Some systems are down during maintenance", "maintenance": "Some systems are under maintenance", diff --git a/server/openapi.json b/server/openapi.json index 237bcea29..4cbcdb1f9 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -466,6 +466,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -523,6 +529,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -694,6 +702,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -753,6 +767,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -7186,6 +7202,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -7245,6 +7267,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -7761,6 +7785,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -7820,6 +7850,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -9308,6 +9340,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "anyOf": [ { @@ -10033,6 +10071,14 @@ "type": "boolean", "default": false }, + "dockerAlertOnStopped": { + "type": "boolean", + "default": false + }, + "dockerAlertOnUnhealthy": { + "type": "boolean", + "default": false + }, "dnsServer": { "type": "string", "minLength": 1 @@ -10600,6 +10646,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "anyOf": [ { diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index 651cf786d..359a2fabf 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -220,6 +220,8 @@ export const createMonitorBodyValidation = z geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), dockerLogsEnabled: z.boolean().optional(), + dockerAlertOnStopped: z.boolean().optional(), + dockerAlertOnUnhealthy: z.boolean().optional(), dockerTlsCa: z.union([z.string(), z.literal("")]).optional(), dockerTlsCert: z.union([z.string(), z.literal("")]).optional(), dockerTlsKey: z.union([z.string(), z.literal("")]).optional(), @@ -270,6 +272,8 @@ export const editMonitorBodyValidation = z geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), dockerLogsEnabled: z.boolean().optional(), + dockerAlertOnStopped: z.boolean().optional(), + dockerAlertOnUnhealthy: z.boolean().optional(), dockerTlsCa: z.union([z.string(), z.literal("")]).optional(), dockerTlsCert: z.union([z.string(), z.literal("")]).optional(), dockerTlsKey: z.union([z.string(), z.literal("")]).optional(), @@ -350,6 +354,8 @@ const importedMonitorSchema = z geoCheckLocations: z.array(z.enum(GeoContinents)).default([]), geoCheckInterval: z.number().min(300000).default(300000), dockerLogsEnabled: z.boolean().default(false), + dockerAlertOnStopped: z.boolean().default(false), + dockerAlertOnUnhealthy: z.boolean().default(false), dnsServer: dnsServerValidation.optional(), dnsRecordType: z.enum(DnsRecordTypes).optional(), createdAt: z.string().optional(), @@ -440,6 +446,8 @@ export const monitorResponseSchema = z geoCheckLocations: z.array(z.enum(GeoContinents)), geoCheckInterval: z.number(), dockerLogsEnabled: z.boolean(), + dockerAlertOnStopped: z.boolean(), + dockerAlertOnUnhealthy: z.boolean(), dockerTlsCa: z.string().optional(), dockerTlsCert: z.string().optional(), dockerTlsKeySet: z.boolean().optional(), diff --git a/server/src/domain/checks/check.model.ts b/server/src/domain/checks/check.model.ts index 1f92fce3c..a4e202d40 100644 --- a/server/src/domain/checks/check.model.ts +++ b/server/src/domain/checks/check.model.ts @@ -220,6 +220,7 @@ const dockerContainerSchema = new Schema( startedAt: { type: String }, ports: { type: [dockerContainerPortSchema], default: undefined }, mounts: { type: [dockerContainerMountSchema], default: undefined }, + exitCode: { type: Number, default: undefined }, }, { _id: false } ); diff --git a/server/src/domain/checks/check.service.ts b/server/src/domain/checks/check.service.ts index 2bc9f2a16..30db653e6 100644 --- a/server/src/domain/checks/check.service.ts +++ b/server/src/domain/checks/check.service.ts @@ -135,6 +135,17 @@ export class CheckService implements ICheckService { return check; }; + private toPayload = (check: Check): HardwareStatusPayload | DockerStatusPayload | undefined => { + switch (check.metadata.type) { + case "hardware": + return { data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload; + case "docker": + return check.containers ? { containers: check.containers, summary: check.containerSummary } : undefined; + default: + return undefined; + } + }; + toStatusResponse = (check: Check) => { const statusResponse: MonitorStatusResponse = { monitorId: check.metadata.monitorId, @@ -145,10 +156,7 @@ export class CheckService implements ICheckService { message: check.message, responseTime: check.responseTime, // re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data - payload: - check.metadata.type === "hardware" - ? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload) - : undefined, + payload: this.toPayload(check), }; return statusResponse; }; diff --git a/server/src/domain/docker/docker-alert.ts b/server/src/domain/docker/docker-alert.ts new file mode 100644 index 000000000..6045625a7 --- /dev/null +++ b/server/src/domain/docker/docker-alert.ts @@ -0,0 +1,47 @@ +import type { DockerContainerInfo } from "@/domain/docker/docker.type.js"; +import type { Monitor } from "@/domain/monitors/monitor.type.js"; + +export const DockerAlertReasons = ["stopped", "unhealthy"] as const; +export type DockerAlertReason = (typeof DockerAlertReasons)[number]; + +export interface DockerContainerBreach { + name: string; + reason: DockerAlertReason; + exitCode?: number; +} + +export const isContainerStopped = (container: DockerContainerInfo): boolean => { + // Container with exit code 0 is probably a one shot container don't report stopped + return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0); +}; + +export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => { + return container.health === "unhealthy"; +}; + +export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => { + const breaches: DockerContainerBreach[] = []; + for (const container of containers) { + if (monitor.dockerAlertOnStopped && isContainerStopped(container)) { + breaches.push({ + name: container.name, + reason: "stopped", + exitCode: container.exitCode, + }); + } else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) { + breaches.push({ + name: container.name, + reason: "unhealthy", + }); + } + } + return breaches; +}; + +export const describeContainerBreach = (breach: DockerContainerBreach): string => { + if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`; + if (breach.exitCode === undefined) { + return `${breach.name}: stopped`; + } + return `${breach.name}: stopped (exit code ${breach.exitCode})`; +}; diff --git a/server/src/domain/docker/docker.type.ts b/server/src/domain/docker/docker.type.ts index 66504e273..da2d51337 100644 --- a/server/src/domain/docker/docker.type.ts +++ b/server/src/domain/docker/docker.type.ts @@ -55,6 +55,7 @@ export interface DockerContainerInfo { startedAt?: string; // ISO date ports?: DockerContainerPort[]; mounts?: DockerContainerMount[]; + exitCode?: number; } export interface DockerContainerSummary { diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index f8c79cb80..1c249626a 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -1,6 +1,6 @@ const SERVICE_NAME = "incidentService"; import type { Monitor } from "@/domain/monitors/monitor.type.js"; -import type { MonitorStatusResponse } from "@/types/network.js"; +import type { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { getDateForRange } from "@/utils/dataUtils.js"; import type { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js"; @@ -12,6 +12,7 @@ import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; import type { ILogger } from "@/utils/logger.js"; import { DateRange } from "@/types/query.js"; +import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; export interface IIncidentService { handleIncident( @@ -106,6 +107,12 @@ export class IncidentService implements IIncidentService { }; private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string { + if (monitor.type === "docker") { + const containers = (monitorStatusResponse?.payload as DockerStatusPayload | undefined)?.containers ?? []; + const breaches = findContainerBreaches(monitor, containers); + return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert"; + } + if (!monitorStatusResponse) { return "Threshold breach detected"; } diff --git a/server/src/domain/monitors/monitor.model.ts b/server/src/domain/monitors/monitor.model.ts index c04107651..94a0fd676 100644 --- a/server/src/domain/monitors/monitor.model.ts +++ b/server/src/domain/monitors/monitor.model.ts @@ -307,6 +307,14 @@ const MonitorSchema = new Schema( type: Boolean, default: false, }, + dockerAlertOnStopped: { + type: Boolean, + default: false, + }, + dockerAlertOnUnhealthy: { + type: Boolean, + default: false, + }, dockerTlsCa: { type: String, }, diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index 385dcebab..882bd5ccb 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -468,6 +468,8 @@ class MongoMonitorsRepository implements IMonitorsRepository { geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, dockerLogsEnabled: doc.dockerLogsEnabled ?? false, + dockerAlertOnStopped: doc.dockerAlertOnStopped ?? false, + dockerAlertOnUnhealthy: doc.dockerAlertOnUnhealthy ?? false, dockerTlsCa: doc.dockerTlsCa ?? undefined, dockerTlsCert: doc.dockerTlsCert ?? undefined, dockerTlsKeySet: doc.dockerTlsKeySet ?? false, diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index d5519af0e..b033c560a 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -51,6 +51,22 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number]; export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type); +const MonitorPaths: Record = { + http: "uptime", + port: "uptime", + ping: "uptime", + game: "uptime", + grpc: "uptime", + websocket: "uptime", + dns: "uptime", + unknown: "uptime", + docker: "docker/host", + hardware: "infrastructure", + pagespeed: "pagespeed", +}; + +export const getMonitorPath = (type: MonitorType): string => MonitorPaths[type]; + // Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's // own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts // however far away the agent is. `unknown` is excluded because no request is made for it. @@ -134,6 +150,8 @@ export interface Monitor { geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; dockerLogsEnabled?: boolean; + dockerAlertOnStopped?: boolean; + dockerAlertOnUnhealthy?: boolean; dockerTlsCa?: string; dockerTlsCert?: string; // EncryptionService ciphertext. Normally excluded diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 641d9527c..5cd78e9d9 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -1,5 +1,5 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; -import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; +import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { NotificationMessage, @@ -8,6 +8,7 @@ import type { ThresholdBreach, NotificationContent, } from "@/domain/notifications/notification.type.js"; +import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; export interface INotificationMessageBuilder { buildMessage( @@ -59,32 +60,29 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return "monitor_down"; } - // Threshold breach (only if not down) - if (decision.notificationReason === "threshold_breach") { - return "threshold_breach"; + const isDocker = monitor.type === "docker"; + switch (decision.notificationReason) { + case "threshold_breach": + return isDocker ? "container_breach" : "threshold_breach"; + case "threshold_resolved": + return isDocker ? "container_resolved" : "threshold_resolved"; + default: + return "monitor_up"; } - - // Recovery from threshold breach (only for hardware monitors) - if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") { - return "threshold_resolved"; - } - - // Standard recovery (up) - if (monitor.status === "up") { - return "monitor_up"; - } - - // Default to monitor_up for any other case - return "monitor_up"; } private determineSeverity(type: NotificationType): NotificationSeverity { switch (type) { + case "monitor_up": + return "success"; case "monitor_down": return "critical"; + case "container_breach": + return "warning"; case "threshold_breach": return "warning"; - case "monitor_up": + case "container_resolved": + return "success"; case "threshold_resolved": return "success"; case "test": @@ -104,6 +102,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); case "threshold_resolved": return this.buildThresholdResolvedContent(monitor); + case "container_breach": + return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); + case "container_resolved": + return this.buildContainersRecoveredContent(monitor); default: return this.buildDefaultContent(monitor); } @@ -174,6 +176,23 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } + private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + const payload = monitorStatusResponse.payload; + const containers = payload && typeof payload !== "string" ? payload.containers : []; + const breaches = findContainerBreaches(monitor, containers); + const title = `Container Alert: ${monitor.name}`; + const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`; + const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)]; + return { title, summary, details, timestamp: new Date() }; + } + + private buildContainersRecoveredContent(monitor: Monitor): NotificationContent { + const title = `Containers Recovered: ${monitor.name}`; + const summary = `All containers on "${monitor.name}" are back to normal.`; + const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`]; + return { title, summary, details, timestamp: new Date() }; + } + private buildDefaultContent(monitor: Monitor): NotificationContent { return { title: `Monitor: ${monitor.name}`, diff --git a/server/src/domain/notifications/notification.type.ts b/server/src/domain/notifications/notification.type.ts index 628633d56..7f2c7df1e 100644 --- a/server/src/domain/notifications/notification.type.ts +++ b/server/src/domain/notifications/notification.type.ts @@ -71,7 +71,8 @@ export interface AlertDiscordPayload { * Part of notification system unification effort */ -export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test"; +export type NotificationType = + "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test"; export type NotificationSeverity = "critical" | "warning" | "info" | "success"; diff --git a/server/src/domain/notifications/providers/email.ts b/server/src/domain/notifications/providers/email.ts index 2b5cde245..01bbaa6be 100644 --- a/server/src/domain/notifications/providers/email.ts +++ b/server/src/domain/notifications/providers/email.ts @@ -92,6 +92,10 @@ export class EmailProvider extends NotificationProvider { return `Monitor ${message.monitor.name} threshold exceeded`; case "threshold_resolved": return `Monitor ${message.monitor.name} thresholds resolved`; + case "container_breach": + return `Monitor ${message.monitor.name} container alert`; + case "container_resolved": + return `Monitor ${message.monitor.name} containers recovered`; default: return `Alert: ${message.monitor.name}`; } diff --git a/server/src/domain/notifications/providers/matrix.ts b/server/src/domain/notifications/providers/matrix.ts index 1fafc44b1..7e66b5c37 100644 --- a/server/src/domain/notifications/providers/matrix.ts +++ b/server/src/domain/notifications/providers/matrix.ts @@ -3,7 +3,7 @@ import got from "got"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { AlertMatrixPayload, Notification } from "@/domain/notifications/notification.type.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import { randomUUID } from "crypto"; export class MatrixProvider extends NotificationProvider { @@ -172,10 +172,10 @@ export class MatrixProvider extends NotificationProvider { // Incident link (if incident exists) if (message.content.incident) { - const incidentUrl = `${message.clientHost}/infrastructure/${message.monitor.id}`; + const url = incidentUrl(message); plainLines.push(""); - plainLines.push(`View Incident: ${incidentUrl}`); - htmlLines.push(`

View Incident

`); + plainLines.push(`View Incident: ${url}`); + htmlLines.push(`

View Incident

`); } // Footer with timestamp diff --git a/server/src/domain/notifications/providers/ntfy.ts b/server/src/domain/notifications/providers/ntfy.ts index b415f648b..35f36f5e3 100644 --- a/server/src/domain/notifications/providers/ntfy.ts +++ b/server/src/domain/notifications/providers/ntfy.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "NtfyProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class NtfyProvider extends NotificationProvider { @@ -142,7 +142,7 @@ export class NtfyProvider extends NotificationProvider { } if (message.content.incident) { - lines.push("", `${message.clientHost}/infrastructure/${message.monitor.id}`); + lines.push("", incidentUrl(message)); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/pagerduty.ts b/server/src/domain/notifications/providers/pagerduty.ts index 6cb264c1d..c20fbc85f 100644 --- a/server/src/domain/notifications/providers/pagerduty.ts +++ b/server/src/domain/notifications/providers/pagerduty.ts @@ -75,7 +75,8 @@ export class PagerDutyProvider extends NotificationProvider { private buildPagerDutyPayload(notification: Notification, message: NotificationMessage): AlertPagerDutyPayload { // Map our notification type to PagerDuty event_action - const eventAction = message.type === "monitor_up" || message.type === "threshold_resolved" ? "resolve" : "trigger"; + const eventAction = + message.type === "monitor_up" || message.type === "threshold_resolved" || message.type === "container_resolved" ? "resolve" : "trigger"; // Map severity to PagerDuty severity levels const severityMap: Record = { diff --git a/server/src/domain/notifications/providers/pushover.ts b/server/src/domain/notifications/providers/pushover.ts index b1831e405..4243a0e32 100644 --- a/server/src/domain/notifications/providers/pushover.ts +++ b/server/src/domain/notifications/providers/pushover.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "PushoverProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class PushoverProvider extends NotificationProvider { @@ -101,7 +101,7 @@ export class PushoverProvider extends NotificationProvider { if (message.content.incident) { lines.push(""); - lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`); + lines.push(`View Incident: ${incidentUrl(message)}`); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/rocketChat.ts b/server/src/domain/notifications/providers/rocketChat.ts index 2ef8d7efb..78d93ce8f 100644 --- a/server/src/domain/notifications/providers/rocketChat.ts +++ b/server/src/domain/notifications/providers/rocketChat.ts @@ -1,5 +1,5 @@ const SERVICE_NAME = "RocketChatProvider"; -import { supportsUptimeDetails, type MonitorType } from "@/domain/monitors/monitor.type.js"; +import { getMonitorPath, type MonitorType } from "@/domain/monitors/monitor.type.js"; import type { Notification, NotificationMessage } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import { getTestMessage } from "@/domain/notifications/providers/utils.js"; @@ -139,19 +139,7 @@ export class RocketChatProvider extends NotificationProvider { return undefined; } - if (message.monitor.type === "hardware") { - return `${clientHost}/infrastructure/${message.monitor.id}`; - } - - if (message.monitor.type === "pagespeed") { - return `${clientHost}/pagespeed/${message.monitor.id}`; - } - - if (supportsUptimeDetails(message.monitor.type as MonitorType)) { - return `${clientHost}/uptime/${message.monitor.id}`; - } - - return undefined; + return `${clientHost}/${getMonitorPath(message.monitor.type as MonitorType)}/${message.monitor.id}`; } private severityColor(severity: NotificationMessage["severity"]): string { diff --git a/server/src/domain/notifications/providers/signalgrid.ts b/server/src/domain/notifications/providers/signalgrid.ts index fdd930724..889206c67 100644 --- a/server/src/domain/notifications/providers/signalgrid.ts +++ b/server/src/domain/notifications/providers/signalgrid.ts @@ -1,7 +1,7 @@ const SERVICE_NAME = "SignalgridProvider"; import type { Notification, NotificationMessage, NotificationSeverity } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class SignalgridProvider extends NotificationProvider { @@ -115,7 +115,7 @@ export class SignalgridProvider extends NotificationProvider { if (message.content.incident) { lines.push(""); - lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`); + lines.push(`View Incident: ${incidentUrl(message)}`); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/slack.ts b/server/src/domain/notifications/providers/slack.ts index 23f9b2f61..f73526ce0 100644 --- a/server/src/domain/notifications/providers/slack.ts +++ b/server/src/domain/notifications/providers/slack.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "SlackProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got, { HTTPError } from "got"; export class SlackProvider extends NotificationProvider { @@ -161,7 +161,7 @@ export class SlackProvider extends NotificationProvider { text: "View Incident", emoji: true, }, - url: `${message.clientHost}/infrastructure/${message.monitor.id}`, + url: incidentUrl(message), style: "primary", }, ], diff --git a/server/src/domain/notifications/providers/teams.ts b/server/src/domain/notifications/providers/teams.ts index 36d37a210..98e9e3c43 100644 --- a/server/src/domain/notifications/providers/teams.ts +++ b/server/src/domain/notifications/providers/teams.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "TeamsProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got, { HTTPError } from "got"; // Types for Adaptive Card elements @@ -254,7 +254,7 @@ export class TeamsProvider extends NotificationProvider { actions.push({ type: "Action.OpenUrl", title: "View Incident", - url: `${message.clientHost}/incidents/${message.content.incident.id}`, + url: incidentUrl(message), }); } diff --git a/server/src/domain/notifications/providers/telegram.ts b/server/src/domain/notifications/providers/telegram.ts index 72e31b5bf..26b2c2786 100644 --- a/server/src/domain/notifications/providers/telegram.ts +++ b/server/src/domain/notifications/providers/telegram.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "TelegramProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class TelegramProvider extends NotificationProvider { @@ -101,7 +101,7 @@ export class TelegramProvider extends NotificationProvider { if (message.content.incident) { lines.push(""); - lines.push(`View Incident`); + lines.push(`View Incident`); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/utils.ts b/server/src/domain/notifications/providers/utils.ts index 3eaf428fe..4ac4d96fc 100644 --- a/server/src/domain/notifications/providers/utils.ts +++ b/server/src/domain/notifications/providers/utils.ts @@ -1,4 +1,8 @@ import { IEmailService } from "@/service/emailService.js"; +import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; + +export const incidentUrl = (message: NotificationMessage): string => `${message.clientHost}/incidents/${message.monitor.id}`; + export const buildTestEmail = async (emailService: IEmailService) => { const context = { testName: "Monitoring System" }; const html = await emailService.buildEmail("testEmailTemplate", context); diff --git a/server/src/domain/notifications/providers/webhook.ts b/server/src/domain/notifications/providers/webhook.ts index fc5a80fde..64fbb26e9 100644 --- a/server/src/domain/notifications/providers/webhook.ts +++ b/server/src/domain/notifications/providers/webhook.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "WebhookProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class WebhookProvider extends NotificationProvider { @@ -74,7 +74,7 @@ export class WebhookProvider extends NotificationProvider { // Incident link if (message.content.incident) { - lines.push(`[View Incident](${message.clientHost}/infrastructure/${message.monitor.id})`); + lines.push(`[View Incident](${incidentUrl(message)})`); } // Return webhook payload with both text and structured data diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index 39011a13b..a25e847ca 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -42,6 +42,8 @@ const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]"; const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /; const DOCKER_TLS_DEFAULT_PORT = 2376; const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]); +const EXITED_STATUS = /^Exited \((\d+)\)/; +const HEALTH_STATUS = /\((healthy|unhealthy|health: starting)\)$/; export interface DockerError extends Error { statusCode?: number; @@ -140,6 +142,17 @@ export class DockerProvider implements IStatusProvider { return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none"; } + private exitCodeFromStatus = (status: string): number | undefined => { + const match = EXITED_STATUS.exec(status); + return match ? Number(match[1]) : undefined; + }; + + private healthFromStatus = (status: string): DockerHealthStatus => { + const match = HEALTH_STATUS.exec(status); + if (!match) return "none"; + return match[1] === "health: starting" ? "starting" : (match[1] as DockerHealthStatus); + }; + private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary { return { total: containers.length, @@ -273,7 +286,10 @@ export class DockerProvider implements IStatusProvider { info.health = this.toHealthStatus(inspectResult.value.State?.Health?.Status); info.ports = this.toPorts(inspectResult.value.NetworkSettings?.Ports); info.mounts = this.toMounts(inspectResult.value.Mounts); + info.exitCode = inspectResult.value.State?.ExitCode; } else { + info.exitCode = this.exitCodeFromStatus(summary.Status); + info.health = this.healthFromStatus(summary.Status); this.logger.warn({ message: `Failed to inspect container ${info.name}`, service: SERVICE_NAME, diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 07ddee2a0..13ae68109 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -19,6 +19,7 @@ import { ILogger } from "@/utils/logger.js"; import type { HardwareStatusMetrics } from "@/types/network.js"; import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js"; import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js"; +import { findContainerBreaches } from "@/domain/docker/docker-alert.js"; const SERVICE_NAME = "StatusService"; const HARDWARE_ALERT_COUNTER_START = 5; @@ -162,6 +163,23 @@ export class StatusService implements IStatusService { return { nextStatus, transitioned, breaches, nextCounters }; }; + private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => { + const { currentStatus, reachabilityDown, breaching } = params; + let nextStatus: MonitorStatus = currentStatus; + let transitioned = false; + + if (!reachabilityDown) { + if (breaching && currentStatus !== "breached") { + nextStatus = "breached"; + transitioned = true; + } else if (!breaching && currentStatus === "breached") { + nextStatus = "up"; + transitioned = true; + } + } + return { nextStatus, transitioned }; + }; + updateMonitorStatus = async ( statusResponse: MonitorStatusResponse< | PingStatusPayload @@ -268,6 +286,20 @@ export class StatusService implements IStatusService { } } + const dockerPayload = statusResponse.payload as DockerStatusPayload | undefined; + if (monitor.type === "docker" && dockerPayload?.containers) { + const breaches = findContainerBreaches(monitor, dockerPayload.containers); + const docker = this.computeDockerStatus({ + currentStatus: newStatus, + reachabilityDown: newStatus === "down", + breaching: breaches.length > 0, + }); + if (docker.transitioned) { + newStatus = docker.nextStatus; + statusChanged = true; + } + } + patch.status = newStatus; // Single atomic write: push arrays + set status/counters diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 379040010..d29b95fbb 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -12,7 +12,7 @@ export interface MonitorActionDecision { shouldResolveIncident: boolean; shouldSendNotification: boolean; incidentReason: "status_down" | "threshold_breach" | null; - notificationReason: "status_change" | "threshold_breach" | null; + notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null; thresholdBreaches?: { cpu?: boolean; memory?: boolean; diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index bdcb86c00..658737826 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -326,7 +326,11 @@ export class WorkerPipeline implements IWorkerPipeline { decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; decision.notificationReason = "threshold_breach"; - } else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) { + } else if (monitor.status === "up" && prevStatus === "breached") { + decision.shouldResolveIncident = true; + decision.shouldSendNotification = true; + decision.notificationReason = "threshold_resolved"; + } else if (monitor.status === "up" && prevStatus === "down") { decision.shouldResolveIncident = true; decision.shouldSendNotification = true; decision.notificationReason = "status_change"; diff --git a/server/test/unit/providers/network/dockerProvider.test.ts b/server/test/unit/providers/network/dockerProvider.test.ts index a90e2b65f..50e666a1d 100644 --- a/server/test/unit/providers/network/dockerProvider.test.ts +++ b/server/test/unit/providers/network/dockerProvider.test.ts @@ -229,6 +229,18 @@ describe("DockerProvider", () => { }); }); + it("leaves the exit code undefined for Capture containers", async () => { + const response = makeCaptureResponse({ + payload: { data: [makeCaptureContainer({ status: "exited", running: false, stats: undefined })], errors: [] }, + }); + const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) }); + + const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" })); + + expect(result.payload?.containers[0]?.state).toBe("exited"); + expect(result.payload?.containers[0]?.exitCode).toBeUndefined(); + }); + it("keeps state-derived Capture health separate from Docker healthchecks", async () => { const response = makeCaptureResponse({ payload: { @@ -558,6 +570,16 @@ describe("DockerProvider", () => { ); }); + it("reads the exit code from inspect", async () => { + const inspect = jest.fn().mockResolvedValue(makeInspect({ State: { StartedAt: "2026-08-28T10:00:00.000Z", ExitCode: 137 } })); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status: "Exited (137) 2 minutes ago" })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ state: "exited", exitCode: 137, health: "none" })); + }); + it("counts running, stopped, and unhealthy states in the summary", async () => { const containers = [ makeContainer({ Id: "a".repeat(64) }), @@ -682,11 +704,40 @@ describe("DockerProvider", () => { expect(result.status).toBe(true); expect(result.payload?.containers).toHaveLength(1); - expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ health: "none" })); expect(result.payload?.containers[0]?.restartCount).toBeUndefined(); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Failed to inspect") })); }); + it.each([ + ["Up 3 hours (healthy)", "healthy"], + ["Up 3 hours (unhealthy)", "unhealthy"], + ["Up 5 seconds (health: starting)", "starting"], + ["Up 3 hours", "none"], + ["Exited (1) 2 minutes ago", "none"], + ])("reads health from the list status %s when inspect rejects", async (Status, health) => { + const inspect = jest.fn().mockRejectedValue(new Error("inspect failed")); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ Status })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]?.health).toBe(health); + }); + + it.each([ + ["Exited (0) 2 minutes ago", 0], + ["Exited (137) 2 minutes ago", 137], + ["Up 3 hours", undefined], + ])("reads the exit code from the list status %s when inspect rejects", async (Status, exitCode) => { + const inspect = jest.fn().mockRejectedValue(new Error("inspect failed")); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]?.exitCode).toBe(exitCode); + }); + it("keeps the container without metrics when stats rejects", async () => { const stats = jest.fn().mockRejectedValue(new Error("stats failed")); const { provider } = setup({ stats }); diff --git a/server/test/unit/providers/notifications/emailProvider.test.ts b/server/test/unit/providers/notifications/emailProvider.test.ts index 7f3b34d4e..3610523a4 100644 --- a/server/test/unit/providers/notifications/emailProvider.test.ts +++ b/server/test/unit/providers/notifications/emailProvider.test.ts @@ -115,6 +115,18 @@ describe("EmailProvider", () => { expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor thresholds resolved", expect.anything()); }); + it("builds correct subject for container_breach", async () => { + const { provider, emailService } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" })); + expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor container alert", expect.anything()); + }); + + it("builds correct subject for container_resolved", async () => { + const { provider, emailService } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" })); + expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor containers recovered", expect.anything()); + }); + it("builds default subject for unknown type", async () => { const { provider, emailService } = createProvider(); await provider.sendMessage(makeNotification() as any, makeMessage({ type: "test" })); diff --git a/server/test/unit/providers/notifications/matrixProvider.test.ts b/server/test/unit/providers/notifications/matrixProvider.test.ts index 1294a677b..7cd0ea427 100644 --- a/server/test/unit/providers/notifications/matrixProvider.test.ts +++ b/server/test/unit/providers/notifications/matrixProvider.test.ts @@ -105,6 +105,14 @@ describe("MatrixProvider", () => { expect(body.formatted_body).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + const body = mockGotPut.mock.calls[0][1].json; + expect(body.body).toContain("View Incident: https://app.example.com/incidents/mon-1"); + expect(body.formatted_body).toContain('href="https://app.example.com/incidents/mon-1"'); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/ntfyProvider.test.ts b/server/test/unit/providers/notifications/ntfyProvider.test.ts index d365584a1..bced0b6b3 100644 --- a/server/test/unit/providers/notifications/ntfyProvider.test.ts +++ b/server/test/unit/providers/notifications/ntfyProvider.test.ts @@ -106,7 +106,7 @@ describe("NtfyProvider", () => { it("includes incident links in text", async () => { const { provider } = createProvider(); await provider.sendMessage(makeNtfyNotification() as any, makeMessageWithIncident()); - expect(mockGotPost.mock.calls[0][1].body).toContain("/infrastructure/mon-1"); + expect(mockGotPost.mock.calls[0][1].body).toContain("https://app.example.com/incidents/mon-1"); }); }); diff --git a/server/test/unit/providers/notifications/pagerdutyProvider.test.ts b/server/test/unit/providers/notifications/pagerdutyProvider.test.ts index 8b8d185b4..b17d212ba 100644 --- a/server/test/unit/providers/notifications/pagerdutyProvider.test.ts +++ b/server/test/unit/providers/notifications/pagerdutyProvider.test.ts @@ -89,6 +89,18 @@ describe("PagerDutyProvider", () => { expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve"); }); + it("uses 'resolve' event_action for container_resolved", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" })); + expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve"); + }); + + it("uses 'trigger' event_action for container_breach", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" })); + expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("trigger"); + }); + it("includes threshold info in summary and custom_details", async () => { const { provider } = createProvider(); await provider.sendMessage(makeNotification() as any, makeMessageWithThresholds()); diff --git a/server/test/unit/providers/notifications/pushoverProvider.test.ts b/server/test/unit/providers/notifications/pushoverProvider.test.ts index a227b0b1b..457803131 100644 --- a/server/test/unit/providers/notifications/pushoverProvider.test.ts +++ b/server/test/unit/providers/notifications/pushoverProvider.test.ts @@ -99,6 +99,12 @@ describe("PushoverProvider", () => { expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident: https://app.example.com/incidents/mon-1"); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/rocketChatProvider.test.ts b/server/test/unit/providers/notifications/rocketChatProvider.test.ts index 816e19801..0568c144e 100644 --- a/server/test/unit/providers/notifications/rocketChatProvider.test.ts +++ b/server/test/unit/providers/notifications/rocketChatProvider.test.ts @@ -134,6 +134,7 @@ describe("RocketChatProvider", () => { ["ping", "https://app.example.com/uptime/mon-1"], ["pagespeed", "https://app.example.com/pagespeed/mon-1"], ["hardware", "https://app.example.com/infrastructure/mon-1"], + ["docker", "https://app.example.com/docker/host/mon-1"], ])("links %s monitors to %s", async (type, expectedLink) => { const { provider } = createProvider(); diff --git a/server/test/unit/providers/notifications/signalgridProvider.test.ts b/server/test/unit/providers/notifications/signalgridProvider.test.ts index 8c40bb1d9..b0812509a 100644 --- a/server/test/unit/providers/notifications/signalgridProvider.test.ts +++ b/server/test/unit/providers/notifications/signalgridProvider.test.ts @@ -139,6 +139,12 @@ describe("SignalgridProvider", () => { expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident: https://app.example.com/incidents/mon-1"); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/slackProvider.test.ts b/server/test/unit/providers/notifications/slackProvider.test.ts index 20058ab11..86c17f99a 100644 --- a/server/test/unit/providers/notifications/slackProvider.test.ts +++ b/server/test/unit/providers/notifications/slackProvider.test.ts @@ -96,6 +96,14 @@ describe("SlackProvider", () => { expect(text).toContain("Additional Information"); }); + it("links the incident button to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + const blocks = mockGotPost.mock.calls[0][1].json.blocks; + const actions = blocks.find((b: any) => b.type === "actions"); + expect(actions.elements[0].url).toBe("https://app.example.com/incidents/mon-1"); + }); + it("omits threshold, details, and incident sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/teamsProvider.test.ts b/server/test/unit/providers/notifications/teamsProvider.test.ts index c3766ec16..514190fe5 100644 --- a/server/test/unit/providers/notifications/teamsProvider.test.ts +++ b/server/test/unit/providers/notifications/teamsProvider.test.ts @@ -154,7 +154,7 @@ describe("TeamsProvider", () => { expect(card.actions).toHaveLength(1); expect(card.actions[0].type).toBe("Action.OpenUrl"); expect(card.actions[0].title).toBe("View Incident"); - expect(card.actions[0].url).toContain("/incidents/inc-1"); + expect(card.actions[0].url).toBe("https://app.example.com/incidents/mon-1"); }); it("omits actions when no incident is present", async () => { diff --git a/server/test/unit/providers/notifications/telegramProvider.test.ts b/server/test/unit/providers/notifications/telegramProvider.test.ts index f8f853e62..8fe90c412 100644 --- a/server/test/unit/providers/notifications/telegramProvider.test.ts +++ b/server/test/unit/providers/notifications/telegramProvider.test.ts @@ -99,6 +99,12 @@ describe("TelegramProvider", () => { expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].json.text).toContain('View Incident'); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/webhookProvider.test.ts b/server/test/unit/providers/notifications/webhookProvider.test.ts index fd16f2c06..659a32e2f 100644 --- a/server/test/unit/providers/notifications/webhookProvider.test.ts +++ b/server/test/unit/providers/notifications/webhookProvider.test.ts @@ -86,6 +86,12 @@ describe("WebhookProvider", () => { expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].json.text).toContain("[View Incident](https://app.example.com/incidents/mon-1)"); + }); + it("omits threshold and incident sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/services/checkService.test.ts b/server/test/unit/services/checkService.test.ts index 4ced65947..230857b0e 100644 --- a/server/test/unit/services/checkService.test.ts +++ b/server/test/unit/services/checkService.test.ts @@ -3,7 +3,7 @@ import { CheckService } from "../../../src/domain/checks/check.service.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; @@ -357,6 +357,34 @@ describe("CheckService", () => { expect(fromRaw[0].metric).toBe("cpu"); expect(fromRebuilt).toEqual(fromRaw); }); + + it("docker: preserves payload.containers and payload.summary verbatim", () => { + const { service } = createService(); + const payload: DockerStatusPayload = { + containers: [ + { id: "1", name: "db", image: "postgres:16", state: "exited", status: "Exited (1) 2 minutes ago", health: "none", exitCode: 1 }, + { id: "2", name: "api", image: "api:latest", state: "running", status: "Up 3 hours (unhealthy)", health: "unhealthy", restartCount: 2 }, + ], + summary: { total: 2, running: 1, stopped: 1, unhealthy: 1 }, + }; + const status = makeStatusResponse({ type: "docker", status: true, payload } as any); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + const roundtripped = rebuilt.payload as DockerStatusPayload; + expect(roundtripped.containers).toEqual(payload.containers); + expect(roundtripped.summary).toEqual(payload.summary); + expect(roundtripped.containers[0].exitCode).toBe(1); + }); + + it("docker: leaves payload undefined for a failed check with no containers", () => { + const { service } = createService(); + const status = makeStatusResponse({ type: "docker", status: false, code: 5000, message: "connect ECONNREFUSED", payload: null } as any); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + expect(rebuilt.payload).toBeUndefined(); + }); }); // ── createChecks ───────────────────────────────────────────────────────── diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index 73a422cf7..a2e161769 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -175,6 +175,46 @@ describe("IncidentService", () => { expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" })); }); + it("builds the container list for docker threshold_breach incidents", async () => { + const created = makeIncident(); + const { service, incidentsRepository, notificationMessageBuilder } = createService(); + (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); + (incidentsRepository.create as jest.Mock).mockResolvedValue(created); + + const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }); + const response = { + monitorId: "mon-1", + payload: { + containers: [ + { id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 }, + { id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" }, + { id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" }, + ], + summary: { total: 3, running: 2, stopped: 1, unhealthy: 1 }, + }, + } as any; + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + await service.handleIncident(monitor, 200, decision, response); + + expect(incidentsRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" }) + ); + expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled(); + }); + + it("uses the container fallback message for docker incidents with no response", async () => { + const created = makeIncident(); + const { service, incidentsRepository } = createService(); + (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); + (incidentsRepository.create as jest.Mock).mockResolvedValue(created); + + const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true }); + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + await service.handleIncident(monitor, 200, decision, undefined); + + expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 9999, message: "Container alert" })); + }); + it("joins multiple threshold breaches with commas", async () => { const created = makeIncident(); const { service, incidentsRepository, notificationMessageBuilder } = createService(); diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index b0b8397e3..afa6519e9 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, beforeEach } from "@jest/globals"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -51,6 +51,29 @@ const makeHardwarePayload = (overrides?: Partial) }, }) as HardwareStatusPayload; +const makeDockerPayload = (containers: Partial[]): DockerStatusPayload => ({ + containers: containers.map((container, index) => ({ + id: `container-${index}`, + name: `container-${index}`, + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...container, + })), + summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 }, +}); + +const makeDockerMonitor = (overrides?: Partial): Monitor => + makeMonitor({ + type: "docker", + status: "breached", + url: "unix:///var/run/docker.sock", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + ...overrides, + }); + // ── Tests ──────────────────────────────────────────────────────────────────── describe("NotificationMessageBuilder", () => { @@ -123,7 +146,7 @@ describe("NotificationMessageBuilder", () => { it("builds a threshold_resolved message for hardware monitor recovering", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); - const decision = makeDecision({ notificationReason: "status_change" }); + const decision = makeDecision({ notificationReason: "threshold_resolved" }); const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com"); @@ -163,13 +186,46 @@ describe("NotificationMessageBuilder", () => { expect(msg.type).toBe("threshold_breach"); }); - it("returns threshold_resolved for hardware monitor with status_change and up", () => { + it("returns threshold_resolved for hardware monitor with threshold_resolved reason", () => { + const monitor = makeMonitor({ status: "up", type: "hardware" }); + const decision = makeDecision({ notificationReason: "threshold_resolved" }); + + const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("threshold_resolved"); + }); + + it("returns monitor_up for hardware monitor recovering from down with status_change", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); const decision = makeDecision({ notificationReason: "status_change" }); const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); - expect(msg.type).toBe("threshold_resolved"); + expect(msg.type).toBe("monitor_up"); + }); + + it("returns container_breach for docker monitor with threshold_breach reason", () => { + const decision = makeDecision({ notificationReason: "threshold_breach" }); + + const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("container_breach"); + }); + + it("returns container_resolved for docker monitor with threshold_resolved reason", () => { + const decision = makeDecision({ notificationReason: "threshold_resolved" }); + + const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("container_resolved"); + }); + + it("returns monitor_down for docker monitor that is down regardless of reason", () => { + const decision = makeDecision({ notificationReason: "threshold_breach" }); + + const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("monitor_down"); }); it("returns monitor_up for non-hardware monitor with status up", () => { @@ -223,7 +279,22 @@ describe("NotificationMessageBuilder", () => { const msg = builder.buildMessage( makeMonitor({ status: "up", type: "hardware" }), makeStatusResponse(), - makeDecision({ notificationReason: "status_change" }), + makeDecision({ notificationReason: "threshold_resolved" }), + "" + ); + expect(msg.severity).toBe("success"); + }); + + it("returns warning for container_breach", () => { + const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), makeDecision({ notificationReason: "threshold_breach" }), ""); + expect(msg.severity).toBe("warning"); + }); + + it("returns success for container_resolved", () => { + const msg = builder.buildMessage( + makeDockerMonitor({ status: "up" }), + makeStatusResponse(), + makeDecision({ notificationReason: "threshold_resolved" }), "" ); expect(msg.severity).toBe("success"); @@ -311,7 +382,7 @@ describe("NotificationMessageBuilder", () => { describe("threshold_resolved", () => { it("includes resolved details", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision({ notificationReason: "threshold_resolved" }), ""); expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor"); expect(msg.content.summary).toBe('Monitor "Test Monitor" thresholds have returned to normal.'); @@ -319,6 +390,63 @@ describe("NotificationMessageBuilder", () => { }); }); + describe("container_breach", () => { + it("lists one detail line per breaching container", () => { + const response = makeStatusResponse({ + type: "docker", + status: true, + payload: makeDockerPayload([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]), + } as any); + + const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.title).toBe("Container Alert: Test Monitor"); + expect(msg.content.summary).toBe('2 container(s) on "Test Monitor" need attention.'); + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "db: stopped (exit code 1)", "api: unhealthy"]); + expect(msg.content.thresholds).toBeUndefined(); + }); + + it("only lists containers matching an enabled switch", () => { + const response = makeStatusResponse({ + type: "docker", + status: true, + payload: makeDockerPayload([ + { name: "db", state: "exited", exitCode: 1 }, + { name: "api", health: "unhealthy" }, + ]), + } as any); + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); + + const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "api: unhealthy"]); + }); + + it("lists no containers when the payload is an error string", () => { + const response = makeStatusResponse({ type: "docker", status: false, payload: "connect ECONNREFUSED" } as any); + + const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.summary).toBe('0 container(s) on "Test Monitor" need attention.'); + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker"]); + }); + }); + + describe("container_resolved", () => { + it("includes recovered details", () => { + const msg = builder.buildMessage( + makeDockerMonitor({ status: "up" }), + makeStatusResponse(), + makeDecision({ notificationReason: "threshold_resolved" }), + "" + ); + + expect(msg.content.title).toBe("Containers Recovered: Test Monitor"); + expect(msg.content.summary).toBe('All containers on "Test Monitor" are back to normal.'); + expect(msg.content.details).toContain("Status: Up"); + }); + }); + describe("default content", () => { it("builds default content for unhandled notification type", () => { // Force default by creating a scenario where type falls through diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 071465d48..393cb5676 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts"; import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; @@ -1086,6 +1086,162 @@ describe("StatusService", () => { // they're private implementation details; exposing them publicly would // widen the API surface just for testability. + describe("docker container alerts", () => { + const makeDockerMonitor = (overrides?: Partial) => + makeMonitor({ + type: "docker", + url: "unix:///var/run/docker.sock", + statusWindow: [true, true, true, true], + statusWindowSize: 5, + status: "up", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + ...overrides, + }); + + const makeContainers = (containers: Partial[]): DockerStatusPayload => ({ + containers: containers.map((container, index) => ({ + id: `container-${index}`, + name: `container-${index}`, + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...container, + })), + summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 }, + }); + + const makeDockerResponse = (payload: DockerStatusPayload | null, overrides?: Partial>) => + makeStatusResponse({ + type: "docker", + status: true, + payload, + ...overrides, + } as any) as MonitorStatusResponse; + + const run = async (monitor: Monitor, response: MonitorStatusResponse, check = makeCheck()) => { + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + const result = await service.updateMonitorStatus(response, check, monitor); + return { result, monitorsRepository }; + }; + + const stopped = { name: "db", state: "exited" as const, exitCode: 1 }; + const unhealthy = { name: "api", health: "unhealthy" as const }; + + it("goes to breached on the first check with a stopped container", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("breached"); + }); + + it("goes to breached on the first check with an unhealthy container", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([unhealthy]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("breached"); + }); + + it("stays up on a stopped container when both switches are off", async () => { + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped, unhealthy]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays up on a stopped container when only the unhealthy switch is on", async () => { + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays up on an exited container with code 0", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "job", state: "exited", exitCode: 0 }]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays breached without a transition while a container is still down", async () => { + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("breached"); + }); + + it("stays up without a transition while every container is clean", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "web" }]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("recovers from breached to up on the first clean check", async () => { + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([{ name: "web" }]))); + + expect(result.statusChanged).toBe(true); + expect(result.prevStatus).toBe("breached"); + expect(result.monitor.status).toBe("up"); + }); + + it("recovers from breached to up when the breaching switch is turned off", async () => { + const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("up"); + }); + + it("does not override down with breached when the host is unreachable", async () => { + const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.monitor.status).toBe("down"); + }); + + it("transitions from breached to down when the reachability threshold trips", async () => { + const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("down"); + }); + + it("does not evaluate containers while the status window is short", async () => { + const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("does not evaluate a failed check with a null payload", async () => { + const monitor = makeDockerMonitor({ status: "breached" }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("breached"); + }); + + it("does not touch the hardware counters or thresholdBreaches", async () => { + const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + + expect(result.thresholdBreaches).toBeUndefined(); + const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; + expect(patch.cpuAlertCounter).toBeUndefined(); + expect(patch.status).toBe("breached"); + }); + }); + describe("computeReachability (pure)", () => { const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => { const { service } = createService(); diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index e7c117152..0b168ef24 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -861,9 +861,14 @@ describe("WorkerPipeline", () => { }); }); - it("resolves incident when monitor recovers from breached", async () => { + it("resolves incident with threshold_resolved when monitor recovers from breached", async () => { const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" })); - expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); + expect(decision).toMatchObject({ + shouldResolveIncident: true, + shouldSendNotification: true, + shouldCreateIncident: false, + notificationReason: "threshold_resolved", + }); }); it("does not create or resolve for unhandled status transitions", async () => { diff --git a/server/test/unit/utils/dockerAlert.test.ts b/server/test/unit/utils/dockerAlert.test.ts new file mode 100644 index 000000000..a9d360511 --- /dev/null +++ b/server/test/unit/utils/dockerAlert.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "@jest/globals"; +import { describeContainerBreach, findContainerBreaches, isContainerStopped, isContainerUnhealthy } from "../../../src/domain/docker/docker-alert.ts"; +import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; + +const makeContainer = (overrides?: Partial): DockerContainerInfo => ({ + id: "abc123", + name: "web", + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...overrides, +}); + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "mon-1", + type: "docker", + dockerAlertOnStopped: false, + dockerAlertOnUnhealthy: false, + ...overrides, + }) as Monitor; + +describe("isContainerStopped", () => { + it("treats dead as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "dead" }))).toBe(true); + }); + + it.each([1, 137, 143])("treats exited with code %s as stopped", (exitCode) => { + expect(isContainerStopped(makeContainer({ state: "exited", exitCode }))).toBe(true); + }); + + it("does not treat exited with code 0 as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "exited", exitCode: 0 }))).toBe(false); + }); + + it("treats exited with an unknown exit code as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "exited" }))).toBe(true); + }); + + it.each(["running", "restarting", "paused", "created", "removing"] as const)("does not treat %s as stopped", (state) => { + expect(isContainerStopped(makeContainer({ state, exitCode: 1 }))).toBe(false); + }); +}); + +describe("isContainerUnhealthy", () => { + it("treats unhealthy as unhealthy", () => { + expect(isContainerUnhealthy(makeContainer({ health: "unhealthy" }))).toBe(true); + }); + + it.each(["healthy", "starting", "none"] as const)("does not treat %s as unhealthy", (health) => { + expect(isContainerUnhealthy(makeContainer({ health }))).toBe(false); + }); +}); + +describe("findContainerBreaches", () => { + const stopped = makeContainer({ name: "db", state: "exited", exitCode: 1 }); + const unhealthy = makeContainer({ name: "api", health: "unhealthy" }); + + it("returns nothing when both switches are off", () => { + expect(findContainerBreaches(makeMonitor(), [stopped, unhealthy])).toEqual([]); + }); + + it("reports only stopped containers when only the stopped switch is on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true }), [stopped, unhealthy]); + expect(breaches).toEqual([{ name: "db", reason: "stopped", exitCode: 1 }]); + }); + + it("reports only unhealthy containers when only the unhealthy switch is on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnUnhealthy: true }), [stopped, unhealthy]); + expect(breaches).toEqual([{ name: "api", reason: "unhealthy" }]); + }); + + it("reports both conditions when both switches are on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [stopped, unhealthy]); + expect(breaches.map((b) => b.reason)).toEqual(["stopped", "unhealthy"]); + }); + + it("reports a container that is both stopped and unhealthy once, as stopped", () => { + const both = makeContainer({ name: "worker", state: "dead", health: "unhealthy" }); + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [both]); + expect(breaches).toEqual([{ name: "worker", reason: "stopped", exitCode: undefined }]); + }); + + it("ignores healthy running containers", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [makeContainer()]); + expect(breaches).toEqual([]); + }); +}); + +describe("describeContainerBreach", () => { + it("describes an unhealthy container", () => { + expect(describeContainerBreach({ name: "api", reason: "unhealthy" })).toBe("api: unhealthy"); + }); + + it("describes a stopped container with its exit code", () => { + expect(describeContainerBreach({ name: "db", reason: "stopped", exitCode: 137 })).toBe("db: stopped (exit code 137)"); + }); + + it("describes a stopped container without an exit code", () => { + expect(describeContainerBreach({ name: "db", reason: "stopped" })).toBe("db: stopped"); + }); +}); diff --git a/server/test/unit/validation/monitorValidation.test.ts b/server/test/unit/validation/monitorValidation.test.ts index 218a0942d..63f3572a8 100644 --- a/server/test/unit/validation/monitorValidation.test.ts +++ b/server/test/unit/validation/monitorValidation.test.ts @@ -247,6 +247,20 @@ describe("monitorValidation — strategy gating", () => { expect(parsed.monitors[0].dockerLogsEnabled).toBe(false); }); + it("defaults the docker alert switches to false on imported docker monitors", () => { + const parsed = importMonitorsBodyValidation.parse({ + monitors: [ + { + name: "Imported Docker", + type: "docker", + url: "unix:///var/run/docker.sock", + }, + ], + }); + expect(parsed.monitors[0].dockerAlertOnStopped).toBe(false); + expect(parsed.monitors[0].dockerAlertOnUnhealthy).toBe(false); + }); + it("rejects strategy on imported non-pagespeed monitors", () => { expect(() => importMonitorsBodyValidation.parse({ @@ -818,3 +832,43 @@ describe("monitorValidation — Docker TLS credentials", () => { }); }); }); + +describe("monitorValidation — docker alert switches", () => { + describe("createMonitorBodyValidation", () => { + it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on a docker monitor", () => { + const parsed = createMonitorBodyValidation.parse({ + name: "Docker host", + type: "docker", + url: "unix:///var/run/docker.sock", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + }); + + expect(parsed.dockerAlertOnStopped).toBe(true); + expect(parsed.dockerAlertOnUnhealthy).toBe(true); + }); + + it("leaves the switches undefined when omitted", () => { + const parsed = createMonitorBodyValidation.parse({ + name: "Docker host", + type: "docker", + url: "unix:///var/run/docker.sock", + }); + + expect(parsed.dockerAlertOnStopped).toBeUndefined(); + expect(parsed.dockerAlertOnUnhealthy).toBeUndefined(); + }); + }); + + describe("editMonitorBodyValidation", () => { + it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on edits", () => { + const parsed = editMonitorBodyValidation.parse({ + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: false, + }); + + expect(parsed.dockerAlertOnStopped).toBe(true); + expect(parsed.dockerAlertOnUnhealthy).toBe(false); + }); + }); +});