From bdd4634a3bbe44f11814f1fd718f4bee77a20a0d Mon Sep 17 00:00:00 2001 From: Malena Aguiar Date: Tue, 22 Sep 2026 16:12:52 -0600 Subject: [PATCH 01/29] feat(dashboard): add currently down and lowest uptime dashboard cards --- .../Dashboard/components/EditCardsModal.tsx | 38 +++++++++--- .../components/cards/CurrentlyDownCard.tsx | 59 ++++++++++++++++++ .../components/cards/LowestUptimeCard.tsx | 60 +++++++++++++++++++ client/src/Pages/Dashboard/index.tsx | 4 ++ client/src/Types/Dashboard.ts | 7 ++- client/src/locales/en.json | 10 +++- 6 files changed, 167 insertions(+), 11 deletions(-) create mode 100644 client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx create mode 100644 client/src/Pages/Dashboard/components/cards/LowestUptimeCard.tsx diff --git a/client/src/Pages/Dashboard/components/EditCardsModal.tsx b/client/src/Pages/Dashboard/components/EditCardsModal.tsx index 84257c1ec..faf13c407 100644 --- a/client/src/Pages/Dashboard/components/EditCardsModal.tsx +++ b/client/src/Pages/Dashboard/components/EditCardsModal.tsx @@ -1,8 +1,11 @@ -import { useState } from "react"; +import { useMemo, useState } from "react"; import Stack from "@mui/material/Stack"; import FormControlLabel from "@mui/material/FormControlLabel"; +import Typography from "@mui/material/Typography"; +import { useTheme } from "@mui/material/styles"; import { useTranslation } from "react-i18next"; import { Checkbox, Dialog } from "@/Components/inputs"; +import { LAYOUT } from "@/Utils/Theme/constants"; import { type DashboardCardKey, dashboardCardKeys } from "@/Types/Dashboard"; interface EditCardsModalProps { @@ -18,16 +21,19 @@ export const EditCardsModal = ({ onClose, onSave, }: EditCardsModalProps) => { + const theme = useTheme(); const { t } = useTranslation(); - const [draft, setDraft] = useState>(new Set(visibleCards)); + const [cardsSelection, setCardsSelection] = useState>( + new Set(visibleCards) + ); const handleClose = () => { - setDraft(new Set(visibleCards)); + setCardsSelection(new Set(visibleCards)); onClose(); }; const handleToggle = (key: DashboardCardKey) => { - setDraft((prev) => { + setCardsSelection((prev) => { const next = new Set(prev); if (next.has(key)) { next.delete(key); @@ -38,26 +44,40 @@ export const EditCardsModal = ({ }); }; + const sortedCardKeys = useMemo( + () => + [...dashboardCardKeys].sort((a, b) => + t(`pages.dashboard.cards.${a}`).localeCompare(t(`pages.dashboard.cards.${b}`)) + ), + [t] + ); + return ( onSave(draft)} + onConfirm={() => onSave(cardsSelection)} confirmText={t("pages.dashboard.editCardsModal.done")} maxWidth="xs" fullWidth > - - {dashboardCardKeys.map((key: DashboardCardKey) => ( + + + {t("pages.dashboard.editCardsModal.groups.uptime")} + + {sortedCardKeys.map((key: DashboardCardKey) => ( handleToggle(key)} /> } diff --git a/client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx b/client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx new file mode 100644 index 000000000..478b2efa3 --- /dev/null +++ b/client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx @@ -0,0 +1,59 @@ +import Stack from "@mui/material/Stack"; +import Typography from "@mui/material/Typography"; +import { useTheme } from "@mui/material/styles"; +import { useTranslation } from "react-i18next"; +import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard"; +import { Dot } from "@/Components/design-elements"; +import { getStatusColor } from "@/Utils/MonitorUtils"; +import { typographyLevels } from "@/Utils/Theme/Palette"; +import { LAYOUT, SPACING } from "@/Utils/Theme/constants"; +import type { Monitor } from "@/Types/Monitor"; + +interface CurrentlyDownCardProps { + monitors: Monitor[]; +} + +export const CurrentlyDownCard = ({ monitors }: CurrentlyDownCardProps) => { + const theme = useTheme(); + const { t } = useTranslation(); + + const downMonitors = monitors.filter((monitor) => monitor.status === "down"); + + return ( + 0 ? downMonitors.length : undefined} + > + {downMonitors.length === 0 ? ( + + {t("pages.dashboard.currentlyDown.empty")} + + ) : ( + + {downMonitors.map((monitor) => ( + + + + {monitor.name} + + + ))} + + )} + + ); +}; diff --git a/client/src/Pages/Dashboard/components/cards/LowestUptimeCard.tsx b/client/src/Pages/Dashboard/components/cards/LowestUptimeCard.tsx new file mode 100644 index 000000000..f8840081e --- /dev/null +++ b/client/src/Pages/Dashboard/components/cards/LowestUptimeCard.tsx @@ -0,0 +1,60 @@ +import Stack from "@mui/material/Stack"; +import Typography from "@mui/material/Typography"; +import { useTheme } from "@mui/material/styles"; +import { useTranslation } from "react-i18next"; +import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard"; +import { BarRow } from "@/Pages/Dashboard/components/cards/BarRow"; +import { getUptimePercentageColor } from "@/Utils/MonitorUtils"; +import { formatPercentage } from "@/Utils/FormatUtils"; +import { typographyLevels } from "@/Utils/Theme/Palette"; +import { SPACING } from "@/Utils/Theme/constants"; +import type { Monitor } from "@/Types/Monitor"; + +const TOP_N = 5; + +interface LowestUptimeCardProps { + monitors: Monitor[]; +} + +export const LowestUptimeCard = ({ monitors }: LowestUptimeCardProps) => { + const theme = useTheme(); + const { t } = useTranslation(); + + const ranked = monitors + .filter((monitor) => monitor.status !== "paused") + .filter( + (monitor): monitor is Monitor & { uptimePercentage: number } => + typeof monitor.uptimePercentage === "number" + ) + .sort((a, b) => a.uptimePercentage - b.uptimePercentage) + .slice(0, TOP_N); + + return ( + + {ranked.length === 0 ? ( + + {t("pages.dashboard.noData")} + + ) : ( + + {ranked.map((monitor) => { + const paletteKey = getUptimePercentageColor(monitor.uptimePercentage); + const color = theme.palette[paletteKey].main; + return ( + + ); + })} + + )} + + ); +}; diff --git a/client/src/Pages/Dashboard/index.tsx b/client/src/Pages/Dashboard/index.tsx index f86b224f1..1656b9709 100644 --- a/client/src/Pages/Dashboard/index.tsx +++ b/client/src/Pages/Dashboard/index.tsx @@ -14,6 +14,8 @@ import { setDashboardVisibleCards } from "@/Features/UI/uiSlice"; import type { RootState, AppDispatch } from "@/store"; import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard"; +import { CurrentlyDownCard } from "@/Pages/Dashboard/components/cards/CurrentlyDownCard"; +import { LowestUptimeCard } from "@/Pages/Dashboard/components/cards/LowestUptimeCard"; import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard"; import { EditCardsModal } from "@/Pages/Dashboard/components/EditCardsModal"; import { type DashboardCardKey, dashboardCardKeys } from "@/Types/Dashboard"; @@ -48,6 +50,8 @@ const Dashboard = () => { const cardComponents: Record = { monitorStatus: , + currentlyDown: , + lowestUptime: , monitorsByType: , }; diff --git a/client/src/Types/Dashboard.ts b/client/src/Types/Dashboard.ts index a397ec807..6e418f5ed 100644 --- a/client/src/Types/Dashboard.ts +++ b/client/src/Types/Dashboard.ts @@ -1,2 +1,7 @@ -export const dashboardCardKeys = ["monitorStatus", "monitorsByType"] as const; +export const dashboardCardKeys = [ + "monitorStatus", + "currentlyDown", + "lowestUptime", + "monitorsByType", +] as const; export type DashboardCardKey = (typeof dashboardCardKeys)[number]; diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 1b1337dd2..ba0e3e897 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -1924,16 +1924,24 @@ }, "cards": { "monitorStatus": "Monitor status", + "currentlyDown": "Currently down", + "lowestUptime": "Lowest uptime", "monitorsByType": "Monitors by type" }, + "currentlyDown": { + "empty": "No monitors are down at this moment." + }, "header": { "title": "Dashboard", "description": "See every monitor type in one place and spot what needs attention." }, "editCards": "Edit cards", "editCardsModal": { - "title": "Edit cards", + "title": "Dashboard cards", "description": "Select the cards to display on your dashboard.", + "groups": { + "uptime": "Uptime" + }, "done": "Done" } } From a75bb5fae62ae1d303cbecc27836b67c5fbe0cac Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 10:16:49 -0700 Subject: [PATCH 02/29] add docker container alert fields to monitors,checks --- server/src/api/validation/monitorValidation.ts | 4 ++++ server/src/domain/checks/check.model.ts | 1 + server/src/domain/checks/check.service.ts | 16 ++++++++++++---- server/src/domain/docker/docker.type.ts | 1 + server/src/domain/monitors/monitor.model.ts | 8 ++++++++ .../domain/monitors/monitor.repository.mongo.ts | 2 ++ server/src/domain/monitors/monitor.type.ts | 2 ++ server/src/service/network/DockerProvider.ts | 16 ++++++++++++++++ 8 files changed, 46 insertions(+), 4 deletions(-) diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index 651cf786d..357cb4aaa 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -220,6 +220,8 @@ export const createMonitorBodyValidation = z geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), dockerLogsEnabled: z.boolean().optional(), + dockerAlertOnStopped: z.boolean().optional(), + dockerAlertOnUnhealthy: z.boolean().optional(), dockerTlsCa: z.union([z.string(), z.literal("")]).optional(), dockerTlsCert: z.union([z.string(), z.literal("")]).optional(), dockerTlsKey: z.union([z.string(), z.literal("")]).optional(), @@ -350,6 +352,8 @@ const importedMonitorSchema = z geoCheckLocations: z.array(z.enum(GeoContinents)).default([]), geoCheckInterval: z.number().min(300000).default(300000), dockerLogsEnabled: z.boolean().default(false), + dockerAlertOnStopped: z.boolean().default(false), + dockerAlertOnUnhealthy: z.boolean().default(false), dnsServer: dnsServerValidation.optional(), dnsRecordType: z.enum(DnsRecordTypes).optional(), createdAt: z.string().optional(), diff --git a/server/src/domain/checks/check.model.ts b/server/src/domain/checks/check.model.ts index 1f92fce3c..a4e202d40 100644 --- a/server/src/domain/checks/check.model.ts +++ b/server/src/domain/checks/check.model.ts @@ -220,6 +220,7 @@ const dockerContainerSchema = new Schema( startedAt: { type: String }, ports: { type: [dockerContainerPortSchema], default: undefined }, mounts: { type: [dockerContainerMountSchema], default: undefined }, + exitCode: { type: Number, default: undefined }, }, { _id: false } ); diff --git a/server/src/domain/checks/check.service.ts b/server/src/domain/checks/check.service.ts index 2bc9f2a16..30db653e6 100644 --- a/server/src/domain/checks/check.service.ts +++ b/server/src/domain/checks/check.service.ts @@ -135,6 +135,17 @@ export class CheckService implements ICheckService { return check; }; + private toPayload = (check: Check): HardwareStatusPayload | DockerStatusPayload | undefined => { + switch (check.metadata.type) { + case "hardware": + return { data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload; + case "docker": + return check.containers ? { containers: check.containers, summary: check.containerSummary } : undefined; + default: + return undefined; + } + }; + toStatusResponse = (check: Check) => { const statusResponse: MonitorStatusResponse = { monitorId: check.metadata.monitorId, @@ -145,10 +156,7 @@ export class CheckService implements ICheckService { message: check.message, responseTime: check.responseTime, // re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data - payload: - check.metadata.type === "hardware" - ? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload) - : undefined, + payload: this.toPayload(check), }; return statusResponse; }; diff --git a/server/src/domain/docker/docker.type.ts b/server/src/domain/docker/docker.type.ts index 66504e273..da2d51337 100644 --- a/server/src/domain/docker/docker.type.ts +++ b/server/src/domain/docker/docker.type.ts @@ -55,6 +55,7 @@ export interface DockerContainerInfo { startedAt?: string; // ISO date ports?: DockerContainerPort[]; mounts?: DockerContainerMount[]; + exitCode?: number; } export interface DockerContainerSummary { diff --git a/server/src/domain/monitors/monitor.model.ts b/server/src/domain/monitors/monitor.model.ts index c04107651..94a0fd676 100644 --- a/server/src/domain/monitors/monitor.model.ts +++ b/server/src/domain/monitors/monitor.model.ts @@ -307,6 +307,14 @@ const MonitorSchema = new Schema( type: Boolean, default: false, }, + dockerAlertOnStopped: { + type: Boolean, + default: false, + }, + dockerAlertOnUnhealthy: { + type: Boolean, + default: false, + }, dockerTlsCa: { type: String, }, diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index 385dcebab..882bd5ccb 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -468,6 +468,8 @@ class MongoMonitorsRepository implements IMonitorsRepository { geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, dockerLogsEnabled: doc.dockerLogsEnabled ?? false, + dockerAlertOnStopped: doc.dockerAlertOnStopped ?? false, + dockerAlertOnUnhealthy: doc.dockerAlertOnUnhealthy ?? false, dockerTlsCa: doc.dockerTlsCa ?? undefined, dockerTlsCert: doc.dockerTlsCert ?? undefined, dockerTlsKeySet: doc.dockerTlsKeySet ?? false, diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index d5519af0e..38c0338eb 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -134,6 +134,8 @@ export interface Monitor { geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; dockerLogsEnabled?: boolean; + dockerAlertOnStopped?: boolean; + dockerAlertOnUnhealthy?: boolean; dockerTlsCa?: string; dockerTlsCert?: string; // EncryptionService ciphertext. Normally excluded diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index 39011a13b..fa5f3c9e5 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -42,6 +42,8 @@ const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]"; const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /; const DOCKER_TLS_DEFAULT_PORT = 2376; const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]); +const EXITED_STATUS = /^Exited \((\d+)\)/; +const HEALTH_STATUS = /\((healthy|unhealthy|health: starting)\)$/; export interface DockerError extends Error { statusCode?: number; @@ -140,6 +142,17 @@ export class DockerProvider implements IStatusProvider { return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none"; } + private exitCodeFromStatus = (status: string): number | undefined => { + const match = EXITED_STATUS.exec(status); + return match ? Number(match[1]) : undefined; + }; + + private healthFromStatus = (status: string): DockerHealthStatus => { + const match = HEALTH_STATUS.exec(status); + if (!match) return "none"; + return match[1] === "health: starting" ? "starting" : match[1]; + }; + private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary { return { total: containers.length, @@ -273,7 +286,10 @@ export class DockerProvider implements IStatusProvider { info.health = this.toHealthStatus(inspectResult.value.State?.Health?.Status); info.ports = this.toPorts(inspectResult.value.NetworkSettings?.Ports); info.mounts = this.toMounts(inspectResult.value.Mounts); + info.exitCode = inspectResult.value.State.ExitCode; } else { + info.exitCode = this.exitCodeFromStatus(summary.Status); + info.health = this.healthFromStatus(summary.Status); this.logger.warn({ message: `Failed to inspect container ${info.name}`, service: SERVICE_NAME, From 4ea6539a7de4c3be3e5319d6855dbf353216f1ef Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 10:54:35 -0700 Subject: [PATCH 03/29] build notifications for container breaches --- server/src/domain/docker/docker-alert.ts | 47 ++++++++++++++++ .../notification.message-builder.ts | 55 +++++++++++++------ .../domain/notifications/notification.type.ts | 3 +- server/src/service/statusService.ts | 30 ++++++++++ server/src/worker/worker.interface.ts | 2 +- server/src/worker/worker.pipeline.ts | 6 +- 6 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 server/src/domain/docker/docker-alert.ts diff --git a/server/src/domain/docker/docker-alert.ts b/server/src/domain/docker/docker-alert.ts new file mode 100644 index 000000000..45b8075ba --- /dev/null +++ b/server/src/domain/docker/docker-alert.ts @@ -0,0 +1,47 @@ +import type { DockerContainerInfo } from "@/domain/docker/docker.type.js"; +import type { Monitor } from "@/domain/monitors/monitor.type.js"; + +export const DockerAlertReasons = ["stopped", "unhealthy"] as const; +export type DockerAlertReason = (typeof DockerAlertReasons)[number]; + +export interface DockerContainerBreach { + name: string; + reason: DockerAlertReason; + exitCode?: number; +} + +export const isContainerStopped = (container: DockerContainerInfo): boolean => { + // Container with exit code 0 is probably a one shot container don't report stopped + return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0); +}; + +export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => { + return container.health === "unhealthy"; +}; + +export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => { + const breaches: DockerContainerBreach[] = []; + for (const container of containers) { + if (monitor.dockerAlertOnStopped && isContainerStopped(container)) { + breaches.push({ + name: container.name, + reason: "stopped", + exitCode: container.exitCode, + }); + } else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) { + breaches.push({ + name: container.name, + reason: "unhealthy", + }); + } + } + return breaches; +}; + +export const describeContainerBreach = (breach: DockerContainerBreach): string => { + if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`; + if (breach.exitCode === undefined) { + return `${breach.name}: stopped`; + } + return `${breach.name} stopped (exit code ${breach.exitCode})`; +}; diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 641d9527c..5cd78e9d9 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -1,5 +1,5 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; -import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; +import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { NotificationMessage, @@ -8,6 +8,7 @@ import type { ThresholdBreach, NotificationContent, } from "@/domain/notifications/notification.type.js"; +import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; export interface INotificationMessageBuilder { buildMessage( @@ -59,32 +60,29 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return "monitor_down"; } - // Threshold breach (only if not down) - if (decision.notificationReason === "threshold_breach") { - return "threshold_breach"; + const isDocker = monitor.type === "docker"; + switch (decision.notificationReason) { + case "threshold_breach": + return isDocker ? "container_breach" : "threshold_breach"; + case "threshold_resolved": + return isDocker ? "container_resolved" : "threshold_resolved"; + default: + return "monitor_up"; } - - // Recovery from threshold breach (only for hardware monitors) - if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") { - return "threshold_resolved"; - } - - // Standard recovery (up) - if (monitor.status === "up") { - return "monitor_up"; - } - - // Default to monitor_up for any other case - return "monitor_up"; } private determineSeverity(type: NotificationType): NotificationSeverity { switch (type) { + case "monitor_up": + return "success"; case "monitor_down": return "critical"; + case "container_breach": + return "warning"; case "threshold_breach": return "warning"; - case "monitor_up": + case "container_resolved": + return "success"; case "threshold_resolved": return "success"; case "test": @@ -104,6 +102,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); case "threshold_resolved": return this.buildThresholdResolvedContent(monitor); + case "container_breach": + return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); + case "container_resolved": + return this.buildContainersRecoveredContent(monitor); default: return this.buildDefaultContent(monitor); } @@ -174,6 +176,23 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } + private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + const payload = monitorStatusResponse.payload; + const containers = payload && typeof payload !== "string" ? payload.containers : []; + const breaches = findContainerBreaches(monitor, containers); + const title = `Container Alert: ${monitor.name}`; + const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`; + const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)]; + return { title, summary, details, timestamp: new Date() }; + } + + private buildContainersRecoveredContent(monitor: Monitor): NotificationContent { + const title = `Containers Recovered: ${monitor.name}`; + const summary = `All containers on "${monitor.name}" are back to normal.`; + const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`]; + return { title, summary, details, timestamp: new Date() }; + } + private buildDefaultContent(monitor: Monitor): NotificationContent { return { title: `Monitor: ${monitor.name}`, diff --git a/server/src/domain/notifications/notification.type.ts b/server/src/domain/notifications/notification.type.ts index 628633d56..7f2c7df1e 100644 --- a/server/src/domain/notifications/notification.type.ts +++ b/server/src/domain/notifications/notification.type.ts @@ -71,7 +71,8 @@ export interface AlertDiscordPayload { * Part of notification system unification effort */ -export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test"; +export type NotificationType = + "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test"; export type NotificationSeverity = "critical" | "warning" | "info" | "success"; diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 07ddee2a0..cb093c479 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -19,6 +19,7 @@ import { ILogger } from "@/utils/logger.js"; import type { HardwareStatusMetrics } from "@/types/network.js"; import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js"; import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js"; +import { findContainerBreaches } from "@/domain/docker/docker-alert.js"; const SERVICE_NAME = "StatusService"; const HARDWARE_ALERT_COUNTER_START = 5; @@ -162,6 +163,21 @@ export class StatusService implements IStatusService { return { nextStatus, transitioned, breaches, nextCounters }; }; + private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => { + const { currentStatus, reachabilityDown, breaching } = params; + let nextStatus: MonitorStatus = currentStatus; + let transitioned = false; + + if (!reachabilityDown) { + if (breaching && currentStatus !== "breached") { + nextStatus = "breached"; + transitioned = true; + } else if (!breaching && currentStatus === "breached") nextStatus = "up"; + transitioned = true; + } + return { nextStatus, transitioned }; + }; + updateMonitorStatus = async ( statusResponse: MonitorStatusResponse< | PingStatusPayload @@ -268,6 +284,20 @@ export class StatusService implements IStatusService { } } + const dockerPayload = statusResponse.payload as DockerStatusPayload | undefined; + if (monitor.type === "docker" && dockerPayload?.containers) { + const breaches = findContainerBreaches(monitor, dockerPayload.containers); + const docker = this.computeDockerStatus({ + currentStatus: newStatus, + reachabilityDown: newStatus === "down", + breaching: breaches.length > 0, + }); + if (docker.transitioned) { + newStatus = docker.nextStatus; + statusChanged = true; + } + } + patch.status = newStatus; // Single atomic write: push arrays + set status/counters diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 379040010..d29b95fbb 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -12,7 +12,7 @@ export interface MonitorActionDecision { shouldResolveIncident: boolean; shouldSendNotification: boolean; incidentReason: "status_down" | "threshold_breach" | null; - notificationReason: "status_change" | "threshold_breach" | null; + notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null; thresholdBreaches?: { cpu?: boolean; memory?: boolean; diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index bdcb86c00..658737826 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -326,7 +326,11 @@ export class WorkerPipeline implements IWorkerPipeline { decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; decision.notificationReason = "threshold_breach"; - } else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) { + } else if (monitor.status === "up" && prevStatus === "breached") { + decision.shouldResolveIncident = true; + decision.shouldSendNotification = true; + decision.notificationReason = "threshold_resolved"; + } else if (monitor.status === "up" && prevStatus === "down") { decision.shouldResolveIncident = true; decision.shouldSendNotification = true; decision.notificationReason = "status_change"; From cd65488cbd138521d856fa4510e693aa37afff75 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 10:57:49 -0700 Subject: [PATCH 04/29] add docker branch to incident service, update providers --- server/src/domain/incidents/incident.service.ts | 9 ++++++++- server/src/domain/notifications/providers/email.ts | 4 ++++ server/src/domain/notifications/providers/pagerduty.ts | 3 ++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index f8c79cb80..138a3fcac 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -1,6 +1,6 @@ const SERVICE_NAME = "incidentService"; import type { Monitor } from "@/domain/monitors/monitor.type.js"; -import type { MonitorStatusResponse } from "@/types/network.js"; +import type { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { getDateForRange } from "@/utils/dataUtils.js"; import type { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js"; @@ -12,6 +12,7 @@ import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; import type { ILogger } from "@/utils/logger.js"; import { DateRange } from "@/types/query.js"; +import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; export interface IIncidentService { handleIncident( @@ -110,6 +111,12 @@ export class IncidentService implements IIncidentService { return "Threshold breach detected"; } + if (monitor.type === "docker") { + const containers = (monitorStatusResponse?.payload as DockerStatusPayload | undefined)?.containers ?? []; + const breaches = findContainerBreaches(monitor, containers); + return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert"; + } + const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse); if (breaches.length === 0) { diff --git a/server/src/domain/notifications/providers/email.ts b/server/src/domain/notifications/providers/email.ts index 2b5cde245..01bbaa6be 100644 --- a/server/src/domain/notifications/providers/email.ts +++ b/server/src/domain/notifications/providers/email.ts @@ -92,6 +92,10 @@ export class EmailProvider extends NotificationProvider { return `Monitor ${message.monitor.name} threshold exceeded`; case "threshold_resolved": return `Monitor ${message.monitor.name} thresholds resolved`; + case "container_breach": + return `Monitor ${message.monitor.name} container alert`; + case "container_resolved": + return `Monitor ${message.monitor.name} containers recovered`; default: return `Alert: ${message.monitor.name}`; } diff --git a/server/src/domain/notifications/providers/pagerduty.ts b/server/src/domain/notifications/providers/pagerduty.ts index 6cb264c1d..c20fbc85f 100644 --- a/server/src/domain/notifications/providers/pagerduty.ts +++ b/server/src/domain/notifications/providers/pagerduty.ts @@ -75,7 +75,8 @@ export class PagerDutyProvider extends NotificationProvider { private buildPagerDutyPayload(notification: Notification, message: NotificationMessage): AlertPagerDutyPayload { // Map our notification type to PagerDuty event_action - const eventAction = message.type === "monitor_up" || message.type === "threshold_resolved" ? "resolve" : "trigger"; + const eventAction = + message.type === "monitor_up" || message.type === "threshold_resolved" || message.type === "container_resolved" ? "resolve" : "trigger"; // Map severity to PagerDuty severity levels const severityMap: Record = { From 5fb26367ed53cd926a929800da87134607207340 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 11:03:33 -0700 Subject: [PATCH 05/29] reorder docker case --- server/src/domain/docker/docker-alert.ts | 2 +- server/src/domain/incidents/incident.service.ts | 8 ++++---- server/src/service/network/DockerProvider.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/domain/docker/docker-alert.ts b/server/src/domain/docker/docker-alert.ts index 45b8075ba..6045625a7 100644 --- a/server/src/domain/docker/docker-alert.ts +++ b/server/src/domain/docker/docker-alert.ts @@ -43,5 +43,5 @@ export const describeContainerBreach = (breach: DockerContainerBreach): string = if (breach.exitCode === undefined) { return `${breach.name}: stopped`; } - return `${breach.name} stopped (exit code ${breach.exitCode})`; + return `${breach.name}: stopped (exit code ${breach.exitCode})`; }; diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index 138a3fcac..1c249626a 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -107,16 +107,16 @@ export class IncidentService implements IIncidentService { }; private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string { - if (!monitorStatusResponse) { - return "Threshold breach detected"; - } - if (monitor.type === "docker") { const containers = (monitorStatusResponse?.payload as DockerStatusPayload | undefined)?.containers ?? []; const breaches = findContainerBreaches(monitor, containers); return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert"; } + if (!monitorStatusResponse) { + return "Threshold breach detected"; + } + const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse); if (breaches.length === 0) { diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index fa5f3c9e5..238bccede 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -286,7 +286,7 @@ export class DockerProvider implements IStatusProvider { info.health = this.toHealthStatus(inspectResult.value.State?.Health?.Status); info.ports = this.toPorts(inspectResult.value.NetworkSettings?.Ports); info.mounts = this.toMounts(inspectResult.value.Mounts); - info.exitCode = inspectResult.value.State.ExitCode; + info.exitCode = inspectResult.value.State?.ExitCode; } else { info.exitCode = this.exitCodeFromStatus(summary.Status); info.health = this.healthFromStatus(summary.Status); From cfe7d2a3594d7bef05043369266a3033485d36bf Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 11:06:25 -0700 Subject: [PATCH 06/29] add missing braces, fix type error --- server/openapi.json | 52 +++++++++++++++++++ .../src/api/validation/monitorValidation.ts | 4 ++ server/src/service/network/DockerProvider.ts | 2 +- server/src/service/statusService.ts | 6 ++- 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/server/openapi.json b/server/openapi.json index 237bcea29..4cbcdb1f9 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -466,6 +466,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -523,6 +529,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -694,6 +702,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -753,6 +767,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -7186,6 +7202,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -7245,6 +7267,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -7761,6 +7785,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "type": "string" }, @@ -7820,6 +7850,8 @@ "geoCheckLocations", "geoCheckInterval", "dockerLogsEnabled", + "dockerAlertOnStopped", + "dockerAlertOnUnhealthy", "teamId", "userId", "createdAt", @@ -9308,6 +9340,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "anyOf": [ { @@ -10033,6 +10071,14 @@ "type": "boolean", "default": false }, + "dockerAlertOnStopped": { + "type": "boolean", + "default": false + }, + "dockerAlertOnUnhealthy": { + "type": "boolean", + "default": false + }, "dnsServer": { "type": "string", "minLength": 1 @@ -10600,6 +10646,12 @@ "dockerLogsEnabled": { "type": "boolean" }, + "dockerAlertOnStopped": { + "type": "boolean" + }, + "dockerAlertOnUnhealthy": { + "type": "boolean" + }, "dockerTlsCa": { "anyOf": [ { diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index 357cb4aaa..359a2fabf 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -272,6 +272,8 @@ export const editMonitorBodyValidation = z geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), dockerLogsEnabled: z.boolean().optional(), + dockerAlertOnStopped: z.boolean().optional(), + dockerAlertOnUnhealthy: z.boolean().optional(), dockerTlsCa: z.union([z.string(), z.literal("")]).optional(), dockerTlsCert: z.union([z.string(), z.literal("")]).optional(), dockerTlsKey: z.union([z.string(), z.literal("")]).optional(), @@ -444,6 +446,8 @@ export const monitorResponseSchema = z geoCheckLocations: z.array(z.enum(GeoContinents)), geoCheckInterval: z.number(), dockerLogsEnabled: z.boolean(), + dockerAlertOnStopped: z.boolean(), + dockerAlertOnUnhealthy: z.boolean(), dockerTlsCa: z.string().optional(), dockerTlsCert: z.string().optional(), dockerTlsKeySet: z.boolean().optional(), diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index 238bccede..a25e847ca 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -150,7 +150,7 @@ export class DockerProvider implements IStatusProvider { private healthFromStatus = (status: string): DockerHealthStatus => { const match = HEALTH_STATUS.exec(status); if (!match) return "none"; - return match[1] === "health: starting" ? "starting" : match[1]; + return match[1] === "health: starting" ? "starting" : (match[1] as DockerHealthStatus); }; private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary { diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index cb093c479..13ae68109 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -172,8 +172,10 @@ export class StatusService implements IStatusService { if (breaching && currentStatus !== "breached") { nextStatus = "breached"; transitioned = true; - } else if (!breaching && currentStatus === "breached") nextStatus = "up"; - transitioned = true; + } else if (!breaching && currentStatus === "breached") { + nextStatus = "up"; + transitioned = true; + } } return { nextStatus, transitioned }; }; From 6d4e944a582c4a7ecdc11cfb120da9853c8e429a Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 11:17:01 -0700 Subject: [PATCH 07/29] Add tests --- .../providers/network/dockerProvider.test.ts | 53 +++++- .../notifications/emailProvider.test.ts | 12 ++ .../notifications/pagerdutyProvider.test.ts | 12 ++ .../test/unit/services/checkService.test.ts | 30 +++- .../unit/services/incidentService.test.ts | 40 +++++ .../notificationMessageBuilder.test.ts | 140 +++++++++++++++- .../test/unit/services/statusService.test.ts | 158 +++++++++++++++++- .../test/unit/services/workerPipeline.test.ts | 9 +- server/test/unit/utils/dockerAlert.test.ts | 104 ++++++++++++ .../unit/validation/monitorValidation.test.ts | 54 ++++++ 10 files changed, 601 insertions(+), 11 deletions(-) create mode 100644 server/test/unit/utils/dockerAlert.test.ts diff --git a/server/test/unit/providers/network/dockerProvider.test.ts b/server/test/unit/providers/network/dockerProvider.test.ts index a90e2b65f..50e666a1d 100644 --- a/server/test/unit/providers/network/dockerProvider.test.ts +++ b/server/test/unit/providers/network/dockerProvider.test.ts @@ -229,6 +229,18 @@ describe("DockerProvider", () => { }); }); + it("leaves the exit code undefined for Capture containers", async () => { + const response = makeCaptureResponse({ + payload: { data: [makeCaptureContainer({ status: "exited", running: false, stats: undefined })], errors: [] }, + }); + const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) }); + + const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" })); + + expect(result.payload?.containers[0]?.state).toBe("exited"); + expect(result.payload?.containers[0]?.exitCode).toBeUndefined(); + }); + it("keeps state-derived Capture health separate from Docker healthchecks", async () => { const response = makeCaptureResponse({ payload: { @@ -558,6 +570,16 @@ describe("DockerProvider", () => { ); }); + it("reads the exit code from inspect", async () => { + const inspect = jest.fn().mockResolvedValue(makeInspect({ State: { StartedAt: "2026-08-28T10:00:00.000Z", ExitCode: 137 } })); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status: "Exited (137) 2 minutes ago" })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ state: "exited", exitCode: 137, health: "none" })); + }); + it("counts running, stopped, and unhealthy states in the summary", async () => { const containers = [ makeContainer({ Id: "a".repeat(64) }), @@ -682,11 +704,40 @@ describe("DockerProvider", () => { expect(result.status).toBe(true); expect(result.payload?.containers).toHaveLength(1); - expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ health: "none" })); expect(result.payload?.containers[0]?.restartCount).toBeUndefined(); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Failed to inspect") })); }); + it.each([ + ["Up 3 hours (healthy)", "healthy"], + ["Up 3 hours (unhealthy)", "unhealthy"], + ["Up 5 seconds (health: starting)", "starting"], + ["Up 3 hours", "none"], + ["Exited (1) 2 minutes ago", "none"], + ])("reads health from the list status %s when inspect rejects", async (Status, health) => { + const inspect = jest.fn().mockRejectedValue(new Error("inspect failed")); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ Status })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]?.health).toBe(health); + }); + + it.each([ + ["Exited (0) 2 minutes ago", 0], + ["Exited (137) 2 minutes ago", 137], + ["Up 3 hours", undefined], + ])("reads the exit code from the list status %s when inspect rejects", async (Status, exitCode) => { + const inspect = jest.fn().mockRejectedValue(new Error("inspect failed")); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]?.exitCode).toBe(exitCode); + }); + it("keeps the container without metrics when stats rejects", async () => { const stats = jest.fn().mockRejectedValue(new Error("stats failed")); const { provider } = setup({ stats }); diff --git a/server/test/unit/providers/notifications/emailProvider.test.ts b/server/test/unit/providers/notifications/emailProvider.test.ts index 7f3b34d4e..3610523a4 100644 --- a/server/test/unit/providers/notifications/emailProvider.test.ts +++ b/server/test/unit/providers/notifications/emailProvider.test.ts @@ -115,6 +115,18 @@ describe("EmailProvider", () => { expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor thresholds resolved", expect.anything()); }); + it("builds correct subject for container_breach", async () => { + const { provider, emailService } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" })); + expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor container alert", expect.anything()); + }); + + it("builds correct subject for container_resolved", async () => { + const { provider, emailService } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" })); + expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor containers recovered", expect.anything()); + }); + it("builds default subject for unknown type", async () => { const { provider, emailService } = createProvider(); await provider.sendMessage(makeNotification() as any, makeMessage({ type: "test" })); diff --git a/server/test/unit/providers/notifications/pagerdutyProvider.test.ts b/server/test/unit/providers/notifications/pagerdutyProvider.test.ts index 8b8d185b4..b17d212ba 100644 --- a/server/test/unit/providers/notifications/pagerdutyProvider.test.ts +++ b/server/test/unit/providers/notifications/pagerdutyProvider.test.ts @@ -89,6 +89,18 @@ describe("PagerDutyProvider", () => { expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve"); }); + it("uses 'resolve' event_action for container_resolved", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" })); + expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve"); + }); + + it("uses 'trigger' event_action for container_breach", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" })); + expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("trigger"); + }); + it("includes threshold info in summary and custom_details", async () => { const { provider } = createProvider(); await provider.sendMessage(makeNotification() as any, makeMessageWithThresholds()); diff --git a/server/test/unit/services/checkService.test.ts b/server/test/unit/services/checkService.test.ts index 4ced65947..230857b0e 100644 --- a/server/test/unit/services/checkService.test.ts +++ b/server/test/unit/services/checkService.test.ts @@ -3,7 +3,7 @@ import { CheckService } from "../../../src/domain/checks/check.service.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; @@ -357,6 +357,34 @@ describe("CheckService", () => { expect(fromRaw[0].metric).toBe("cpu"); expect(fromRebuilt).toEqual(fromRaw); }); + + it("docker: preserves payload.containers and payload.summary verbatim", () => { + const { service } = createService(); + const payload: DockerStatusPayload = { + containers: [ + { id: "1", name: "db", image: "postgres:16", state: "exited", status: "Exited (1) 2 minutes ago", health: "none", exitCode: 1 }, + { id: "2", name: "api", image: "api:latest", state: "running", status: "Up 3 hours (unhealthy)", health: "unhealthy", restartCount: 2 }, + ], + summary: { total: 2, running: 1, stopped: 1, unhealthy: 1 }, + }; + const status = makeStatusResponse({ type: "docker", status: true, payload } as any); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + const roundtripped = rebuilt.payload as DockerStatusPayload; + expect(roundtripped.containers).toEqual(payload.containers); + expect(roundtripped.summary).toEqual(payload.summary); + expect(roundtripped.containers[0].exitCode).toBe(1); + }); + + it("docker: leaves payload undefined for a failed check with no containers", () => { + const { service } = createService(); + const status = makeStatusResponse({ type: "docker", status: false, code: 5000, message: "connect ECONNREFUSED", payload: null } as any); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + expect(rebuilt.payload).toBeUndefined(); + }); }); // ── createChecks ───────────────────────────────────────────────────────── diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index 73a422cf7..a2e161769 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -175,6 +175,46 @@ describe("IncidentService", () => { expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" })); }); + it("builds the container list for docker threshold_breach incidents", async () => { + const created = makeIncident(); + const { service, incidentsRepository, notificationMessageBuilder } = createService(); + (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); + (incidentsRepository.create as jest.Mock).mockResolvedValue(created); + + const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }); + const response = { + monitorId: "mon-1", + payload: { + containers: [ + { id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 }, + { id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" }, + { id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" }, + ], + summary: { total: 3, running: 2, stopped: 1, unhealthy: 1 }, + }, + } as any; + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + await service.handleIncident(monitor, 200, decision, response); + + expect(incidentsRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" }) + ); + expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled(); + }); + + it("uses the container fallback message for docker incidents with no response", async () => { + const created = makeIncident(); + const { service, incidentsRepository } = createService(); + (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); + (incidentsRepository.create as jest.Mock).mockResolvedValue(created); + + const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true }); + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + await service.handleIncident(monitor, 200, decision, undefined); + + expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 9999, message: "Container alert" })); + }); + it("joins multiple threshold breaches with commas", async () => { const created = makeIncident(); const { service, incidentsRepository, notificationMessageBuilder } = createService(); diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index b0b8397e3..afa6519e9 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, beforeEach } from "@jest/globals"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -51,6 +51,29 @@ const makeHardwarePayload = (overrides?: Partial) }, }) as HardwareStatusPayload; +const makeDockerPayload = (containers: Partial[]): DockerStatusPayload => ({ + containers: containers.map((container, index) => ({ + id: `container-${index}`, + name: `container-${index}`, + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...container, + })), + summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 }, +}); + +const makeDockerMonitor = (overrides?: Partial): Monitor => + makeMonitor({ + type: "docker", + status: "breached", + url: "unix:///var/run/docker.sock", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + ...overrides, + }); + // ── Tests ──────────────────────────────────────────────────────────────────── describe("NotificationMessageBuilder", () => { @@ -123,7 +146,7 @@ describe("NotificationMessageBuilder", () => { it("builds a threshold_resolved message for hardware monitor recovering", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); - const decision = makeDecision({ notificationReason: "status_change" }); + const decision = makeDecision({ notificationReason: "threshold_resolved" }); const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com"); @@ -163,13 +186,46 @@ describe("NotificationMessageBuilder", () => { expect(msg.type).toBe("threshold_breach"); }); - it("returns threshold_resolved for hardware monitor with status_change and up", () => { + it("returns threshold_resolved for hardware monitor with threshold_resolved reason", () => { + const monitor = makeMonitor({ status: "up", type: "hardware" }); + const decision = makeDecision({ notificationReason: "threshold_resolved" }); + + const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("threshold_resolved"); + }); + + it("returns monitor_up for hardware monitor recovering from down with status_change", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); const decision = makeDecision({ notificationReason: "status_change" }); const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); - expect(msg.type).toBe("threshold_resolved"); + expect(msg.type).toBe("monitor_up"); + }); + + it("returns container_breach for docker monitor with threshold_breach reason", () => { + const decision = makeDecision({ notificationReason: "threshold_breach" }); + + const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("container_breach"); + }); + + it("returns container_resolved for docker monitor with threshold_resolved reason", () => { + const decision = makeDecision({ notificationReason: "threshold_resolved" }); + + const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("container_resolved"); + }); + + it("returns monitor_down for docker monitor that is down regardless of reason", () => { + const decision = makeDecision({ notificationReason: "threshold_breach" }); + + const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("monitor_down"); }); it("returns monitor_up for non-hardware monitor with status up", () => { @@ -223,7 +279,22 @@ describe("NotificationMessageBuilder", () => { const msg = builder.buildMessage( makeMonitor({ status: "up", type: "hardware" }), makeStatusResponse(), - makeDecision({ notificationReason: "status_change" }), + makeDecision({ notificationReason: "threshold_resolved" }), + "" + ); + expect(msg.severity).toBe("success"); + }); + + it("returns warning for container_breach", () => { + const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), makeDecision({ notificationReason: "threshold_breach" }), ""); + expect(msg.severity).toBe("warning"); + }); + + it("returns success for container_resolved", () => { + const msg = builder.buildMessage( + makeDockerMonitor({ status: "up" }), + makeStatusResponse(), + makeDecision({ notificationReason: "threshold_resolved" }), "" ); expect(msg.severity).toBe("success"); @@ -311,7 +382,7 @@ describe("NotificationMessageBuilder", () => { describe("threshold_resolved", () => { it("includes resolved details", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision({ notificationReason: "threshold_resolved" }), ""); expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor"); expect(msg.content.summary).toBe('Monitor "Test Monitor" thresholds have returned to normal.'); @@ -319,6 +390,63 @@ describe("NotificationMessageBuilder", () => { }); }); + describe("container_breach", () => { + it("lists one detail line per breaching container", () => { + const response = makeStatusResponse({ + type: "docker", + status: true, + payload: makeDockerPayload([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]), + } as any); + + const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.title).toBe("Container Alert: Test Monitor"); + expect(msg.content.summary).toBe('2 container(s) on "Test Monitor" need attention.'); + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "db: stopped (exit code 1)", "api: unhealthy"]); + expect(msg.content.thresholds).toBeUndefined(); + }); + + it("only lists containers matching an enabled switch", () => { + const response = makeStatusResponse({ + type: "docker", + status: true, + payload: makeDockerPayload([ + { name: "db", state: "exited", exitCode: 1 }, + { name: "api", health: "unhealthy" }, + ]), + } as any); + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); + + const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "api: unhealthy"]); + }); + + it("lists no containers when the payload is an error string", () => { + const response = makeStatusResponse({ type: "docker", status: false, payload: "connect ECONNREFUSED" } as any); + + const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.summary).toBe('0 container(s) on "Test Monitor" need attention.'); + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker"]); + }); + }); + + describe("container_resolved", () => { + it("includes recovered details", () => { + const msg = builder.buildMessage( + makeDockerMonitor({ status: "up" }), + makeStatusResponse(), + makeDecision({ notificationReason: "threshold_resolved" }), + "" + ); + + expect(msg.content.title).toBe("Containers Recovered: Test Monitor"); + expect(msg.content.summary).toBe('All containers on "Test Monitor" are back to normal.'); + expect(msg.content.details).toContain("Status: Up"); + }); + }); + describe("default content", () => { it("builds default content for unhandled notification type", () => { // Force default by creating a scenario where type falls through diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 071465d48..393cb5676 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts"; import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; @@ -1086,6 +1086,162 @@ describe("StatusService", () => { // they're private implementation details; exposing them publicly would // widen the API surface just for testability. + describe("docker container alerts", () => { + const makeDockerMonitor = (overrides?: Partial) => + makeMonitor({ + type: "docker", + url: "unix:///var/run/docker.sock", + statusWindow: [true, true, true, true], + statusWindowSize: 5, + status: "up", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + ...overrides, + }); + + const makeContainers = (containers: Partial[]): DockerStatusPayload => ({ + containers: containers.map((container, index) => ({ + id: `container-${index}`, + name: `container-${index}`, + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...container, + })), + summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 }, + }); + + const makeDockerResponse = (payload: DockerStatusPayload | null, overrides?: Partial>) => + makeStatusResponse({ + type: "docker", + status: true, + payload, + ...overrides, + } as any) as MonitorStatusResponse; + + const run = async (monitor: Monitor, response: MonitorStatusResponse, check = makeCheck()) => { + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + const result = await service.updateMonitorStatus(response, check, monitor); + return { result, monitorsRepository }; + }; + + const stopped = { name: "db", state: "exited" as const, exitCode: 1 }; + const unhealthy = { name: "api", health: "unhealthy" as const }; + + it("goes to breached on the first check with a stopped container", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("breached"); + }); + + it("goes to breached on the first check with an unhealthy container", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([unhealthy]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("breached"); + }); + + it("stays up on a stopped container when both switches are off", async () => { + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped, unhealthy]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays up on a stopped container when only the unhealthy switch is on", async () => { + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays up on an exited container with code 0", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "job", state: "exited", exitCode: 0 }]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays breached without a transition while a container is still down", async () => { + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("breached"); + }); + + it("stays up without a transition while every container is clean", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "web" }]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("recovers from breached to up on the first clean check", async () => { + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([{ name: "web" }]))); + + expect(result.statusChanged).toBe(true); + expect(result.prevStatus).toBe("breached"); + expect(result.monitor.status).toBe("up"); + }); + + it("recovers from breached to up when the breaching switch is turned off", async () => { + const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("up"); + }); + + it("does not override down with breached when the host is unreachable", async () => { + const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.monitor.status).toBe("down"); + }); + + it("transitions from breached to down when the reachability threshold trips", async () => { + const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("down"); + }); + + it("does not evaluate containers while the status window is short", async () => { + const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("does not evaluate a failed check with a null payload", async () => { + const monitor = makeDockerMonitor({ status: "breached" }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("breached"); + }); + + it("does not touch the hardware counters or thresholdBreaches", async () => { + const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + + expect(result.thresholdBreaches).toBeUndefined(); + const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; + expect(patch.cpuAlertCounter).toBeUndefined(); + expect(patch.status).toBe("breached"); + }); + }); + describe("computeReachability (pure)", () => { const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => { const { service } = createService(); diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index e7c117152..0b168ef24 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -861,9 +861,14 @@ describe("WorkerPipeline", () => { }); }); - it("resolves incident when monitor recovers from breached", async () => { + it("resolves incident with threshold_resolved when monitor recovers from breached", async () => { const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" })); - expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); + expect(decision).toMatchObject({ + shouldResolveIncident: true, + shouldSendNotification: true, + shouldCreateIncident: false, + notificationReason: "threshold_resolved", + }); }); it("does not create or resolve for unhandled status transitions", async () => { diff --git a/server/test/unit/utils/dockerAlert.test.ts b/server/test/unit/utils/dockerAlert.test.ts new file mode 100644 index 000000000..a9d360511 --- /dev/null +++ b/server/test/unit/utils/dockerAlert.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "@jest/globals"; +import { describeContainerBreach, findContainerBreaches, isContainerStopped, isContainerUnhealthy } from "../../../src/domain/docker/docker-alert.ts"; +import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; + +const makeContainer = (overrides?: Partial): DockerContainerInfo => ({ + id: "abc123", + name: "web", + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...overrides, +}); + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "mon-1", + type: "docker", + dockerAlertOnStopped: false, + dockerAlertOnUnhealthy: false, + ...overrides, + }) as Monitor; + +describe("isContainerStopped", () => { + it("treats dead as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "dead" }))).toBe(true); + }); + + it.each([1, 137, 143])("treats exited with code %s as stopped", (exitCode) => { + expect(isContainerStopped(makeContainer({ state: "exited", exitCode }))).toBe(true); + }); + + it("does not treat exited with code 0 as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "exited", exitCode: 0 }))).toBe(false); + }); + + it("treats exited with an unknown exit code as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "exited" }))).toBe(true); + }); + + it.each(["running", "restarting", "paused", "created", "removing"] as const)("does not treat %s as stopped", (state) => { + expect(isContainerStopped(makeContainer({ state, exitCode: 1 }))).toBe(false); + }); +}); + +describe("isContainerUnhealthy", () => { + it("treats unhealthy as unhealthy", () => { + expect(isContainerUnhealthy(makeContainer({ health: "unhealthy" }))).toBe(true); + }); + + it.each(["healthy", "starting", "none"] as const)("does not treat %s as unhealthy", (health) => { + expect(isContainerUnhealthy(makeContainer({ health }))).toBe(false); + }); +}); + +describe("findContainerBreaches", () => { + const stopped = makeContainer({ name: "db", state: "exited", exitCode: 1 }); + const unhealthy = makeContainer({ name: "api", health: "unhealthy" }); + + it("returns nothing when both switches are off", () => { + expect(findContainerBreaches(makeMonitor(), [stopped, unhealthy])).toEqual([]); + }); + + it("reports only stopped containers when only the stopped switch is on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true }), [stopped, unhealthy]); + expect(breaches).toEqual([{ name: "db", reason: "stopped", exitCode: 1 }]); + }); + + it("reports only unhealthy containers when only the unhealthy switch is on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnUnhealthy: true }), [stopped, unhealthy]); + expect(breaches).toEqual([{ name: "api", reason: "unhealthy" }]); + }); + + it("reports both conditions when both switches are on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [stopped, unhealthy]); + expect(breaches.map((b) => b.reason)).toEqual(["stopped", "unhealthy"]); + }); + + it("reports a container that is both stopped and unhealthy once, as stopped", () => { + const both = makeContainer({ name: "worker", state: "dead", health: "unhealthy" }); + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [both]); + expect(breaches).toEqual([{ name: "worker", reason: "stopped", exitCode: undefined }]); + }); + + it("ignores healthy running containers", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [makeContainer()]); + expect(breaches).toEqual([]); + }); +}); + +describe("describeContainerBreach", () => { + it("describes an unhealthy container", () => { + expect(describeContainerBreach({ name: "api", reason: "unhealthy" })).toBe("api: unhealthy"); + }); + + it("describes a stopped container with its exit code", () => { + expect(describeContainerBreach({ name: "db", reason: "stopped", exitCode: 137 })).toBe("db: stopped (exit code 137)"); + }); + + it("describes a stopped container without an exit code", () => { + expect(describeContainerBreach({ name: "db", reason: "stopped" })).toBe("db: stopped"); + }); +}); diff --git a/server/test/unit/validation/monitorValidation.test.ts b/server/test/unit/validation/monitorValidation.test.ts index 218a0942d..63f3572a8 100644 --- a/server/test/unit/validation/monitorValidation.test.ts +++ b/server/test/unit/validation/monitorValidation.test.ts @@ -247,6 +247,20 @@ describe("monitorValidation — strategy gating", () => { expect(parsed.monitors[0].dockerLogsEnabled).toBe(false); }); + it("defaults the docker alert switches to false on imported docker monitors", () => { + const parsed = importMonitorsBodyValidation.parse({ + monitors: [ + { + name: "Imported Docker", + type: "docker", + url: "unix:///var/run/docker.sock", + }, + ], + }); + expect(parsed.monitors[0].dockerAlertOnStopped).toBe(false); + expect(parsed.monitors[0].dockerAlertOnUnhealthy).toBe(false); + }); + it("rejects strategy on imported non-pagespeed monitors", () => { expect(() => importMonitorsBodyValidation.parse({ @@ -818,3 +832,43 @@ describe("monitorValidation — Docker TLS credentials", () => { }); }); }); + +describe("monitorValidation — docker alert switches", () => { + describe("createMonitorBodyValidation", () => { + it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on a docker monitor", () => { + const parsed = createMonitorBodyValidation.parse({ + name: "Docker host", + type: "docker", + url: "unix:///var/run/docker.sock", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + }); + + expect(parsed.dockerAlertOnStopped).toBe(true); + expect(parsed.dockerAlertOnUnhealthy).toBe(true); + }); + + it("leaves the switches undefined when omitted", () => { + const parsed = createMonitorBodyValidation.parse({ + name: "Docker host", + type: "docker", + url: "unix:///var/run/docker.sock", + }); + + expect(parsed.dockerAlertOnStopped).toBeUndefined(); + expect(parsed.dockerAlertOnUnhealthy).toBeUndefined(); + }); + }); + + describe("editMonitorBodyValidation", () => { + it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on edits", () => { + const parsed = editMonitorBodyValidation.parse({ + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: false, + }); + + expect(parsed.dockerAlertOnStopped).toBe(true); + expect(parsed.dockerAlertOnUnhealthy).toBe(false); + }); + }); +}); From 5fa894cb79ce3fec246225586ee4937a1d9a8c27 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 11:17:19 -0700 Subject: [PATCH 08/29] Add client side types, validation, add swithces to monitor form --- client/src/Hooks/useMonitorForm.ts | 2 ++ client/src/Pages/CreateMonitor/index.tsx | 34 +++++++++++++++--------- client/src/Types/Monitor.ts | 2 ++ client/src/Validation/monitor.ts | 2 ++ client/src/locales/en.json | 12 +++++++++ 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index c487f8d81..1010a94ed 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -85,6 +85,8 @@ export const getMonitorDefaults = ( dockerTlsKey: "", dockerTlsKeySet: data?.dockerTlsKeySet ?? false, dockerLogsEnabled: data?.dockerLogsEnabled ?? false, + dockerAlertOnStopped: data?.dockerAlertOnStopped ?? false, + dockerAlertOnUnhealthy: data?.dockerAlertOnUnhealthy ?? false, }; break; case "game": diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 1de12d265..d936d5496 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -617,7 +617,6 @@ const CreateMonitorPage = () => { } /> )} - {showStep(0) && ( { } /> )} - {showStep(0) && watchedType === "docker" && isDockerTlsUrl(watchedUrl) && ( { } /> )} - {showStep(1) && ( { } /> )} - {/* Alert Thresholds - only for hardware type */} {watchedType === "hardware" && showStep(1) && ( { } /> )} - {showStep(1) && ( { } /> )} - {showStep(1) && ( { } /> )} - {showStep(1) && ( { } /> )} - {showStep(1) && watchedType === "docker" && !isCaptureDocker && ( { } /> )} - + {showStep(1) && watchedType === "docker" && ( + + + + + } + /> + )} {showStep(2) && (watchedType === "http" || watchedType === "grpc" || @@ -989,7 +1002,6 @@ const CreateMonitorPage = () => { } /> )} - {showStep(2) && watchedType === "http" && ( { } /> )} - {showStep(2) && supportsGeoCheck(watchedType) && ( { } /> )} - Date: Wed, 23 Sep 2026 11:31:37 -0700 Subject: [PATCH 09/29] add summary props to docker monitor list page, add localisation strings --- client/src/Pages/Docker/Monitors/index.tsx | 1 + client/src/locales/en.json | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/client/src/Pages/Docker/Monitors/index.tsx b/client/src/Pages/Docker/Monitors/index.tsx index 96d13abbf..88176a966 100644 --- a/client/src/Pages/Docker/Monitors/index.tsx +++ b/client/src/Pages/Docker/Monitors/index.tsx @@ -19,6 +19,7 @@ const DockerMonitorsPage = () => { controller={c} bulkActions showTypeFilter={false} + summaryProps={{ showBreached: true }} > Date: Wed, 23 Sep 2026 11:39:16 -0700 Subject: [PATCH 10/29] fix docker path --- client/src/Utils/MonitorUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/Utils/MonitorUtils.ts b/client/src/Utils/MonitorUtils.ts index c8bbca168..4a2cee7a9 100644 --- a/client/src/Utils/MonitorUtils.ts +++ b/client/src/Utils/MonitorUtils.ts @@ -20,7 +20,7 @@ export const getMonitorPath = (type: MonitorType): string => { websocket: "uptime", dns: "uptime", unknown: "uptime", - docker: "docker", + docker: "docker/host", hardware: "infrastructure", pagespeed: "pagespeed", }; From 7fb57d8090fe80a7989af010b422b196fb1acad8 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 11:46:58 -0700 Subject: [PATCH 11/29] fix incident links with helper --- server/src/domain/monitors/monitor.type.ts | 16 ++++++++++++++++ .../src/domain/notifications/providers/matrix.ts | 8 ++++---- .../src/domain/notifications/providers/ntfy.ts | 4 ++-- .../domain/notifications/providers/pushover.ts | 4 ++-- .../domain/notifications/providers/rocketChat.ts | 16 ++-------------- .../domain/notifications/providers/signalgrid.ts | 4 ++-- .../src/domain/notifications/providers/slack.ts | 4 ++-- .../src/domain/notifications/providers/teams.ts | 4 ++-- .../domain/notifications/providers/telegram.ts | 4 ++-- .../src/domain/notifications/providers/utils.ts | 4 ++++ .../domain/notifications/providers/webhook.ts | 4 ++-- .../notifications/matrixProvider.test.ts | 8 ++++++++ .../providers/notifications/ntfyProvider.test.ts | 2 +- .../notifications/pushoverProvider.test.ts | 6 ++++++ .../notifications/rocketChatProvider.test.ts | 1 + .../notifications/signalgridProvider.test.ts | 6 ++++++ .../notifications/slackProvider.test.ts | 8 ++++++++ .../notifications/teamsProvider.test.ts | 2 +- .../notifications/telegramProvider.test.ts | 6 ++++++ .../notifications/webhookProvider.test.ts | 6 ++++++ 20 files changed, 83 insertions(+), 34 deletions(-) diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index 38c0338eb..b033c560a 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -51,6 +51,22 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number]; export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type); +const MonitorPaths: Record = { + http: "uptime", + port: "uptime", + ping: "uptime", + game: "uptime", + grpc: "uptime", + websocket: "uptime", + dns: "uptime", + unknown: "uptime", + docker: "docker/host", + hardware: "infrastructure", + pagespeed: "pagespeed", +}; + +export const getMonitorPath = (type: MonitorType): string => MonitorPaths[type]; + // Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's // own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts // however far away the agent is. `unknown` is excluded because no request is made for it. diff --git a/server/src/domain/notifications/providers/matrix.ts b/server/src/domain/notifications/providers/matrix.ts index 1fafc44b1..7e66b5c37 100644 --- a/server/src/domain/notifications/providers/matrix.ts +++ b/server/src/domain/notifications/providers/matrix.ts @@ -3,7 +3,7 @@ import got from "got"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { AlertMatrixPayload, Notification } from "@/domain/notifications/notification.type.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import { randomUUID } from "crypto"; export class MatrixProvider extends NotificationProvider { @@ -172,10 +172,10 @@ export class MatrixProvider extends NotificationProvider { // Incident link (if incident exists) if (message.content.incident) { - const incidentUrl = `${message.clientHost}/infrastructure/${message.monitor.id}`; + const url = incidentUrl(message); plainLines.push(""); - plainLines.push(`View Incident: ${incidentUrl}`); - htmlLines.push(`

View Incident

`); + plainLines.push(`View Incident: ${url}`); + htmlLines.push(`

View Incident

`); } // Footer with timestamp diff --git a/server/src/domain/notifications/providers/ntfy.ts b/server/src/domain/notifications/providers/ntfy.ts index b415f648b..35f36f5e3 100644 --- a/server/src/domain/notifications/providers/ntfy.ts +++ b/server/src/domain/notifications/providers/ntfy.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "NtfyProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class NtfyProvider extends NotificationProvider { @@ -142,7 +142,7 @@ export class NtfyProvider extends NotificationProvider { } if (message.content.incident) { - lines.push("", `${message.clientHost}/infrastructure/${message.monitor.id}`); + lines.push("", incidentUrl(message)); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/pushover.ts b/server/src/domain/notifications/providers/pushover.ts index b1831e405..4243a0e32 100644 --- a/server/src/domain/notifications/providers/pushover.ts +++ b/server/src/domain/notifications/providers/pushover.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "PushoverProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class PushoverProvider extends NotificationProvider { @@ -101,7 +101,7 @@ export class PushoverProvider extends NotificationProvider { if (message.content.incident) { lines.push(""); - lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`); + lines.push(`View Incident: ${incidentUrl(message)}`); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/rocketChat.ts b/server/src/domain/notifications/providers/rocketChat.ts index 2ef8d7efb..78d93ce8f 100644 --- a/server/src/domain/notifications/providers/rocketChat.ts +++ b/server/src/domain/notifications/providers/rocketChat.ts @@ -1,5 +1,5 @@ const SERVICE_NAME = "RocketChatProvider"; -import { supportsUptimeDetails, type MonitorType } from "@/domain/monitors/monitor.type.js"; +import { getMonitorPath, type MonitorType } from "@/domain/monitors/monitor.type.js"; import type { Notification, NotificationMessage } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import { getTestMessage } from "@/domain/notifications/providers/utils.js"; @@ -139,19 +139,7 @@ export class RocketChatProvider extends NotificationProvider { return undefined; } - if (message.monitor.type === "hardware") { - return `${clientHost}/infrastructure/${message.monitor.id}`; - } - - if (message.monitor.type === "pagespeed") { - return `${clientHost}/pagespeed/${message.monitor.id}`; - } - - if (supportsUptimeDetails(message.monitor.type as MonitorType)) { - return `${clientHost}/uptime/${message.monitor.id}`; - } - - return undefined; + return `${clientHost}/${getMonitorPath(message.monitor.type as MonitorType)}/${message.monitor.id}`; } private severityColor(severity: NotificationMessage["severity"]): string { diff --git a/server/src/domain/notifications/providers/signalgrid.ts b/server/src/domain/notifications/providers/signalgrid.ts index fdd930724..889206c67 100644 --- a/server/src/domain/notifications/providers/signalgrid.ts +++ b/server/src/domain/notifications/providers/signalgrid.ts @@ -1,7 +1,7 @@ const SERVICE_NAME = "SignalgridProvider"; import type { Notification, NotificationMessage, NotificationSeverity } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class SignalgridProvider extends NotificationProvider { @@ -115,7 +115,7 @@ export class SignalgridProvider extends NotificationProvider { if (message.content.incident) { lines.push(""); - lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`); + lines.push(`View Incident: ${incidentUrl(message)}`); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/slack.ts b/server/src/domain/notifications/providers/slack.ts index 23f9b2f61..f73526ce0 100644 --- a/server/src/domain/notifications/providers/slack.ts +++ b/server/src/domain/notifications/providers/slack.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "SlackProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got, { HTTPError } from "got"; export class SlackProvider extends NotificationProvider { @@ -161,7 +161,7 @@ export class SlackProvider extends NotificationProvider { text: "View Incident", emoji: true, }, - url: `${message.clientHost}/infrastructure/${message.monitor.id}`, + url: incidentUrl(message), style: "primary", }, ], diff --git a/server/src/domain/notifications/providers/teams.ts b/server/src/domain/notifications/providers/teams.ts index 36d37a210..98e9e3c43 100644 --- a/server/src/domain/notifications/providers/teams.ts +++ b/server/src/domain/notifications/providers/teams.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "TeamsProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got, { HTTPError } from "got"; // Types for Adaptive Card elements @@ -254,7 +254,7 @@ export class TeamsProvider extends NotificationProvider { actions.push({ type: "Action.OpenUrl", title: "View Incident", - url: `${message.clientHost}/incidents/${message.content.incident.id}`, + url: incidentUrl(message), }); } diff --git a/server/src/domain/notifications/providers/telegram.ts b/server/src/domain/notifications/providers/telegram.ts index 72e31b5bf..26b2c2786 100644 --- a/server/src/domain/notifications/providers/telegram.ts +++ b/server/src/domain/notifications/providers/telegram.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "TelegramProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class TelegramProvider extends NotificationProvider { @@ -101,7 +101,7 @@ export class TelegramProvider extends NotificationProvider { if (message.content.incident) { lines.push(""); - lines.push(`View Incident`); + lines.push(`View Incident`); } return lines.join("\n"); diff --git a/server/src/domain/notifications/providers/utils.ts b/server/src/domain/notifications/providers/utils.ts index 3eaf428fe..4ac4d96fc 100644 --- a/server/src/domain/notifications/providers/utils.ts +++ b/server/src/domain/notifications/providers/utils.ts @@ -1,4 +1,8 @@ import { IEmailService } from "@/service/emailService.js"; +import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; + +export const incidentUrl = (message: NotificationMessage): string => `${message.clientHost}/incidents/${message.monitor.id}`; + export const buildTestEmail = async (emailService: IEmailService) => { const context = { testName: "Monitoring System" }; const html = await emailService.buildEmail("testEmailTemplate", context); diff --git a/server/src/domain/notifications/providers/webhook.ts b/server/src/domain/notifications/providers/webhook.ts index fc5a80fde..64fbb26e9 100644 --- a/server/src/domain/notifications/providers/webhook.ts +++ b/server/src/domain/notifications/providers/webhook.ts @@ -2,7 +2,7 @@ const SERVICE_NAME = "WebhookProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; -import { getTestMessage } from "@/domain/notifications/providers/utils.js"; +import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js"; import got from "got"; export class WebhookProvider extends NotificationProvider { @@ -74,7 +74,7 @@ export class WebhookProvider extends NotificationProvider { // Incident link if (message.content.incident) { - lines.push(`[View Incident](${message.clientHost}/infrastructure/${message.monitor.id})`); + lines.push(`[View Incident](${incidentUrl(message)})`); } // Return webhook payload with both text and structured data diff --git a/server/test/unit/providers/notifications/matrixProvider.test.ts b/server/test/unit/providers/notifications/matrixProvider.test.ts index 1294a677b..7cd0ea427 100644 --- a/server/test/unit/providers/notifications/matrixProvider.test.ts +++ b/server/test/unit/providers/notifications/matrixProvider.test.ts @@ -105,6 +105,14 @@ describe("MatrixProvider", () => { expect(body.formatted_body).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + const body = mockGotPut.mock.calls[0][1].json; + expect(body.body).toContain("View Incident: https://app.example.com/incidents/mon-1"); + expect(body.formatted_body).toContain('href="https://app.example.com/incidents/mon-1"'); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/ntfyProvider.test.ts b/server/test/unit/providers/notifications/ntfyProvider.test.ts index d365584a1..bced0b6b3 100644 --- a/server/test/unit/providers/notifications/ntfyProvider.test.ts +++ b/server/test/unit/providers/notifications/ntfyProvider.test.ts @@ -106,7 +106,7 @@ describe("NtfyProvider", () => { it("includes incident links in text", async () => { const { provider } = createProvider(); await provider.sendMessage(makeNtfyNotification() as any, makeMessageWithIncident()); - expect(mockGotPost.mock.calls[0][1].body).toContain("/infrastructure/mon-1"); + expect(mockGotPost.mock.calls[0][1].body).toContain("https://app.example.com/incidents/mon-1"); }); }); diff --git a/server/test/unit/providers/notifications/pushoverProvider.test.ts b/server/test/unit/providers/notifications/pushoverProvider.test.ts index a227b0b1b..457803131 100644 --- a/server/test/unit/providers/notifications/pushoverProvider.test.ts +++ b/server/test/unit/providers/notifications/pushoverProvider.test.ts @@ -99,6 +99,12 @@ describe("PushoverProvider", () => { expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident: https://app.example.com/incidents/mon-1"); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/rocketChatProvider.test.ts b/server/test/unit/providers/notifications/rocketChatProvider.test.ts index 816e19801..0568c144e 100644 --- a/server/test/unit/providers/notifications/rocketChatProvider.test.ts +++ b/server/test/unit/providers/notifications/rocketChatProvider.test.ts @@ -134,6 +134,7 @@ describe("RocketChatProvider", () => { ["ping", "https://app.example.com/uptime/mon-1"], ["pagespeed", "https://app.example.com/pagespeed/mon-1"], ["hardware", "https://app.example.com/infrastructure/mon-1"], + ["docker", "https://app.example.com/docker/host/mon-1"], ])("links %s monitors to %s", async (type, expectedLink) => { const { provider } = createProvider(); diff --git a/server/test/unit/providers/notifications/signalgridProvider.test.ts b/server/test/unit/providers/notifications/signalgridProvider.test.ts index 8c40bb1d9..b0812509a 100644 --- a/server/test/unit/providers/notifications/signalgridProvider.test.ts +++ b/server/test/unit/providers/notifications/signalgridProvider.test.ts @@ -139,6 +139,12 @@ describe("SignalgridProvider", () => { expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident: https://app.example.com/incidents/mon-1"); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/slackProvider.test.ts b/server/test/unit/providers/notifications/slackProvider.test.ts index 20058ab11..86c17f99a 100644 --- a/server/test/unit/providers/notifications/slackProvider.test.ts +++ b/server/test/unit/providers/notifications/slackProvider.test.ts @@ -96,6 +96,14 @@ describe("SlackProvider", () => { expect(text).toContain("Additional Information"); }); + it("links the incident button to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + const blocks = mockGotPost.mock.calls[0][1].json.blocks; + const actions = blocks.find((b: any) => b.type === "actions"); + expect(actions.elements[0].url).toBe("https://app.example.com/incidents/mon-1"); + }); + it("omits threshold, details, and incident sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/teamsProvider.test.ts b/server/test/unit/providers/notifications/teamsProvider.test.ts index c3766ec16..514190fe5 100644 --- a/server/test/unit/providers/notifications/teamsProvider.test.ts +++ b/server/test/unit/providers/notifications/teamsProvider.test.ts @@ -154,7 +154,7 @@ describe("TeamsProvider", () => { expect(card.actions).toHaveLength(1); expect(card.actions[0].type).toBe("Action.OpenUrl"); expect(card.actions[0].title).toBe("View Incident"); - expect(card.actions[0].url).toContain("/incidents/inc-1"); + expect(card.actions[0].url).toBe("https://app.example.com/incidents/mon-1"); }); it("omits actions when no incident is present", async () => { diff --git a/server/test/unit/providers/notifications/telegramProvider.test.ts b/server/test/unit/providers/notifications/telegramProvider.test.ts index f8f853e62..8fe90c412 100644 --- a/server/test/unit/providers/notifications/telegramProvider.test.ts +++ b/server/test/unit/providers/notifications/telegramProvider.test.ts @@ -99,6 +99,12 @@ describe("TelegramProvider", () => { expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].json.text).toContain('View Incident'); + }); + it("omits optional sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); diff --git a/server/test/unit/providers/notifications/webhookProvider.test.ts b/server/test/unit/providers/notifications/webhookProvider.test.ts index fd16f2c06..659a32e2f 100644 --- a/server/test/unit/providers/notifications/webhookProvider.test.ts +++ b/server/test/unit/providers/notifications/webhookProvider.test.ts @@ -86,6 +86,12 @@ describe("WebhookProvider", () => { expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident"); }); + it("links the incident to the monitor's incidents page", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessageWithIncident()); + expect(mockGotPost.mock.calls[0][1].json.text).toContain("[View Incident](https://app.example.com/incidents/mon-1)"); + }); + it("omits threshold and incident sections when not present", async () => { const { provider } = createProvider(); const msg = makeMessage(); From 4a66cf6e11c652604c2c99fd3b5bc238a22d5506 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 12:13:27 -0700 Subject: [PATCH 12/29] improve email error handling --- .../api/controllers/notificationController.ts | 11 ++++- .../src/api/controllers/settingsController.ts | 11 +++-- .../notifications/notification.service.ts | 12 ++++- .../domain/notifications/providers/email.ts | 8 ++- server/src/service/emailService.ts | 36 ++++++++++++-- server/src/utils/logger.ts | 27 +++++----- .../notifications/emailProvider.test.ts | 45 +++++++++++++++-- .../test/unit/services/emailService.test.ts | 49 +++++++++++++++++-- .../services/notificationsService.test.ts | 17 +++++++ 9 files changed, 183 insertions(+), 33 deletions(-) diff --git a/server/src/api/controllers/notificationController.ts b/server/src/api/controllers/notificationController.ts index 53dd7645c..e94cfa43e 100644 --- a/server/src/api/controllers/notificationController.ts +++ b/server/src/api/controllers/notificationController.ts @@ -35,11 +35,18 @@ class NotificationController implements INotificationController { testNotification = catchAsync(async (req: Request, res: Response) => { const notification = testNotificationBodyValidation.parse(req.body); - const success = await this.notificationsService.sendTestNotification(notification); + let success = false; + let reason: string | undefined; + try { + success = await this.notificationsService.sendTestNotification(notification); + } catch (error: unknown) { + if (!(error instanceof AppError)) throw error; + reason = error.message; + } return res.status(200).json({ success, - msg: success ? "Notification sent successfully" : "Notification could not be sent — check the destination details.", + msg: success ? "Notification sent successfully" : (reason ?? "Notification could not be sent — check the destination details."), details: { service: SERVICE_NAME }, }); }); diff --git a/server/src/api/controllers/settingsController.ts b/server/src/api/controllers/settingsController.ts index cd36467e2..856973c68 100644 --- a/server/src/api/controllers/settingsController.ts +++ b/server/src/api/controllers/settingsController.ts @@ -9,6 +9,8 @@ import { IProxiesService } from "@/domain/proxies/proxy.service.js"; import { IEgressStateService } from "@/domain/egress/egress-state.service.js"; import { Settings } from "@/domain/app-settings/app-settings.type.js"; +const SERVICE_NAME = "SettingsController"; + export interface ISettingsController { getAppSettings: RequestHandler; updateAppSettings: RequestHandler; @@ -119,7 +121,7 @@ class SettingsController implements ISettingsController { const html = await this.emailService.buildEmail("testEmailTemplate", context); if (!html) { - throw new AppError({ message: "Failed to build email template.", status: 500 }); + throw new AppError({ message: "Failed to build email template.", status: 500, service: SERVICE_NAME, method: "sendTestEmail" }); } let messageId: string; try { @@ -142,8 +144,11 @@ class SettingsController implements ISettingsController { // Surface the underlying SMTP failure: diagnosing the settings is the whole // point of the test email endpoint. throw new AppError({ - message: error instanceof Error ? `Failed to send test email. ${error.message}` : "Failed to send test email.", - status: 500, + message: error instanceof AppError ? error.message : "Failed to send test email.", + status: error instanceof AppError ? error.status : 500, + service: SERVICE_NAME, + method: "sendTestEmail", + details: error instanceof AppError ? error.details : undefined, }); } diff --git a/server/src/domain/notifications/notification.service.ts b/server/src/domain/notifications/notification.service.ts index affc0ea8a..b852e8a49 100644 --- a/server/src/domain/notifications/notification.service.ts +++ b/server/src/domain/notifications/notification.service.ts @@ -148,7 +148,17 @@ export class NotificationsService implements INotificationsService { testAllNotifications = async (notificationIds: string[]) => { const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds); - const tasks = notifications.map((notification) => this.sendTestNotification(notification)); + const tasks = notifications.map((notification) => + this.sendTestNotification(notification).catch((error: unknown) => { + this.logger.warn({ + message: `Test notification failed: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "testAllNotifications", + details: { notificationId: notification.id, type: notification.type }, + }); + return false; + }) + ); const outcomes = await Promise.all(tasks); const succeeded = outcomes.filter(Boolean).length; const failed = outcomes.length - succeeded; diff --git a/server/src/domain/notifications/providers/email.ts b/server/src/domain/notifications/providers/email.ts index 01bbaa6be..6ea7ffbd3 100644 --- a/server/src/domain/notifications/providers/email.ts +++ b/server/src/domain/notifications/providers/email.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "EmailProvider"; import type { Notification } from "@/domain/notifications/notification.type.js"; import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; +import { AppError } from "@/utils/AppError.js"; import { buildTestEmail } from "@/domain/notifications/providers/utils.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; import type { ILogger } from "@/utils/logger.js"; @@ -40,11 +41,13 @@ export class EmailProvider extends NotificationProvider { return true; } catch (error: unknown) { this.logger.warn({ - message: "Email test alert failed", + message: `Email test alert failed: ${error instanceof Error ? error.message : "Unknown error"}`, service: SERVICE_NAME, method: "sendTestAlert", + details: error instanceof AppError ? error.details : undefined, stack: error instanceof Error ? error.stack : undefined, }); + if (error instanceof AppError) throw error; return false; } } @@ -73,9 +76,10 @@ export class EmailProvider extends NotificationProvider { return true; } catch (error: unknown) { this.logger.warn({ - message: "Email notification failed", + message: `Email notification failed: ${error instanceof Error ? error.message : "Unknown error"}`, service: SERVICE_NAME, method: "sendMessage", + details: error instanceof AppError ? error.details : undefined, stack: error instanceof Error ? error.stack : undefined, }); return false; diff --git a/server/src/service/emailService.ts b/server/src/service/emailService.ts index 3aca5362c..e257ed44b 100755 --- a/server/src/service/emailService.ts +++ b/server/src/service/emailService.ts @@ -12,6 +12,23 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const SERVICE_NAME = "EmailService"; + +const SMTP_FAILURE_REASONS: Record = { + EAUTH: "rejected the login. Check the user name and password", + ENOAUTH: "requires a login. Set the user name and password", + EDNS: "could not be resolved. Check the host name", + ECONNECTION: "refused the connection. Check the host and port", + ESOCKET: "refused the connection. Check the host and port", + ETIMEDOUT: "did not respond. Check the port and the secure setting", + ETLS: "failed the TLS handshake. Check the secure and TLS settings", + EPROTOCOL: "sent an invalid response. Check the port and the secure setting", +}; + +const describeSmtpFailure = (error: unknown, host: string, port: number): string => { + const code = typeof error === "object" && error !== null && "code" in error ? String((error as { code: unknown }).code) : undefined; + const reason = (code !== undefined && SMTP_FAILURE_REASONS[code]) || "failed verification"; + return `SMTP host ${host}:${port} ${reason}.`; +}; type MjmlFn = typeof mjml2html; type FileSystem = typeof fs; type PathModule = typeof path; @@ -123,6 +140,15 @@ export class EmailService implements IEmailService { systemEmailRejectUnauthorized, } = config; + if (!systemEmailHost) { + throw new AppError({ + message: "Email is not configured. Set the system email host in settings.", + status: 400, + service: SERVICE_NAME, + method: "sendEmail", + }); + } + const emailConfig = { host: systemEmailHost, port: Number(systemEmailPort), @@ -146,17 +172,20 @@ export class EmailService implements IEmailService { try { await this.transporter.verify(); } catch (error: unknown) { + const cause = error instanceof Error ? error.message : "Unknown error"; this.logger.warn({ - message: "Email transporter verification failed", + message: `Email transporter verification failed: ${cause}`, service: SERVICE_NAME, method: "verifyTransporter", + details: { host: systemEmailHost, port: emailConfig.port }, stack: error instanceof Error ? error.stack : undefined, }); throw new AppError({ - message: "Email transporter verification failed", + message: describeSmtpFailure(error, systemEmailHost, emailConfig.port), + status: 502, service: SERVICE_NAME, method: "sendEmail", - details: { cause: error instanceof Error ? error.message : "Unknown error" }, + details: { cause }, }); } @@ -180,6 +209,7 @@ export class EmailService implements IEmailService { }); throw new AppError({ message: "Failed to send email", + status: 502, service: SERVICE_NAME, method: "sendEmail", details: { cause: error instanceof Error ? error.message : "Unknown error" }, diff --git a/server/src/utils/logger.ts b/server/src/utils/logger.ts index 3cd8570ca..6552a0bc9 100755 --- a/server/src/utils/logger.ts +++ b/server/src/utils/logger.ts @@ -60,21 +60,18 @@ class Logger implements ILogger { if (formattedDetails) msg += ` (details: ${formattedDetails})`; if (typeof stack === "string") { - const stackTrace = stack - .split("\n") - .slice(1) // Remove first line (error message) - .map((line: string) => { - const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/); - if (match) { - return { - function: match[1], - file: match[2], - line: parseInt(match[3] ?? "0", 10), - column: parseInt(match[4] ?? "0", 10), - }; - } - return line.trim(); - }); + const stackTrace = stack.split("\n").map((line: string) => { + const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/); + if (match) { + return { + function: match[1], + file: match[2], + line: parseInt(match[3] ?? "0", 10), + column: parseInt(match[4] ?? "0", 10), + }; + } + return line.trim(); + }); msg += ` (stack: ${JSON.stringify(stackTrace, null, 2)})`; } diff --git a/server/test/unit/providers/notifications/emailProvider.test.ts b/server/test/unit/providers/notifications/emailProvider.test.ts index 3610523a4..ebac6cc76 100644 --- a/server/test/unit/providers/notifications/emailProvider.test.ts +++ b/server/test/unit/providers/notifications/emailProvider.test.ts @@ -4,6 +4,7 @@ import { createMockLogger } from "../../../helpers/createMockLogger.ts"; import { makeNotification, makeMessage } from "../../../helpers/notificationMessage.ts"; import { testNotificationProviderContract } from "../../../helpers/notificationProviderContract.ts"; import type { IEmailService } from "../../../../src/service/emailService.ts"; +import { AppError } from "../../../../src/utils/AppError.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -50,11 +51,29 @@ describe("EmailProvider", () => { expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build test email content" })); }); - it("returns false when sendEmail rejects", async () => { + it("returns false and logs the cause when sendEmail rejects with a plain error", async () => { const { provider, emailService, logger } = createProvider(); (emailService.sendEmail as jest.Mock).mockRejectedValue(new Error("SMTP auth failed")); expect(await provider.sendTestAlert(makeNotification())).toBe(false); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email test alert failed" })); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email test alert failed: SMTP auth failed" })); + }); + + it("rethrows an AppError from sendEmail so the caller can report the reason", async () => { + const { provider, emailService, logger } = createProvider(); + const error = new AppError({ + message: "Email is not configured. Set the system email host in settings.", + status: 400, + details: { cause: "no host" }, + }); + (emailService.sendEmail as jest.Mock).mockRejectedValue(error); + + await expect(provider.sendTestAlert(makeNotification())).rejects.toBe(error); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + message: "Email test alert failed: Email is not configured. Set the system email host in settings.", + details: { cause: "no host" }, + }) + ); }); }); @@ -84,11 +103,29 @@ describe("EmailProvider", () => { // NotificationsService fans providers out through Promise.all, so a failed email must // resolve to false rather than reject and take the other channels down with it. - it("returns false when sendEmail rejects", async () => { + it("returns false and logs the cause when sendEmail rejects", async () => { const { provider, emailService, logger } = createProvider(); (emailService.sendEmail as jest.Mock).mockRejectedValue(new Error("SMTP auth failed")); expect(await provider.sendMessage(makeNotification() as any, makeMessage())).toBe(false); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email notification failed" })); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email notification failed: SMTP auth failed" })); + }); + + it("returns false rather than throwing when sendEmail rejects with an AppError", async () => { + const { provider, emailService, logger } = createProvider(); + (emailService.sendEmail as jest.Mock).mockRejectedValue( + new AppError({ + message: "SMTP verification failed for smtp.example.com:587: 535 Username and Password not accepted", + status: 502, + details: { cause: "ECONNREFUSED" }, + }) + ); + expect(await provider.sendMessage(makeNotification() as any, makeMessage())).toBe(false); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + message: "Email notification failed: SMTP verification failed for smtp.example.com:587: 535 Username and Password not accepted", + details: { cause: "ECONNREFUSED" }, + }) + ); }); it("builds correct subject for monitor_down", async () => { diff --git a/server/test/unit/services/emailService.test.ts b/server/test/unit/services/emailService.test.ts index d18b83917..484aeb495 100644 --- a/server/test/unit/services/emailService.test.ts +++ b/server/test/unit/services/emailService.test.ts @@ -335,17 +335,60 @@ describe("EmailService", () => { const { service, logger } = createService({ nodemailer }); await expect(service.sendEmail("to@example.com", "Subject", "

body

", makeTransportConfig())).rejects.toThrow( - "Email transporter verification failed" + "SMTP host smtp.example.com:587 failed verification." ); expect(logger.warn).toHaveBeenCalledWith( expect.objectContaining({ - message: "Email transporter verification failed", + message: "Email transporter verification failed: SMTP auth failed", service: "EmailService", method: "verifyTransporter", + details: { host: "smtp.example.com", port: 587 }, }) ); }); + it.each([ + ["EAUTH", "SMTP host smtp.example.com:587 rejected the login. Check the user name and password."], + ["ETIMEDOUT", "SMTP host smtp.example.com:587 did not respond. Check the port and the secure setting."], + ["EDNS", "SMTP host smtp.example.com:587 could not be resolved. Check the host name."], + ["ECONNECTION", "SMTP host smtp.example.com:587 refused the connection. Check the host and port."], + ["ETLS", "SMTP host smtp.example.com:587 failed the TLS handshake. Check the secure and TLS settings."], + ])("describes a %s verification failure without the server's reply text", async (code, expected) => { + const transporter = createMockTransporter(); + (transporter.verify as jest.Mock).mockRejectedValue(Object.assign(new Error("535-5.7.8 Username and Password not accepted"), { code })); + const nodemailer = createMockNodemailer(transporter); + const { service } = createService({ nodemailer }); + + await expect(service.sendEmail("to@example.com", "Subject", "

body

", makeTransportConfig())).rejects.toMatchObject({ + message: expected, + status: 502, + details: { cause: "535-5.7.8 Username and Password not accepted" }, + }); + }); + + it("throws a 400 without building a transport when the host is not configured", async () => { + const nodemailer = createMockNodemailer(); + const { service } = createService({ nodemailer }); + + await expect( + service.sendEmail("to@example.com", "Subject", "

body

", makeTransportConfig({ systemEmailHost: undefined })) + ).rejects.toMatchObject({ + status: 400, + message: "Email is not configured. Set the system email host in settings.", + }); + expect(nodemailer.createTransport).not.toHaveBeenCalled(); + }); + + it("treats an empty host as not configured", async () => { + const { service } = createService(); + + await expect(service.sendEmail("to@example.com", "Subject", "

body

", makeTransportConfig({ systemEmailHost: "" }))).rejects.toMatchObject( + { + status: 400, + } + ); + }); + it("reports the underlying cause when verification fails", async () => { const transporter = createMockTransporter(); (transporter.verify as jest.Mock).mockRejectedValue(new Error("SMTP auth failed")); @@ -353,7 +396,7 @@ describe("EmailService", () => { const { service } = createService({ nodemailer }); await expect(service.sendEmail("to@example.com", "Subject", "

body

", makeTransportConfig())).rejects.toMatchObject({ - status: 500, + status: 502, details: { cause: "SMTP auth failed" }, }); }); diff --git a/server/test/unit/services/notificationsService.test.ts b/server/test/unit/services/notificationsService.test.ts index adfdb6c90..15ca97e85 100644 --- a/server/test/unit/services/notificationsService.test.ts +++ b/server/test/unit/services/notificationsService.test.ts @@ -311,6 +311,23 @@ describe("NotificationsService", () => { const result = await service.testAllNotifications(["notif-1"]); expect(result).toBe(false); }); + + it("counts a provider that throws as a failed test and still tests the others", async () => { + const { service, notificationsRepository, emailProvider, slackProvider, logger } = createService(); + emailProvider.sendTestAlert.mockRejectedValue(new Error("Email is not configured. Set the system email host in settings.")); + (notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([ + makeNotification({ type: "email" }), + makeNotification({ type: "slack" }), + ]); + + const result = await service.testAllNotifications(["notif-1", "notif-2"]); + + expect(result).toBe(false); + expect(slackProvider.sendTestAlert).toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ message: "Test notification failed: Email is not configured. Set the system email host in settings." }) + ); + }); }); // ── CRUD operations ────────────────────────────────────────────────────── From 0d75629dcb06435bc791116c5fda7a6428c63eef Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 13:38:02 -0700 Subject: [PATCH 13/29] refactor out rebuilding of status response --- server/src/domain/checks/check.service.ts | 27 -- server/src/domain/checks/check.snapshot.ts | 14 +- .../src/domain/incidents/incident.service.ts | 28 +- .../notification.message-builder.ts | 75 ++-- .../notifications/notification.service.ts | 16 +- server/src/service/statusService.ts | 77 +--- .../src/worker/reactors/reactor.incident.ts | 2 +- .../worker/reactors/reactor.notification.ts | 2 +- server/src/worker/worker.interface.ts | 3 +- server/src/worker/worker.pipeline.ts | 16 +- server/test/helpers/heartbeatTestHarness.ts | 30 +- .../evaluateCursorMultiWorker.test.ts | 2 +- .../test/unit/services/checkService.test.ts | 110 +----- .../unit/services/incidentService.test.ts | 90 ++--- .../notificationMessageBuilder.test.ts | 346 +++++++----------- .../services/notificationsService.test.ts | 30 +- .../test/unit/services/statusService.test.ts | 300 +++++++-------- .../test/unit/services/workerPipeline.test.ts | 54 ++- 18 files changed, 436 insertions(+), 786 deletions(-) diff --git a/server/src/domain/checks/check.service.ts b/server/src/domain/checks/check.service.ts index 30db653e6..a7b127cfc 100644 --- a/server/src/domain/checks/check.service.ts +++ b/server/src/domain/checks/check.service.ts @@ -13,7 +13,6 @@ const SERVICE_NAME = "checkService"; export interface ICheckService { createChecks(checks: Check[]): Promise; toCheck(statusResponse: MonitorStatusResponse): Check | undefined; - toStatusResponse(check: Check): MonitorStatusResponse; getChecksByMonitor(params: { monitorId: string; teamId: string; @@ -135,32 +134,6 @@ export class CheckService implements ICheckService { return check; }; - private toPayload = (check: Check): HardwareStatusPayload | DockerStatusPayload | undefined => { - switch (check.metadata.type) { - case "hardware": - return { data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload; - case "docker": - return check.containers ? { containers: check.containers, summary: check.containerSummary } : undefined; - default: - return undefined; - } - }; - - toStatusResponse = (check: Check) => { - const statusResponse: MonitorStatusResponse = { - monitorId: check.metadata.monitorId, - teamId: check.metadata.teamId, - type: check.metadata.type, - status: check.status, - code: check.statusCode, - message: check.message, - responseTime: check.responseTime, - // re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data - payload: this.toPayload(check), - }; - return statusResponse; - }; - getChecksByMonitor = async ({ monitorId, teamId, diff --git a/server/src/domain/checks/check.snapshot.ts b/server/src/domain/checks/check.snapshot.ts index 65ef4f60a..19bdbe8fa 100644 --- a/server/src/domain/checks/check.snapshot.ts +++ b/server/src/domain/checks/check.snapshot.ts @@ -45,12 +45,14 @@ const mapMemory = (memory?: CheckMemoryInfo): SnapshotMemoryInfo | undefined => }; const mapDisk = (disk?: CheckDiskInfo[]): SnapshotDiskInfo[] | undefined => - disk?.map((entry) => ({ - device: entry.device, - total_bytes: entry.total_bytes, - used_bytes: entry.used_bytes, - usage_percent: entry.usage_percent, - })); + disk + ?.filter((entry) => entry != null) + .map((entry) => ({ + device: entry.device, + total_bytes: entry.total_bytes, + used_bytes: entry.used_bytes, + usage_percent: entry.usage_percent, + })); const mapHost = (host?: CheckHostInfo): SnapshotHostInfo | undefined => host && { diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index 1c249626a..e77fa5498 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -1,6 +1,6 @@ const SERVICE_NAME = "incidentService"; import type { Monitor } from "@/domain/monitors/monitor.type.js"; -import type { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js"; +import type { Check } from "@/domain/checks/check.type.js"; import { AppError } from "@/utils/AppError.js"; import { getDateForRange } from "@/utils/dataUtils.js"; import type { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js"; @@ -15,12 +15,7 @@ import { DateRange } from "@/types/query.js"; import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; export interface IIncidentService { - handleIncident( - monitor: Monitor, - code: number, - decision: MonitorActionDecision, - monitorStatusResponse?: MonitorStatusResponse - ): Promise; + handleIncident(monitor: Monitor, code: number, decision: MonitorActionDecision, check: Check): Promise; resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise; getIncidentsByTeam( teamId: string, @@ -59,12 +54,7 @@ export class IncidentService implements IIncidentService { this.notificationMessageBuilder = notificationMessageBuilder; } - handleIncident = async ( - monitor: Monitor, - code: number, - decision: MonitorActionDecision, - monitorStatusResponse?: MonitorStatusResponse - ): Promise => { + handleIncident = async (monitor: Monitor, code: number, decision: MonitorActionDecision, check: Check): Promise => { if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) { return null; } @@ -81,7 +71,7 @@ export class IncidentService implements IIncidentService { // For threshold breaches, use 9999 status code and build descriptive message if (decision.incidentReason === "threshold_breach") { statusCode = 9999; - message = this.buildThresholdBreachMessage(monitor, monitorStatusResponse); + message = this.buildThresholdBreachMessage(monitor, check); } const incident = { @@ -106,18 +96,14 @@ export class IncidentService implements IIncidentService { return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); }; - private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string { + private buildThresholdBreachMessage(monitor: Monitor, check: Check): string { if (monitor.type === "docker") { - const containers = (monitorStatusResponse?.payload as DockerStatusPayload | undefined)?.containers ?? []; + const containers = check.containers ?? []; const breaches = findContainerBreaches(monitor, containers); return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert"; } - if (!monitorStatusResponse) { - return "Threshold breach detected"; - } - - const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse); + const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check); if (breaches.length === 0) { return "Threshold breach detected"; diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 5cd78e9d9..11d11c069 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -1,5 +1,4 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; -import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { NotificationMessage, @@ -9,15 +8,11 @@ import type { NotificationContent, } from "@/domain/notifications/notification.type.js"; import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; +import type { Check } from "@/domain/checks/check.type.js"; export interface INotificationMessageBuilder { - buildMessage( - monitor: Monitor, - monitorStatusResponse: MonitorStatusResponse, - decision: MonitorActionDecision, - clientHost: string - ): NotificationMessage; - extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[]; + buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage; + extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[]; } const SERVICE_NAME = "NotificationMessageBuilder"; @@ -25,15 +20,10 @@ const SERVICE_NAME = "NotificationMessageBuilder"; export class NotificationMessageBuilder implements INotificationMessageBuilder { static SERVICE_NAME = SERVICE_NAME; - buildMessage( - monitor: Monitor, - monitorStatusResponse: MonitorStatusResponse, - decision: MonitorActionDecision, - clientHost: string - ): NotificationMessage { + buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage { const type = this.determineNotificationType(decision, monitor); const severity = this.determineSeverity(type); - const content = this.buildContent(type, monitor, monitorStatusResponse); + const content = this.buildContent(type, monitor, check); return { type, @@ -92,18 +82,18 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildContent(type: NotificationType, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + private buildContent(type: NotificationType, monitor: Monitor, check: Check): NotificationContent { switch (type) { case "monitor_down": - return this.buildMonitorDownContent(monitor, monitorStatusResponse); + return this.buildMonitorDownContent(monitor, check); case "monitor_up": return this.buildMonitorUpContent(monitor); case "threshold_breach": - return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); + return this.buildThresholdBreachContent(monitor, check); case "threshold_resolved": return this.buildThresholdResolvedContent(monitor); case "container_breach": - return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse); + return this.buildContainerBreachContent(monitor, check); case "container_resolved": return this.buildContainersRecoveredContent(monitor); default: @@ -111,19 +101,19 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildMonitorDownContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + private buildMonitorDownContent(monitor: Monitor, check: Check): NotificationContent { const title = `Monitor Down: ${monitor.name}`; const summary = `Monitor "${monitor.name}" is currently down and unreachable.`; const details = [`URL: ${monitor.url}`, `Status: Down`, `Type: ${monitor.type}`]; // Add response code if available - if (monitorStatusResponse.code) { - details.push(`Response Code: ${monitorStatusResponse.code}`); + if (check.statusCode) { + details.push(`Response Code: ${check.statusCode}`); } // Add error message if available - if (monitorStatusResponse.message) { - details.push(`Error: ${monitorStatusResponse.message}`); + if (check.message) { + details.push(`Error: ${check.message}`); } return { @@ -147,12 +137,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildThresholdBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { + private buildThresholdBreachContent(monitor: Monitor, check: Check): NotificationContent { const title = `Threshold Exceeded: ${monitor.name}`; const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`; const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`]; - const thresholds = this.extractThresholdBreaches(monitor, monitorStatusResponse); + const thresholds = this.extractThresholdBreaches(monitor, check); return { title, @@ -176,9 +166,8 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent { - const payload = monitorStatusResponse.payload; - const containers = payload && typeof payload !== "string" ? payload.containers : []; + private buildContainerBreachContent(monitor: Monitor, check: Check): NotificationContent { + const containers = check.containers ?? []; const breaches = findContainerBreaches(monitor, containers); const title = `Container Alert: ${monitor.name}`; const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`; @@ -202,25 +191,17 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[] { + public extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[] { const breaches: ThresholdBreach[] = []; // Check if this is a hardware monitor with threshold data - if (monitor.type !== "hardware" || !monitorStatusResponse.payload || typeof monitorStatusResponse.payload === "string") { - return breaches; - } - - // Cast to HardwareStatusPayload type - const payload = monitorStatusResponse.payload; - const hardware = payload.data; - - if (!hardware) { + if (monitor.type !== "hardware") { return breaches; } // Note: usage_percent values in hardware payload are decimals (0-1) - if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && hardware.cpu?.usage_percent !== undefined) { - const cpuUsageDecimal = hardware.cpu.usage_percent; + if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && check.cpu?.usage_percent !== undefined) { + const cpuUsageDecimal = check.cpu.usage_percent; const cpuPercent = cpuUsageDecimal * 100; const threshold = monitor.cpuAlertThreshold; if (cpuPercent > threshold) { @@ -235,8 +216,8 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } // Memory threshold breach - if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && hardware.memory?.usage_percent !== undefined) { - const memoryUsageDecimal = hardware.memory.usage_percent; + if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && check.memory?.usage_percent !== undefined) { + const memoryUsageDecimal = check.memory.usage_percent; const memoryPercent = memoryUsageDecimal * 100; const threshold = monitor.memoryAlertThreshold; if (memoryPercent > threshold) { @@ -251,10 +232,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } // Disk threshold breach - if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(hardware.disk)) { + if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(check.disk)) { // Find the highest disk usage let maxDiskUsageDecimal = 0; - for (const disk of hardware.disk) { + for (const disk of check.disk) { if (disk.usage_percent !== undefined && disk.usage_percent > maxDiskUsageDecimal) { maxDiskUsageDecimal = disk.usage_percent; } @@ -273,9 +254,9 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } // Temperature threshold breach - if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && hardware.cpu?.temperature) { + if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && check.cpu?.temperature) { // Temperature is an array in cpu.temperature - const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature]; + const temps = Array.isArray(check.cpu.temperature) ? check.cpu.temperature : [check.cpu.temperature]; const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t))); const threshold = monitor.tempAlertThreshold; if (maxTemp > threshold) { diff --git a/server/src/domain/notifications/notification.service.ts b/server/src/domain/notifications/notification.service.ts index b852e8a49..b782b7e0a 100644 --- a/server/src/domain/notifications/notification.service.ts +++ b/server/src/domain/notifications/notification.service.ts @@ -1,6 +1,5 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; import type { Notification } from "@/domain/notifications/notification.type.js"; -import type { MonitorStatusResponse } from "@/types/network.js"; import type { NotificationMessage } from "@/domain/notifications/notification.type.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js"; @@ -10,6 +9,7 @@ import type { ISettingsService } from "@/domain/app-settings/app-settings.servic import { ILogger } from "@/utils/logger.js"; import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; import type { NotificationChannel } from "@/domain/notifications/notification.type.js"; +import type { Check } from "@/domain/checks/check.type.js"; export type NotificationProviderRegistry = Record; @@ -19,7 +19,7 @@ export interface INotificationsService { findNotificationsByTeamId: (teamId: string) => Promise; updateById(id: string, teamId: string, updateData: Partial): Promise; deleteById: (id: string, teamId: string) => Promise; - handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; + handleNotifications: (monitor: Monitor, check: Check, decision: MonitorActionDecision) => Promise; sendTestNotification: (notification: Partial) => Promise; testAllNotifications: (notificationIds: string[]) => Promise; @@ -63,7 +63,7 @@ export class NotificationsService implements INotificationsService { private send = async ( notification: Notification, monitor: Monitor, - monitorStatusResponse: MonitorStatusResponse, + check: Check, decision: MonitorActionDecision, notificationMessage: NotificationMessage | undefined ): Promise => { @@ -89,16 +89,16 @@ export class NotificationsService implements INotificationsService { return await provider.sendMessage(notification, notificationMessage); }; - private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + private sendNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => { const notificationIds = monitor.notifications ?? []; const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds); // Build notification message once for all notifications const settings = this.settingsService.getSettings(); const clientHost = settings.clientHost || "Host not defined"; - const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost); + const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, check, decision, clientHost); - const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage)); + const tasks = notifications.map((notification) => this.send(notification, monitor, check, decision, notificationMessage)); const outcomes = await Promise.all(tasks); const succeeded = outcomes.filter(Boolean).length; @@ -114,13 +114,13 @@ export class NotificationsService implements INotificationsService { return succeeded === notifications.length; }; - handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + handleNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => { if (!decision.shouldSendNotification) { return false; } // Send notifications based on decision - return await this.sendNotifications(monitor, monitorStatusResponse, decision); + return await this.sendNotifications(monitor, check, decision); }; sendTestNotification = async (notification: Partial) => { diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 13ae68109..64a6f2e61 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -2,18 +2,7 @@ import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.re import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js"; import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js"; -import type { - DockerStatusPayload, - GameStatusPayload, - GrpcStatusPayload, - HardwareStatusPayload, - HttpStatusPayload, - MonitorStatusResponse, - PageSpeedStatusPayload, - PingStatusPayload, - PortStatusPayload, - StatusChangeResult, -} from "@/types/network.js"; +import type { StatusChangeResult } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { ILogger } from "@/utils/logger.js"; import type { HardwareStatusMetrics } from "@/types/network.js"; @@ -29,22 +18,8 @@ type HardwareBreaches = Record; type HardwareCounters = Record; export interface IStatusService { - updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise; - updateMonitorStatus( - statusResponse: MonitorStatusResponse< - | PingStatusPayload - | HttpStatusPayload - | PageSpeedStatusPayload - | HardwareStatusPayload - | DockerStatusPayload - | PortStatusPayload - | GameStatusPayload - | GrpcStatusPayload - | undefined - >, - check: Check, - monitor: Monitor - ): Promise; + updateRunningStats(check: Check, monitor: Monitor): Promise; + updateMonitorStatus(check: Check, monitor: Monitor): Promise; } export class StatusService implements IStatusService { @@ -59,11 +34,11 @@ export class StatusService implements IStatusService { this.monitorStatsRepository = monitorStatsRepository; } - async updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse) { + async updateRunningStats(check: Check, monitor: Monitor) { try { await this.monitorStatsRepository.updateByMonitorId(monitor.id, { - status: networkResponse.status === true, - responseTime: networkResponse.responseTime ?? 0, + status: check.status === true, + responseTime: check.responseTime ?? 0, now: Date.now(), }); return true; @@ -78,8 +53,8 @@ export class StatusService implements IStatusService { } } - private tryUpdateRunningStats = async (monitor: Monitor, statusResponse: MonitorStatusResponse) => { - const statsOk = await this.updateRunningStats(monitor, statusResponse); + private tryUpdateRunningStats = async (check: Check, monitor: Monitor) => { + const statsOk = await this.updateRunningStats(check, monitor); if (!statsOk) { this.logger.warn({ service: SERVICE_NAME, @@ -180,26 +155,12 @@ export class StatusService implements IStatusService { return { nextStatus, transitioned }; }; - updateMonitorStatus = async ( - statusResponse: MonitorStatusResponse< - | PingStatusPayload - | HttpStatusPayload - | PageSpeedStatusPayload - | HardwareStatusPayload - | DockerStatusPayload - | PortStatusPayload - | GameStatusPayload - | GrpcStatusPayload - | undefined - >, - check: Check, - monitor: Monitor - ): Promise => { + updateMonitorStatus = async (check: Check, monitor: Monitor): Promise => { try { - const { status, code } = statusResponse; + const { status, statusCode } = check; // Update running stats - await this.tryUpdateRunningStats(monitor, statusResponse); + await this.tryUpdateRunningStats(check, monitor); const prevStatus = monitor.status; const checkSnapshot = toCheckSnapshot(check); @@ -240,7 +201,7 @@ export class StatusService implements IStatusService { monitor: updated, statusChanged, prevStatus, - code, + code: statusCode, timestamp: Date.now(), }; } @@ -253,14 +214,13 @@ export class StatusService implements IStatusService { statusChanged = true; } - // Evaluate hardware threshold breaches (only for hardware monitors with metrics payload) + // Evaluate hardware threshold breaches let thresholdBreaches: HardwareBreaches | undefined; - const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined; - if (monitor.type === "hardware" && hardwarePayload?.data) { + if (monitor.type === "hardware") { const hardware = this.computeHardwareStatus({ currentStatus: newStatus, reachabilityDown: newStatus === "down", - metrics: hardwarePayload.data, + metrics: check, thresholds: { cpu: monitor.cpuAlertThreshold, memory: monitor.memoryAlertThreshold, @@ -286,9 +246,8 @@ export class StatusService implements IStatusService { } } - const dockerPayload = statusResponse.payload as DockerStatusPayload | undefined; - if (monitor.type === "docker" && dockerPayload?.containers) { - const breaches = findContainerBreaches(monitor, dockerPayload.containers); + if (monitor.type === "docker" && check.containers) { + const breaches = findContainerBreaches(monitor, check.containers); const docker = this.computeDockerStatus({ currentStatus: newStatus, reachabilityDown: newStatus === "down", @@ -317,7 +276,7 @@ export class StatusService implements IStatusService { monitor: updated, statusChanged, prevStatus, - code, + code: statusCode, timestamp: new Date().getTime(), thresholdBreaches, }; diff --git a/server/src/worker/reactors/reactor.incident.ts b/server/src/worker/reactors/reactor.incident.ts index 3679a30cf..24ca60e7b 100644 --- a/server/src/worker/reactors/reactor.incident.ts +++ b/server/src/worker/reactors/reactor.incident.ts @@ -8,6 +8,6 @@ export class IncidentReactor implements IMonitorReactor { constructor(private incidentService: IIncidentService) {} react = async (evaluation: MonitorEvaluation) => { - await this.incidentService.handleIncident(evaluation.monitor, evaluation.statusChange.code, evaluation.decision, evaluation.status); + await this.incidentService.handleIncident(evaluation.monitor, evaluation.statusChange.code, evaluation.decision, evaluation.check); }; } diff --git a/server/src/worker/reactors/reactor.notification.ts b/server/src/worker/reactors/reactor.notification.ts index 1daf528f4..91b9b7cd4 100644 --- a/server/src/worker/reactors/reactor.notification.ts +++ b/server/src/worker/reactors/reactor.notification.ts @@ -8,6 +8,6 @@ export class NotificationReactor implements IMonitorReactor { react = async (evaluation: MonitorEvaluation) => { if (!evaluation.decision.shouldSendNotification) return; - await this.notificationService.handleNotifications(evaluation.monitor, evaluation.status, evaluation.decision); + await this.notificationService.handleNotifications(evaluation.monitor, evaluation.check, evaluation.decision); }; } diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index d29b95fbb..4b5bba2c5 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,7 +1,7 @@ import { Monitor } from "@/domain/monitors/monitor.type.js"; import { Check } from "@/domain/checks/check.type.js"; import { Job, JobType } from "@/domain/jobs/job.type.js"; -import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js"; +import { StatusChangeResult } from "@/types/network.js"; import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; @@ -23,7 +23,6 @@ export interface MonitorActionDecision { export type MonitorEvaluation = { monitor: Monitor; - status: MonitorStatusResponse; // raw result from networkService.requestStatus check: Check; statusChange: StatusChangeResult; // from statusService.updateMonitorStatus decision: MonitorActionDecision; diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index 658737826..1faef2a11 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -240,8 +240,7 @@ export class WorkerPipeline implements IWorkerPipeline { let current = monitor; for (const check of checks) { - const status = this.deps.checkService.toStatusResponse(check); - const evaluation = await this.evaluateCheck(status, check, current); + const evaluation = await this.evaluateCheck(check, current); await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]); if (!leaseHeld) { @@ -272,7 +271,7 @@ export class WorkerPipeline implements IWorkerPipeline { // A check taken while the instance had no outbound connectivity says nothing about the target. // It is left out of the status window, running stats and monitor status entirely, so that no incident // opens for it and no spurious "resolved"/"up" fires once egress returns. - private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => { + private skipDegradedEgressCheck = (check: Check, monitor: Monitor): MonitorEvaluation => { this.deps.logger.debug({ message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`, service: SERVICE_NAME, @@ -287,22 +286,21 @@ export class WorkerPipeline implements IWorkerPipeline { }; return { monitor, - status, check, - statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() }, + statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: check.statusCode, timestamp: Date.now() }, decision, }; }; - evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise => { + evaluateCheck = async (check: Check, monitor: Monitor): Promise => { if (check.egressStatus === "degraded") { - return this.skipDegradedEgressCheck(status, check, monitor); + return this.skipDegradedEgressCheck(check, monitor); } - const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor); + const statusChange = await this.deps.statusService.updateMonitorStatus(check, monitor); const decision = this.decide(statusChange); - return { monitor: statusChange.monitor, status, check, statusChange, decision }; + return { monitor: statusChange.monitor, check, statusChange, decision }; }; private decide = (statusChange: StatusChangeResult): MonitorActionDecision => { diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index c4de7f6ec..f99be611c 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -5,17 +5,15 @@ import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts"; import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts"; import { StatusService } from "../../src/service/statusService.ts"; import { IncidentService } from "../../src/domain/incidents/incident.service.ts"; +import { CheckService } from "../../src/domain/checks/check.service.ts"; import { InMemoryMonitorsRepository } from "./InMemoryMonitorsRepository.ts"; import { InMemoryIncidentsRepository } from "./InMemoryIncidentsRepository.ts"; import { createMockLogger } from "./createMockLogger.ts"; import type { Monitor } from "../../src/domain/monitors/monitor.type.ts"; import type { MonitorStatusResponse } from "../../src/types/network.ts"; -import type { Check } from "../../src/domain/checks/check.type.ts"; import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts"; import type { EgressStatus } from "../../src/domain/egress/egress.type.ts"; -let checkCounter = 0; - export const makeMonitor = (overrides?: Partial): Monitor => ({ id: "mon-1", @@ -45,20 +43,6 @@ export const makeStatusResponse = (status: boolean, code: number): MonitorStatus responseTime: status ? 150 : 0, }); -export const makeCheck = (status: boolean, code: number): Check => { - const now = new Date().toISOString(); - return { - id: `check-${++checkCounter}`, - metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" }, - status, - statusCode: code, - responseTime: status ? 150 : 0, - message: status ? "OK" : "Service Unavailable", - createdAt: now, - updatedAt: now, - }; -}; - const createStubMonitorStatsRepo = () => ({ findByMonitorId: jest.fn().mockRejectedValue(new Error("no stats")), create: jest.fn().mockResolvedValue({}), @@ -86,8 +70,6 @@ export interface HeartbeatTestHarness { } export function createHeartbeatTestHarness(): HeartbeatTestHarness { - checkCounter = 0; - const monitorsRepo = new InMemoryMonitorsRepository(); const incidentsRepo = new InMemoryIncidentsRepository(); const logger = createMockLogger() as any; @@ -111,11 +93,9 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { return Promise.resolve(makeStatusResponse(nextStatus, nextCode)); }), }; - const checkService = { - toCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => { - return makeCheck(response.status, response.code); - }), - }; + // The real mapper, so the stored check carries the hardware/docker payload the evaluator reads. + // Only toCheck is reached; the repositories are never touched. + const checkService = new CheckService(monitorsRepo as any, logger, {} as any); const setNextResponse = (status: boolean, code: number) => { nextResponse = null; @@ -165,7 +145,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { if (!monitor) return; const result = await pipeline.produce(monitor); if (!result) return; // skipped (e.g. maintenance window) - const evaluation = await pipeline.evaluateCheck(result.status, result.check, monitor); + const evaluation = await pipeline.evaluateCheck(result.check, monitor); await reactorDispatcher.dispatch(evaluation); }; diff --git a/server/test/integration/evaluateCursorMultiWorker.test.ts b/server/test/integration/evaluateCursorMultiWorker.test.ts index 50be5ea72..a3fe75b04 100644 --- a/server/test/integration/evaluateCursorMultiWorker.test.ts +++ b/server/test/integration/evaluateCursorMultiWorker.test.ts @@ -131,7 +131,7 @@ describe("Evaluate cursor with more than one processing worker", () => { bufferService, dockerLogsService: { buildDockerLogs: jest.fn() } as any, statusService: { - updateMonitorStatus: async (_status: MonitorStatusResponse, check: Check, monitor: Monitor) => { + updateMonitorStatus: async (check: Check, monitor: Monitor) => { evaluatedCheckIds.push(check.id); return { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 }; }, diff --git a/server/test/unit/services/checkService.test.ts b/server/test/unit/services/checkService.test.ts index 230857b0e..daf238036 100644 --- a/server/test/unit/services/checkService.test.ts +++ b/server/test/unit/services/checkService.test.ts @@ -3,9 +3,7 @@ import { CheckService } from "../../../src/domain/checks/check.service.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; -import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -281,112 +279,6 @@ describe("CheckService", () => { }); }); - // ── toStatusResponse round-trip (Phase 2 §5 guard) ───────────────────────── - // The DB-queue split persists a Check, then the evaluator rebuilds the status response from it. - // toStatusResponse is only correct because every field a downstream consumer reads survives - // toCheck → toStatusResponse. These tests pin that: if a future change drops a consumed field - // from the round-trip, they fail instead of silently regressing the evaluator/reactors. - - describe("toStatusResponse round-trip", () => { - const makeHardwareStatus = (overrides?: Partial): MonitorStatusResponse => - makeStatusResponse({ - type: "hardware", - status: true, - code: 200, - message: "OK", - responseTime: 42, - payload: { - data: { - cpu: { usage_percent: 0.92 }, - memory: { usage_percent: 0.81 }, - disk: [{ device: "/dev/sda", usage_percent: 0.75 }], - host: { os: "linux" }, - net: [{ name: "eth0" }], - }, - } as HardwareStatusPayload, - ...overrides, - } as any); - - it("uptime: preserves every consumed scalar field", () => { - const { service } = createService(); - const status = makeStatusResponse({ type: "http", status: false, code: 503, message: "Service Unavailable", responseTime: 87 }); - - const rebuilt = service.toStatusResponse(service.toCheck(status)!); - - expect(rebuilt.monitorId).toBe(status.monitorId); - expect(rebuilt.teamId).toBe(status.teamId); - expect(rebuilt.type).toBe(status.type); - expect(rebuilt.status).toBe(status.status); - expect(rebuilt.code).toBe(status.code); - expect(rebuilt.message).toBe(status.message); - expect(rebuilt.responseTime).toBe(status.responseTime); - }); - - it("hardware: preserves payload.data metrics verbatim, same units (usage_percent stays a 0–1 decimal)", () => { - const { service } = createService(); - const status = makeHardwareStatus(); - - const rebuilt = service.toStatusResponse(service.toCheck(status)!); - - const original = status.payload as HardwareStatusPayload; - const roundtripped = rebuilt.payload as HardwareStatusPayload; - expect(roundtripped.data).toEqual(original.data); - expect(roundtripped.data.cpu!.usage_percent).toBe(0.92); // not rescaled to 92 - }); - - it("hardware: a real consumer (extractThresholdBreaches) behaves identically on raw vs round-tripped status", () => { - const { service } = createService(); - const builder = new NotificationMessageBuilder(); - const monitor = { - id: "mon-1", - type: "hardware", - cpuAlertThreshold: 90, // 0.92 → 92% breaches - memoryAlertThreshold: 90, // 0.81 → 81% does not - diskAlertThreshold: 90, // 0.75 → 75% does not - tempAlertThreshold: 100, - } as Monitor; - const status = makeHardwareStatus(); - - const rebuilt = service.toStatusResponse(service.toCheck(status)!); - - const fromRaw = builder.extractThresholdBreaches(monitor, status as MonitorStatusResponse); - const fromRebuilt = builder.extractThresholdBreaches(monitor, rebuilt as MonitorStatusResponse); - - // sanity: the fixture actually triggers a breach, so the equality below is not vacuous - expect(fromRaw).toHaveLength(1); - expect(fromRaw[0].metric).toBe("cpu"); - expect(fromRebuilt).toEqual(fromRaw); - }); - - it("docker: preserves payload.containers and payload.summary verbatim", () => { - const { service } = createService(); - const payload: DockerStatusPayload = { - containers: [ - { id: "1", name: "db", image: "postgres:16", state: "exited", status: "Exited (1) 2 minutes ago", health: "none", exitCode: 1 }, - { id: "2", name: "api", image: "api:latest", state: "running", status: "Up 3 hours (unhealthy)", health: "unhealthy", restartCount: 2 }, - ], - summary: { total: 2, running: 1, stopped: 1, unhealthy: 1 }, - }; - const status = makeStatusResponse({ type: "docker", status: true, payload } as any); - - const rebuilt = service.toStatusResponse(service.toCheck(status)!); - - const roundtripped = rebuilt.payload as DockerStatusPayload; - expect(roundtripped.containers).toEqual(payload.containers); - expect(roundtripped.summary).toEqual(payload.summary); - expect(roundtripped.containers[0].exitCode).toBe(1); - }); - - it("docker: leaves payload undefined for a failed check with no containers", () => { - const { service } = createService(); - const status = makeStatusResponse({ type: "docker", status: false, code: 5000, message: "connect ECONNREFUSED", payload: null } as any); - - const rebuilt = service.toStatusResponse(service.toCheck(status)!); - - expect(rebuilt.payload).toBeUndefined(); - }); - }); - // ── createChecks ───────────────────────────────────────────────────────── describe("createChecks", () => { diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index a2e161769..f5f978327 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -8,6 +8,7 @@ import type { INotificationMessageBuilder } from "../../../src/domain/notificati import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import type { Incident } from "../../../src/domain/incidents/incident.type.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; +import type { Check } from "../../../src/domain/checks/check.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -79,6 +80,19 @@ const makeIncident = (overrides?: Partial): Incident => ...overrides, }) as Incident; +const makeCheck = (overrides?: Partial): Check => + ({ + id: "check-1", + metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" }, + status: true, + statusCode: 200, + responseTime: 100, + message: "OK", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + ...overrides, + }) as Check; + const makeDecision = (overrides?: Partial): MonitorActionDecision => ({ shouldCreateIncident: false, shouldResolveIncident: false, @@ -96,7 +110,7 @@ describe("IncidentService", () => { describe("handleIncident", () => { it("returns null when neither create nor resolve is requested", async () => { const { service } = createService(); - const result = await service.handleIncident(makeMonitor(), 200, makeDecision()); + const result = await service.handleIncident(makeMonitor(), 200, makeDecision(), makeCheck()); expect(result).toBeNull(); }); @@ -105,7 +119,12 @@ describe("IncidentService", () => { const { service, incidentsRepository } = createService(); (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(existing); - const result = await service.handleIncident(makeMonitor(), 500, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" })); + const result = await service.handleIncident( + makeMonitor(), + 500, + makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), + makeCheck() + ); expect(result).toBe(existing); expect(incidentsRepository.create).not.toHaveBeenCalled(); @@ -117,7 +136,12 @@ describe("IncidentService", () => { (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); (incidentsRepository.create as jest.Mock).mockResolvedValue(created); - const result = await service.handleIncident(makeMonitor(), 500, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" })); + const result = await service.handleIncident( + makeMonitor(), + 500, + makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), + makeCheck() + ); expect(result).toBe(created); expect(incidentsRepository.create).toHaveBeenCalledWith( @@ -139,9 +163,12 @@ describe("IncidentService", () => { { metric: "cpu", formattedValue: "95%", threshold: 80, unit: "%" }, ]); + const monitor = makeMonitor({ type: "hardware" }); + const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), 200, decision, { monitorId: "mon-1" } as any); + await service.handleIncident(monitor, 200, decision, check); + expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ statusCode: 9999, @@ -150,18 +177,6 @@ describe("IncidentService", () => { ); }); - it("uses fallback message when monitorStatusResponse is undefined for threshold_breach", async () => { - const created = makeIncident(); - const { service, incidentsRepository } = createService(); - (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); - (incidentsRepository.create as jest.Mock).mockResolvedValue(created); - - const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), 200, decision, undefined); - - expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" })); - }); - it("uses fallback message when extractThresholdBreaches returns empty array", async () => { const created = makeIncident(); const { service, incidentsRepository, notificationMessageBuilder } = createService(); @@ -170,7 +185,7 @@ describe("IncidentService", () => { (notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([]); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), 200, decision, { monitorId: "mon-1" } as any); + await service.handleIncident(makeMonitor(), 200, decision, makeCheck()); expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" })); }); @@ -182,19 +197,17 @@ describe("IncidentService", () => { (incidentsRepository.create as jest.Mock).mockResolvedValue(created); const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }); - const response = { - monitorId: "mon-1", - payload: { - containers: [ - { id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 }, - { id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" }, - { id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" }, - ], - summary: { total: 3, running: 2, stopped: 1, unhealthy: 1 }, - }, - } as any; + const check = makeCheck({ + metadata: { monitorId: "mon-1", teamId: "team-1", type: "docker" }, + containers: [ + { id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 }, + { id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" }, + { id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" }, + ], + containerSummary: { total: 3, running: 2, stopped: 1, unhealthy: 1 }, + }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(monitor, 200, decision, response); + await service.handleIncident(monitor, 200, decision, check); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" }) @@ -202,19 +215,6 @@ describe("IncidentService", () => { expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled(); }); - it("uses the container fallback message for docker incidents with no response", async () => { - const created = makeIncident(); - const { service, incidentsRepository } = createService(); - (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); - (incidentsRepository.create as jest.Mock).mockResolvedValue(created); - - const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true }); - const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(monitor, 200, decision, undefined); - - expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 9999, message: "Container alert" })); - }); - it("joins multiple threshold breaches with commas", async () => { const created = makeIncident(); const { service, incidentsRepository, notificationMessageBuilder } = createService(); @@ -226,7 +226,7 @@ describe("IncidentService", () => { ]); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), 200, decision, { monitorId: "mon-1" } as any); + await service.handleIncident(makeMonitor(), 200, decision, makeCheck()); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ message: "CPU: 95% (threshold: 80%), MEMORY: 90% (threshold: 80%)" }) @@ -240,7 +240,7 @@ describe("IncidentService", () => { (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(active); (incidentsRepository.updateById as jest.Mock).mockResolvedValue(resolved); - const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true })); + const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true }), makeCheck()); expect(result).toBe(resolved); expect(incidentsRepository.updateById).toHaveBeenCalledWith( @@ -254,7 +254,7 @@ describe("IncidentService", () => { const { service, incidentsRepository } = createService(); (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); - const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true })); + const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true }), makeCheck()); expect(result).toBeNull(); }); diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index afa6519e9..3937e3bcf 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it, beforeEach } from "@jest/globals"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; +import type { Check } from "../../../src/domain/checks/check.type.ts"; +import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -30,29 +31,20 @@ const makeDecision = (overrides?: Partial): MonitorAction ...overrides, }); -const makeStatusResponse = (overrides?: Partial): MonitorStatusResponse => - ({ - monitorId: "mon-1", - teamId: "team-1", - type: "http", - status: false, - code: 500, - message: "Internal Server Error", - ...overrides, - }) as MonitorStatusResponse; +const makeCheck = (overrides?: Partial): Check => ({ + id: "check-1", + metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" }, + status: false, + responseTime: 100, + statusCode: 500, + message: "Internal Server Error", + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + ...overrides, +}); -const makeHardwarePayload = (overrides?: Partial): HardwareStatusPayload => - ({ - data: { - cpu: { usage_percent: 0.5, temperature: [45] }, - memory: { usage_percent: 0.6 }, - disk: [{ usage_percent: 0.7 }], - ...overrides, - }, - }) as HardwareStatusPayload; - -const makeDockerPayload = (containers: Partial[]): DockerStatusPayload => ({ - containers: containers.map((container, index) => ({ +const makeContainers = (containers: Partial[]): DockerContainerInfo[] => + containers.map((container, index) => ({ id: `container-${index}`, name: `container-${index}`, image: "nginx:latest", @@ -60,9 +52,7 @@ const makeDockerPayload = (containers: Partial): Monitor => makeMonitor({ @@ -95,9 +85,9 @@ describe("NotificationMessageBuilder", () => { it("builds a monitor_down message", () => { const monitor = makeMonitor({ status: "down" }); const decision = makeDecision(); - const response = makeStatusResponse(); + const check = makeCheck(); - const msg = builder.buildMessage(monitor, response, decision, "https://app.example.com"); + const msg = builder.buildMessage(monitor, check, decision, "https://app.example.com"); expect(msg.type).toBe("monitor_down"); expect(msg.severity).toBe("critical"); @@ -120,7 +110,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "up", type: "http" }); const decision = makeDecision({ notificationReason: "status_change" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com"); + const msg = builder.buildMessage(monitor, makeCheck(), decision, "https://app.example.com"); expect(msg.type).toBe("monitor_up"); expect(msg.severity).toBe("success"); @@ -130,12 +120,9 @@ describe("NotificationMessageBuilder", () => { it("builds a threshold_breach message", () => { const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); const decision = makeDecision({ notificationReason: "threshold_breach" }); - const response = makeStatusResponse({ - type: "hardware", - payload: makeHardwarePayload({ cpu: { usage_percent: 0.9, temperature: [50] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); - const msg = builder.buildMessage(monitor, response, decision, "https://app.example.com"); + const msg = builder.buildMessage(monitor, check, decision, "https://app.example.com"); expect(msg.type).toBe("threshold_breach"); expect(msg.severity).toBe("warning"); @@ -148,7 +135,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); const decision = makeDecision({ notificationReason: "threshold_resolved" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com"); + const msg = builder.buildMessage(monitor, makeCheck(), decision, "https://app.example.com"); expect(msg.type).toBe("threshold_resolved"); expect(msg.severity).toBe("success"); @@ -159,7 +146,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "down" }); const decision = makeDecision({ notificationReason: null }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com"); + const msg = builder.buildMessage(monitor, makeCheck(), decision, "https://app.example.com"); expect(msg.metadata.notificationReason).toBe("status_change"); }); @@ -172,7 +159,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "down" }); const decision = makeDecision({ notificationReason: "threshold_breach" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(monitor, makeCheck(), decision, ""); expect(msg.type).toBe("monitor_down"); }); @@ -181,7 +168,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); const decision = makeDecision({ notificationReason: "threshold_breach" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(monitor, makeCheck(), decision, ""); expect(msg.type).toBe("threshold_breach"); }); @@ -190,7 +177,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); const decision = makeDecision({ notificationReason: "threshold_resolved" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(monitor, makeCheck(), decision, ""); expect(msg.type).toBe("threshold_resolved"); }); @@ -199,7 +186,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); const decision = makeDecision({ notificationReason: "status_change" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(monitor, makeCheck(), decision, ""); expect(msg.type).toBe("monitor_up"); }); @@ -207,7 +194,7 @@ describe("NotificationMessageBuilder", () => { it("returns container_breach for docker monitor with threshold_breach reason", () => { const decision = makeDecision({ notificationReason: "threshold_breach" }); - const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(makeDockerMonitor(), makeCheck(), decision, ""); expect(msg.type).toBe("container_breach"); }); @@ -215,7 +202,7 @@ describe("NotificationMessageBuilder", () => { it("returns container_resolved for docker monitor with threshold_resolved reason", () => { const decision = makeDecision({ notificationReason: "threshold_resolved" }); - const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeCheck(), decision, ""); expect(msg.type).toBe("container_resolved"); }); @@ -223,7 +210,7 @@ describe("NotificationMessageBuilder", () => { it("returns monitor_down for docker monitor that is down regardless of reason", () => { const decision = makeDecision({ notificationReason: "threshold_breach" }); - const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeCheck(), decision, ""); expect(msg.type).toBe("monitor_down"); }); @@ -232,7 +219,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "up", type: "http" }); const decision = makeDecision({ notificationReason: "status_change" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(monitor, makeCheck(), decision, ""); expect(msg.type).toBe("monitor_up"); }); @@ -241,7 +228,7 @@ describe("NotificationMessageBuilder", () => { const monitor = makeMonitor({ status: "unknown" as any }); const decision = makeDecision({ notificationReason: null }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + const msg = builder.buildMessage(monitor, makeCheck(), decision, ""); expect(msg.type).toBe("monitor_up"); }); @@ -251,14 +238,14 @@ describe("NotificationMessageBuilder", () => { describe("determineSeverity", () => { it("returns critical for monitor_down", () => { - const msg = builder.buildMessage(makeMonitor({ status: "down" }), makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(makeMonitor({ status: "down" }), makeCheck(), makeDecision(), ""); expect(msg.severity).toBe("critical"); }); it("returns warning for threshold_breach", () => { const msg = builder.buildMessage( makeMonitor({ status: "up", type: "hardware" }), - makeStatusResponse(), + makeCheck(), makeDecision({ notificationReason: "threshold_breach" }), "" ); @@ -268,7 +255,7 @@ describe("NotificationMessageBuilder", () => { it("returns success for monitor_up", () => { const msg = builder.buildMessage( makeMonitor({ status: "up", type: "http" }), - makeStatusResponse(), + makeCheck(), makeDecision({ notificationReason: "status_change" }), "" ); @@ -278,7 +265,7 @@ describe("NotificationMessageBuilder", () => { it("returns success for threshold_resolved", () => { const msg = builder.buildMessage( makeMonitor({ status: "up", type: "hardware" }), - makeStatusResponse(), + makeCheck(), makeDecision({ notificationReason: "threshold_resolved" }), "" ); @@ -286,14 +273,14 @@ describe("NotificationMessageBuilder", () => { }); it("returns warning for container_breach", () => { - const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), makeDecision({ notificationReason: "threshold_breach" }), ""); + const msg = builder.buildMessage(makeDockerMonitor(), makeCheck(), makeDecision({ notificationReason: "threshold_breach" }), ""); expect(msg.severity).toBe("warning"); }); it("returns success for container_resolved", () => { const msg = builder.buildMessage( makeDockerMonitor({ status: "up" }), - makeStatusResponse(), + makeCheck(), makeDecision({ notificationReason: "threshold_resolved" }), "" ); @@ -307,9 +294,9 @@ describe("NotificationMessageBuilder", () => { describe("monitor_down", () => { it("includes response code and error message when present", () => { const monitor = makeMonitor({ status: "down" }); - const response = makeStatusResponse({ code: 503, message: "Service Unavailable" }); + const check = makeCheck({ statusCode: 503, message: "Service Unavailable" }); - const msg = builder.buildMessage(monitor, response, makeDecision(), ""); + const msg = builder.buildMessage(monitor, check, makeDecision(), ""); expect(msg.content.details).toContain("Response Code: 503"); expect(msg.content.details).toContain("Error: Service Unavailable"); @@ -317,9 +304,9 @@ describe("NotificationMessageBuilder", () => { it("omits response code when falsy", () => { const monitor = makeMonitor({ status: "down" }); - const response = makeStatusResponse({ code: 0, message: "" }); + const check = makeCheck({ statusCode: 0, message: "" }); - const msg = builder.buildMessage(monitor, response, makeDecision(), ""); + const msg = builder.buildMessage(monitor, check, makeDecision(), ""); expect(msg.content.details).not.toContainEqual(expect.stringContaining("Response Code:")); expect(msg.content.details).not.toContainEqual(expect.stringContaining("Error:")); @@ -327,7 +314,7 @@ describe("NotificationMessageBuilder", () => { it("includes URL, Status, and Type in details", () => { const monitor = makeMonitor({ status: "down" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), ""); expect(msg.content.details).toContain("URL: https://example.com"); expect(msg.content.details).toContain("Status: Down"); @@ -336,13 +323,13 @@ describe("NotificationMessageBuilder", () => { it("sets summary text", () => { const monitor = makeMonitor({ status: "down" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), ""); expect(msg.content.summary).toBe('Monitor "Test Monitor" is currently down and unreachable.'); }); it("sets timestamp", () => { const monitor = makeMonitor({ status: "down" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), ""); expect(msg.content.timestamp).toBeInstanceOf(Date); }); }); @@ -350,7 +337,7 @@ describe("NotificationMessageBuilder", () => { describe("monitor_up", () => { it("includes recovery details", () => { const monitor = makeMonitor({ status: "up", type: "http" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), ""); expect(msg.content.title).toBe("Monitor Recovered: Test Monitor"); expect(msg.content.summary).toBe('Monitor "Test Monitor" is back up and operational.'); @@ -361,11 +348,9 @@ describe("NotificationMessageBuilder", () => { describe("threshold_breach", () => { it("includes threshold breaches in content", () => { const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.9, temperature: [50] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); - const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), ""); + const msg = builder.buildMessage(monitor, check, makeDecision({ notificationReason: "threshold_breach" }), ""); expect(msg.content.thresholds).toEqual( expect.arrayContaining([ @@ -382,7 +367,7 @@ describe("NotificationMessageBuilder", () => { describe("threshold_resolved", () => { it("includes resolved details", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision({ notificationReason: "threshold_resolved" }), ""); + const msg = builder.buildMessage(monitor, makeCheck(), makeDecision({ notificationReason: "threshold_resolved" }), ""); expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor"); expect(msg.content.summary).toBe('Monitor "Test Monitor" thresholds have returned to normal.'); @@ -392,13 +377,12 @@ describe("NotificationMessageBuilder", () => { describe("container_breach", () => { it("lists one detail line per breaching container", () => { - const response = makeStatusResponse({ - type: "docker", + const check = makeCheck({ status: true, - payload: makeDockerPayload([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]), - } as any); + containers: makeContainers([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]), + }); - const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + const msg = builder.buildMessage(makeDockerMonitor(), check, makeDecision({ notificationReason: "threshold_breach" }), ""); expect(msg.content.title).toBe("Container Alert: Test Monitor"); expect(msg.content.summary).toBe('2 container(s) on "Test Monitor" need attention.'); @@ -407,25 +391,24 @@ describe("NotificationMessageBuilder", () => { }); it("only lists containers matching an enabled switch", () => { - const response = makeStatusResponse({ - type: "docker", + const check = makeCheck({ status: true, - payload: makeDockerPayload([ + containers: makeContainers([ { name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, ]), - } as any); + }); const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); - const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), ""); + const msg = builder.buildMessage(monitor, check, makeDecision({ notificationReason: "threshold_breach" }), ""); expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "api: unhealthy"]); }); - it("lists no containers when the payload is an error string", () => { - const response = makeStatusResponse({ type: "docker", status: false, payload: "connect ECONNREFUSED" } as any); + it("lists no containers when the check carries no container data", () => { + const check = makeCheck({ status: false, message: "connect ECONNREFUSED" }); - const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + const msg = builder.buildMessage(makeDockerMonitor(), check, makeDecision({ notificationReason: "threshold_breach" }), ""); expect(msg.content.summary).toBe('0 container(s) on "Test Monitor" need attention.'); expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker"]); @@ -436,7 +419,7 @@ describe("NotificationMessageBuilder", () => { it("includes recovered details", () => { const msg = builder.buildMessage( makeDockerMonitor({ status: "up" }), - makeStatusResponse(), + makeCheck(), makeDecision({ notificationReason: "threshold_resolved" }), "" ); @@ -457,7 +440,7 @@ describe("NotificationMessageBuilder", () => { // Instead, let's verify the default case exists by checking that // non-standard types would get default content const monitor = makeMonitor({ status: "up", type: "http" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), ""); // monitor_up is handled, so content should be monitor_up specific expect(msg.content.title).toBe("Monitor Recovered: Test Monitor"); @@ -484,7 +467,7 @@ describe("NotificationMessageBuilder", () => { describe("buildContent edge cases", () => { it("returns default content for unhandled notification type", () => { const monitor = makeMonitor({ status: "up", type: "http" }); - const result = (builder as any).buildContent("test", monitor, makeStatusResponse()); + const result = (builder as any).buildContent("test", monitor, makeCheck()); expect(result.title).toBe("Monitor: Test Monitor"); expect(result.summary).toBe('Status update for monitor "Test Monitor".'); @@ -499,27 +482,24 @@ describe("NotificationMessageBuilder", () => { describe("extractThresholdBreaches", () => { it("returns empty array for non-hardware monitor", () => { const monitor = makeMonitor({ type: "http" }); - const response = makeStatusResponse(); + const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches).toEqual([]); }); - it("returns empty array when payload is missing", () => { - const monitor = makeMonitor({ type: "hardware" }); - const response = makeStatusResponse({ payload: undefined } as any); + it("returns empty array when the check carries no hardware data", () => { + const monitor = makeMonitor({ + type: "hardware", + cpuAlertThreshold: 80, + memoryAlertThreshold: 80, + diskAlertThreshold: 80, + tempAlertThreshold: 80, + }); + const check = makeCheck(); - const breaches = builder.extractThresholdBreaches(monitor, response); - - expect(breaches).toEqual([]); - }); - - it("returns empty array when hardware data is missing", () => { - const monitor = makeMonitor({ type: "hardware" }); - const response = makeStatusResponse({ payload: { data: undefined } } as any); - - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches).toEqual([]); }); @@ -529,11 +509,9 @@ describe("NotificationMessageBuilder", () => { describe("cpu threshold", () => { it("detects CPU breach when usage exceeds threshold", () => { const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.9, temperature: [50] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches).toContainEqual( expect.objectContaining({ @@ -548,55 +526,45 @@ describe("NotificationMessageBuilder", () => { it("does not report CPU breach when usage is below threshold", () => { const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [50] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); }); it("skips CPU check when cpuAlertThreshold is undefined", () => { const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: undefined }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.99 } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); }); it("skips CPU check when cpuAlertThreshold is null", () => { const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: null as any }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.99 } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); }); it("skips CPU check when cpu usage_percent is undefined", () => { const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: undefined } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: undefined } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); }); it("skips CPU check when cpu object is undefined", () => { const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: undefined as any }), - } as any); + const check = makeCheck({ cpu: undefined }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); }); @@ -607,11 +575,9 @@ describe("NotificationMessageBuilder", () => { describe("memory threshold", () => { it("detects memory breach when usage exceeds threshold", () => { const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ memory: { usage_percent: 0.85 } as any }), - } as any); + const check = makeCheck({ memory: { usage_percent: 0.85 } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches).toContainEqual( expect.objectContaining({ @@ -626,55 +592,45 @@ describe("NotificationMessageBuilder", () => { it("does not report memory breach when usage is below threshold", () => { const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 90 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ memory: { usage_percent: 0.5 } as any }), - } as any); + const check = makeCheck({ memory: { usage_percent: 0.5 } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); }); it("skips memory check when memoryAlertThreshold is undefined", () => { const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: undefined }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ memory: { usage_percent: 0.99 } as any }), - } as any); + const check = makeCheck({ memory: { usage_percent: 0.99 } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); }); it("skips memory check when memoryAlertThreshold is null", () => { const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: null as any }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ memory: { usage_percent: 0.99 } as any }), - } as any); + const check = makeCheck({ memory: { usage_percent: 0.99 } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); }); it("skips memory check when memory usage_percent is undefined", () => { const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ memory: { usage_percent: undefined } as any }), - } as any); + const check = makeCheck({ memory: { usage_percent: undefined } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); }); it("skips memory check when memory object is undefined", () => { const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ memory: undefined as any }), - } as any); + const check = makeCheck({ memory: undefined }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); }); @@ -685,13 +641,9 @@ describe("NotificationMessageBuilder", () => { describe("disk threshold", () => { it("detects disk breach using highest usage across multiple disks", () => { const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ - disk: [{ usage_percent: 0.5 }, { usage_percent: 0.95 }, { usage_percent: 0.7 }] as any, - }), - } as any); + const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.95 }, { usage_percent: 0.7 }] }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches).toContainEqual( expect.objectContaining({ @@ -705,59 +657,45 @@ describe("NotificationMessageBuilder", () => { it("does not report disk breach when all disks are below threshold", () => { const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 90 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ - disk: [{ usage_percent: 0.5 }, { usage_percent: 0.6 }] as any, - }), - } as any); + const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.6 }] }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); }); it("skips disk check when diskAlertThreshold is undefined", () => { const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: undefined }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ disk: [{ usage_percent: 0.99 }] as any }), - } as any); + const check = makeCheck({ disk: [{ usage_percent: 0.99 }] }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); }); it("skips disk check when diskAlertThreshold is null", () => { const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: null as any }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ disk: [{ usage_percent: 0.99 }] as any }), - } as any); + const check = makeCheck({ disk: [{ usage_percent: 0.99 }] }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); }); it("skips disk check when disk is not an array", () => { const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ disk: "not-an-array" as any }), - } as any); + const check = makeCheck({ disk: "not-an-array" as any }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); }); it("skips disks with undefined usage_percent", () => { const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 50 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ - disk: [{ usage_percent: undefined }, { usage_percent: 0.3 }] as any, - }), - } as any); + const check = makeCheck({ disk: [{ usage_percent: undefined }, { usage_percent: 0.3 }] }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); }); @@ -768,11 +706,9 @@ describe("NotificationMessageBuilder", () => { describe("temperature threshold", () => { it("detects temperature breach from array of temperatures", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [65, 75, 68] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [65, 75, 68] } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches).toContainEqual( expect.objectContaining({ @@ -787,11 +723,9 @@ describe("NotificationMessageBuilder", () => { it("handles single temperature value (non-array)", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: 80 } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: 80 as any } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches).toContainEqual( expect.objectContaining({ @@ -804,11 +738,9 @@ describe("NotificationMessageBuilder", () => { it("does not report temperature breach when below threshold", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [60, 65] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [60, 65] } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); @@ -820,55 +752,45 @@ describe("NotificationMessageBuilder", () => { // content for a breach some OTHER metric triggered could still list temp as breaching). it("does not report temperature breach when exactly at threshold", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [80] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [80] } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); it("skips temperature check when tempAlertThreshold is undefined", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [99] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); it("skips temperature check when tempAlertThreshold is null", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: null as any }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [99] } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); it("skips temperature check when cpu.temperature is falsy", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: null } as any }), - } as any); + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: null as any } }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); it("skips temperature check when cpu object is undefined", () => { const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ cpu: undefined as any }), - } as any); + const check = makeCheck({ cpu: undefined }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); @@ -885,15 +807,13 @@ describe("NotificationMessageBuilder", () => { diskAlertThreshold: 85, tempAlertThreshold: 65, }); - const response = makeStatusResponse({ - payload: makeHardwarePayload({ - cpu: { usage_percent: 0.9, temperature: [70] } as any, - memory: { usage_percent: 0.85 } as any, - disk: [{ usage_percent: 0.95 }] as any, - }), - } as any); + const check = makeCheck({ + cpu: { usage_percent: 0.9, temperature: [70] }, + memory: { usage_percent: 0.85 }, + disk: [{ usage_percent: 0.95 }], + }); - const breaches = builder.extractThresholdBreaches(monitor, response); + const breaches = builder.extractThresholdBreaches(monitor, check); const metrics = breaches.map((b) => b.metric); expect(metrics).toContain("cpu"); diff --git a/server/test/unit/services/notificationsService.test.ts b/server/test/unit/services/notificationsService.test.ts index 15ca97e85..7abba9d7f 100644 --- a/server/test/unit/services/notificationsService.test.ts +++ b/server/test/unit/services/notificationsService.test.ts @@ -3,7 +3,7 @@ import { NotificationsService } from "../../../src/domain/notifications/notifica import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import type { Notification } from "../../../src/domain/notifications/notification.type.ts"; -import type { MonitorStatusResponse } from "../../../src/types/network.ts"; +import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -130,7 +130,17 @@ const makeDecision = (overrides?: Partial): MonitorAction ...overrides, }); -const makeStatusResponse = () => ({ monitorId: "mon-1", status: false, code: 500 }) as unknown as MonitorStatusResponse; +const makeCheck = (overrides?: Partial): Check => ({ + id: "check-1", + metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" }, + status: false, + responseTime: 100, + statusCode: 500, + message: "Internal Server Error", + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + ...overrides, +}); // ── Tests ──────────────────────────────────────────────────────────────────── @@ -140,7 +150,7 @@ describe("NotificationsService", () => { describe("handleNotifications", () => { it("returns false when shouldSendNotification is false", async () => { const { service } = createService(); - const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision({ shouldSendNotification: false })); + const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision({ shouldSendNotification: false })); expect(result).toBe(false); }); @@ -148,7 +158,7 @@ describe("NotificationsService", () => { const { service, notificationsRepository, emailProvider } = createService(); (notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification({ type: "email" })]); - const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision()); + const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision()); expect(result).toBe(true); expect(emailProvider.sendMessage).toHaveBeenCalledTimes(1); @@ -160,7 +170,7 @@ describe("NotificationsService", () => { const deps = createService(); (deps.notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification({ type })]); - await deps.service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision()); + await deps.service.handleNotifications(makeMonitor(), makeCheck(), makeDecision()); const providerMap: Record> = { webhook: deps.webhookProvider, @@ -183,7 +193,7 @@ describe("NotificationsService", () => { const { service, notificationsRepository, logger } = createService(); (notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification({ type: "carrier_pigeon" as any })]); - const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision()); + const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision()); expect(result).toBe(false); expect(logger.warn).toHaveBeenCalledWith( @@ -197,7 +207,7 @@ describe("NotificationsService", () => { const { service, notificationsRepository, logger } = createService({ notificationMessageBuilder }); (notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification()]); - const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision()); + const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision()); expect(result).toBe(false); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Notification message not provided" })); @@ -207,7 +217,7 @@ describe("NotificationsService", () => { const { service, notificationsRepository } = createService(); (notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([]); - const result = await service.handleNotifications(makeMonitor({ notifications: undefined as any }), makeStatusResponse(), makeDecision()); + const result = await service.handleNotifications(makeMonitor({ notifications: undefined as any }), makeCheck(), makeDecision()); expect(result).toBe(true); expect(notificationsRepository.findNotificationsByIds).toHaveBeenCalledWith([]); @@ -222,7 +232,7 @@ describe("NotificationsService", () => { emailProvider.sendMessage.mockResolvedValue(true); slackProvider.sendMessage.mockResolvedValue(false); - const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision()); + const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision()); expect(result).toBe(false); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 success, 1 failure") })); @@ -233,7 +243,7 @@ describe("NotificationsService", () => { const { service, notificationsRepository, notificationMessageBuilder } = createService({ settingsService }); (notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification()]); - await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision()); + await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision()); expect(notificationMessageBuilder.buildMessage).toHaveBeenCalledWith( expect.anything(), diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 393cb5676..50005312d 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts"; import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; +import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; @@ -98,18 +98,6 @@ const makeMonitor = (overrides?: Partial): Monitor => ...overrides, }) as Monitor; -const makeStatusResponse = (overrides?: Partial): MonitorStatusResponse => - ({ - monitorId: "mon-1", - teamId: "team-1", - type: "http", - status: true, - code: 200, - message: "OK", - responseTime: 100, - ...overrides, - }) as MonitorStatusResponse; - const makeCheck = (overrides?: Partial): Check => ({ id: "check-1", @@ -155,7 +143,7 @@ describe("StatusService", () => { const { service, monitorStatsRepository } = createService(); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse({ responseTime: 50 })); + const result = await service.updateRunningStats(makeCheck({ responseTime: 50 }), makeMonitor()); expect(result).toBe(true); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith( @@ -172,7 +160,7 @@ describe("StatusService", () => { const { service, monitorStatsRepository } = createService(); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse({ status: false, responseTime: 100 })); + const result = await service.updateRunningStats(makeCheck({ status: false, responseTime: 100 }), makeMonitor()); expect(result).toBe(true); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false, responseTime: 100 })); @@ -182,7 +170,7 @@ describe("StatusService", () => { const { service, monitorStatsRepository } = createService(); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - await service.updateRunningStats(makeMonitor(), makeStatusResponse({ responseTime: undefined })); + await service.updateRunningStats(makeCheck({ responseTime: undefined as any }), makeMonitor()); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 })); }); @@ -191,7 +179,7 @@ describe("StatusService", () => { const { service, monitorStatsRepository } = createService(); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - await service.updateRunningStats(makeMonitor(), makeStatusResponse({ responseTime: 0 })); + await service.updateRunningStats(makeCheck({ responseTime: 0 }), makeMonitor()); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 })); }); @@ -200,7 +188,7 @@ describe("StatusService", () => { const { service, monitorStatsRepository } = createService(); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - await service.updateRunningStats(makeMonitor(), makeStatusResponse({ status: undefined as unknown as boolean })); + await service.updateRunningStats(makeCheck({ status: undefined as any }), makeMonitor()); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false })); }); @@ -209,7 +197,7 @@ describe("StatusService", () => { const { service, logger, monitorStatsRepository } = createService(); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("db write failed")); - const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse()); + const result = await service.updateRunningStats(makeCheck(), makeMonitor()); expect(result).toBe(false); expect(logger.error).toHaveBeenCalledWith( @@ -225,7 +213,7 @@ describe("StatusService", () => { const { service, logger, monitorStatsRepository } = createService(); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue("string error"); - const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse()); + const result = await service.updateRunningStats(makeCheck(), makeMonitor()); expect(result).toBe(false); expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined })); @@ -241,7 +229,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.prevStatus).toBe("up"); @@ -252,7 +240,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(monitorsRepository.findById).not.toHaveBeenCalled(); expect(result.prevStatus).toBe("up"); @@ -265,7 +253,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + await service.updateMonitorStatus(makeCheck({ status: false }), monitor); // Atomic push should have been called with the correct status and window size expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith( @@ -286,7 +274,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }), monitor); + await service.updateMonitorStatus(makeCheck({ id: "new-check" }), monitor); expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS); expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" })); @@ -304,7 +292,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -322,7 +310,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); @@ -339,7 +327,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -359,7 +347,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -376,7 +364,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -388,7 +376,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); + await service.updateMonitorStatus(makeCheck(), monitor); expect(monitor.statusWindow).toEqual([true]); }); @@ -399,7 +387,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); + await service.updateMonitorStatus(makeCheck(), monitor); expect(monitor.recentChecks).toHaveLength(1); }); @@ -409,7 +397,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error")); - await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Failed to update monitor"); + await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Failed to update monitor"); }); it("throws AppError with 'Unknown error' for non-Error exceptions", async () => { @@ -417,7 +405,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error"); - await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Unknown error"); + await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Unknown error"); }); it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => { @@ -435,7 +423,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -453,7 +441,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -475,7 +463,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); @@ -493,7 +481,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); }); @@ -514,7 +502,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); expect(result.statusChanged).toBe(false); // was effectively up already — no notification storm @@ -531,7 +519,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); expect(result.statusChanged).toBe(true); // down must surface so an incident is opened @@ -548,7 +536,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -565,7 +553,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -580,7 +568,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -596,7 +584,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -611,7 +599,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -631,7 +619,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); @@ -646,7 +634,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -659,7 +647,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down")); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); expect(logger.warn).toHaveBeenCalledWith( @@ -677,7 +665,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck(), monitor); + const result = await service.updateMonitorStatus(makeCheck({ statusCode: 201 }), monitor); expect(result.code).toBe(201); expect(result.timestamp).toBeGreaterThan(0); @@ -703,13 +691,12 @@ describe("StatusService", () => { ...overrides, }); - const makeHardwareResponse = (payload: HardwareStatusPayload, overrides?: Partial>) => - makeStatusResponse({ - type: "hardware", + const makeHardwareCheck = (metrics: Pick, overrides?: Partial) => + makeCheck({ status: true, - payload, + ...metrics, ...overrides, - } as any) as MonitorStatusResponse; + }); it("detects CPU threshold breach and decrements counter", async () => { const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 }); @@ -717,8 +704,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.cpu).toBe(true); expect(result.thresholdBreaches?.memory).toBe(false); @@ -734,8 +721,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.memory).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -749,10 +736,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -766,10 +751,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.temp).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -788,10 +771,13 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} }, - } as any); - await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ + cpu: { usage_percent: 0.1, temperature: [30] }, + memory: { usage_percent: 0.1 }, + disk: [{ usage_percent: 0.1 }], + host: {}, + }); + await service.updateMonitorStatus(check, monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(5); @@ -809,8 +795,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.monitor.status).toBe("breached"); expect(result.statusChanged).toBe(false); @@ -822,10 +808,13 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ + cpu: { usage_percent: 0.1, temperature: [30] }, + memory: { usage_percent: 0.1 }, + disk: [{ usage_percent: 0.1 }], + host: {}, + }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); @@ -844,11 +833,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - status: false, - data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.monitor.status).toBe("down"); }); @@ -859,8 +845,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ data: { cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.cpu).toBe(false); }); @@ -871,8 +857,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); @@ -883,10 +869,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); @@ -897,36 +881,25 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); - it("skips threshold evaluation when payload is undefined", async () => { - const monitor = makeHardwareMonitor(); + it("reports no breaches when the check carries no hardware metrics", async () => { + // A stored hardware check with no metrics (cpu/memory/disk/host all absent) must not + // throw and must not report any breach. + const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse(undefined as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const result = await service.updateMonitorStatus(makeCheck(), monitor); - expect(result.thresholdBreaches).toBeUndefined(); - }); - - it("skips threshold evaluation when payload.data is undefined", async () => { - const monitor = makeHardwareMonitor(); - const { service, monitorsRepository } = createService(); - (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - - const response = makeHardwareResponse({} as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); - - expect(result.thresholdBreaches).toBeUndefined(); + expect(result.thresholdBreaches).toEqual({ cpu: false, memory: false, disk: false, temp: false }); + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); }); it("does not set thresholdBreaches for non-hardware monitors", async () => { @@ -939,7 +912,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); + const result = await service.updateMonitorStatus(makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); @@ -950,10 +923,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -964,10 +935,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -978,10 +947,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.cpu).toBe(false); expect(result.thresholdBreaches?.temp).toBe(false); @@ -993,10 +960,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); @@ -1007,10 +972,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -1023,8 +986,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); + const result = await service.updateMonitorStatus(check, monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(2); @@ -1046,11 +1009,8 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - status: false, - data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, - } as any); - const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor); + const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false }); + const result = await service.updateMonitorStatus(check, monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -1067,10 +1027,13 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const response = makeHardwareResponse({ - data: { cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {} }, - } as any); - await service.updateMonitorStatus(response, makeCheck(), monitor); + const check = makeHardwareCheck({ + cpu: { usage_percent: 0.9, temperature: [90] }, + memory: { usage_percent: 0.9 }, + disk: [{ usage_percent: 0.95 }], + host: {}, + }); + await service.updateMonitorStatus(check, monitor); expect(monitor.cpuAlertCounter).toBe(0); expect(monitor.memoryAlertCounter).toBe(0); @@ -1099,8 +1062,8 @@ describe("StatusService", () => { ...overrides, }); - const makeContainers = (containers: Partial[]): DockerStatusPayload => ({ - containers: containers.map((container, index) => ({ + const makeContainers = (containers: Partial[]): DockerContainerInfo[] => + containers.map((container, index) => ({ id: `container-${index}`, name: `container-${index}`, image: "nginx:latest", @@ -1108,23 +1071,20 @@ describe("StatusService", () => { status: "Up 3 hours", health: "none", ...container, - })), - summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 }, - }); + })); - const makeDockerResponse = (payload: DockerStatusPayload | null, overrides?: Partial>) => - makeStatusResponse({ - type: "docker", + const makeDockerCheck = (containers: DockerContainerInfo[] | null, overrides?: Partial) => + makeCheck({ status: true, - payload, + ...(containers ? { containers } : {}), ...overrides, - } as any) as MonitorStatusResponse; + }); - const run = async (monitor: Monitor, response: MonitorStatusResponse, check = makeCheck()) => { + const run = async (monitor: Monitor, check: Check) => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(response, check, monitor); + const result = await service.updateMonitorStatus(check, monitor); return { result, monitorsRepository }; }; @@ -1132,14 +1092,14 @@ describe("StatusService", () => { const unhealthy = { name: "api", health: "unhealthy" as const }; it("goes to breached on the first check with a stopped container", async () => { - const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("breached"); }); it("goes to breached on the first check with an unhealthy container", async () => { - const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([unhealthy]))); + const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([unhealthy]))); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("breached"); @@ -1147,7 +1107,7 @@ describe("StatusService", () => { it("stays up on a stopped container when both switches are off", async () => { const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false }); - const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped, unhealthy]))); + const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped, unhealthy]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -1155,35 +1115,35 @@ describe("StatusService", () => { it("stays up on a stopped container when only the unhealthy switch is on", async () => { const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); - const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("stays up on an exited container with code 0", async () => { - const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "job", state: "exited", exitCode: 0 }]))); + const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "job", state: "exited", exitCode: 0 }]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("stays breached without a transition while a container is still down", async () => { - const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([stopped]))); + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("breached"); }); it("stays up without a transition while every container is clean", async () => { - const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "web" }]))); + const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "web" }]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("recovers from breached to up on the first clean check", async () => { - const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([{ name: "web" }]))); + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([{ name: "web" }]))); expect(result.statusChanged).toBe(true); expect(result.prevStatus).toBe("breached"); @@ -1192,7 +1152,7 @@ describe("StatusService", () => { it("recovers from breached to up when the breaching switch is turned off", async () => { const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false }); - const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); @@ -1200,16 +1160,16 @@ describe("StatusService", () => { it("does not override down with breached when the host is unreachable", async () => { const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); - const response = makeDockerResponse(null, { status: false, code: 5000 }); - const { result } = await run(monitor, response, makeCheck({ status: false })); + const check = makeDockerCheck(null, { status: false, statusCode: 5000 }); + const { result } = await run(monitor, check); expect(result.monitor.status).toBe("down"); }); it("transitions from breached to down when the reachability threshold trips", async () => { const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); - const response = makeDockerResponse(null, { status: false, code: 5000 }); - const { result } = await run(monitor, response, makeCheck({ status: false })); + const check = makeDockerCheck(null, { status: false, statusCode: 5000 }); + const { result } = await run(monitor, check); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -1217,7 +1177,7 @@ describe("StatusService", () => { it("does not evaluate containers while the status window is short", async () => { const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 }); - const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -1225,15 +1185,15 @@ describe("StatusService", () => { it("does not evaluate a failed check with a null payload", async () => { const monitor = makeDockerMonitor({ status: "breached" }); - const response = makeDockerResponse(null, { status: false, code: 5000 }); - const { result } = await run(monitor, response, makeCheck({ status: false })); + const check = makeDockerCheck(null, { status: false, statusCode: 5000 }); + const { result } = await run(monitor, check); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("breached"); }); it("does not touch the hardware counters or thresholdBreaches", async () => { - const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped]))); expect(result.thresholdBreaches).toBeUndefined(); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index 0b168ef24..2e1335f88 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -92,7 +92,6 @@ const createPipeline = (overrides?: Record) => { }, checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }), - toStatusResponse: jest.fn().mockReturnValue({ monitorId: "m1", status: true, code: 200, message: "OK" }), createChecks: jest.fn().mockResolvedValue([]), }, networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, @@ -624,7 +623,7 @@ describe("WorkerPipeline", () => { it("loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => { const job = makeEvaluateJob("c1"); const { pipeline, deps } = createPipeline({ - checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }]) }, + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([makeCheck({ id: "c1" })]) }, }); await pipeline.handleEvaluate(job); @@ -661,7 +660,9 @@ describe("WorkerPipeline", () => { it("stops applying checks once the lease is lost so a second claimer does not double-apply", async () => { const job = makeEvaluateJob("c1", "c2", "c3"); const { pipeline, deps } = createPipeline({ - checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) }, + checksRepository: { + findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([makeCheck({ id: "c1" }), makeCheck({ id: "c2" }), makeCheck({ id: "c3" })]), + }, jobsRepository: { upsertEvaluate: jest.fn(), pullEvaluated: jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2 @@ -679,7 +680,7 @@ describe("WorkerPipeline", () => { it("pulls ids whose check no longer exists so they do not sit on the row forever", async () => { const job = makeEvaluateJob("c1", "c-gone"); const { pipeline, deps } = createPipeline({ - checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }]) }, // c-gone was cleaned up + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([makeCheck({ id: "c1" })]) }, // c-gone was cleaned up }); await pipeline.handleEvaluate(job); @@ -695,7 +696,7 @@ describe("WorkerPipeline", () => { const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1 const { pipeline, deps } = createPipeline({ monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(initialMonitor), updateById: jest.fn() }, - checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) }, + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([makeCheck({ id: "c1" }), makeCheck({ id: "c2" })]) }, statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(makeStatusChange({ monitor: postWriteMonitor })) }, }); @@ -705,36 +706,33 @@ describe("WorkerPipeline", () => { expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1); expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2); // First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first - expect(deps.statusService.updateMonitorStatus.mock.calls[0][2]).toBe(initialMonitor); - expect(deps.statusService.updateMonitorStatus.mock.calls[1][2]).toBe(postWriteMonitor); + expect(deps.statusService.updateMonitorStatus.mock.calls[0][1]).toBe(initialMonitor); + expect(deps.statusService.updateMonitorStatus.mock.calls[1][1]).toBe(postWriteMonitor); }); }); describe("evaluateCheck", () => { - it("threads status, check, and monitor through to the status service", async () => { + it("threads check and monitor through to the status service", async () => { const { pipeline, deps } = createPipeline(); - const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; - const check = { id: "check-1" } as any; + const check = makeCheck({ status: true, statusCode: 200, message: "OK" }); const monitor = makeMonitor({ status: "up" }); - await pipeline.evaluateCheck(status, check, monitor); + await pipeline.evaluateCheck(check, monitor); - expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor); + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(check, monitor); }); it("returns an evaluation that opens an incident on a down transition", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; - const check = { id: "check-1" } as any; + const check = makeCheck({ status: false, statusCode: 500, message: "Error" }); const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 }); const { pipeline } = createPipeline({ statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, }); - const result = await pipeline.evaluateCheck(status, check, makeMonitor()); + const result = await pipeline.evaluateCheck(check, makeMonitor()); expect(result).toMatchObject({ monitor: statusChange.monitor, - status, check, statusChange, decision: expect.objectContaining({ @@ -746,22 +744,20 @@ describe("WorkerPipeline", () => { }); it("returns an evaluation that resolves an incident on recovery", async () => { - const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down" }); const { pipeline } = createPipeline({ statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, }); - const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor({ status: "down" })); + const result = await pipeline.evaluateCheck(makeCheck({ status: true, statusCode: 200, message: "OK" }), makeMonitor({ status: "down" })); expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); }); it("produces a no-op decision when status did not change", async () => { - const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; const { pipeline } = createPipeline(); - const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor()); + const result = await pipeline.evaluateCheck(makeCheck({ status: true, statusCode: 200, message: "OK" }), makeMonitor()); expect(result.decision).toMatchObject({ shouldCreateIncident: false, @@ -773,12 +769,11 @@ describe("WorkerPipeline", () => { // ── degraded egress ─────────────────────────────────────────────────── it("short-circuits a check flagged with degraded egress without touching monitor status", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; - const check = { id: "check-1", egressStatus: "degraded" } as any; + const check = makeCheck({ status: false, statusCode: 500, message: "Error", egressStatus: "degraded" }); const monitor = makeMonitor({ status: "up" }); const { pipeline, deps } = createPipeline(); - const result = await pipeline.evaluateCheck(status, check, monitor); + const result = await pipeline.evaluateCheck(check, monitor); expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled(); expect(result.monitor).toBe(monitor); @@ -795,13 +790,12 @@ describe("WorkerPipeline", () => { }); it("evaluates a check flagged with egress ok as normal", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; - const check = { id: "check-1", egressStatus: "ok" } as any; + const check = makeCheck({ status: false, statusCode: 500, message: "Error", egressStatus: "ok" }); const { pipeline, deps } = createPipeline(); - await pipeline.evaluateCheck(status, check, makeMonitor()); + await pipeline.evaluateCheck(check, makeMonitor()); - expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything()); + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(check, expect.anything()); }); }); @@ -812,11 +806,7 @@ describe("WorkerPipeline", () => { const { pipeline } = createPipeline({ statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, }); - const result = await pipeline.evaluateCheck( - { monitorId: "m1", status: true, code: 200, message: "OK" } as any, - { id: "check-1" } as any, - makeMonitor() - ); + const result = await pipeline.evaluateCheck(makeCheck({ status: true, statusCode: 200, message: "OK" }), makeMonitor()); return result.decision; }; From a1b10c7469ff81428a11f486f9010673399ecb5c Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 13:51:30 -0700 Subject: [PATCH 14/29] remove unused param --- server/src/domain/incidents/incident.service.ts | 6 +++--- server/src/worker/reactors/reactor.incident.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index e77fa5498..c4208aafb 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -15,7 +15,7 @@ import { DateRange } from "@/types/query.js"; import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; export interface IIncidentService { - handleIncident(monitor: Monitor, code: number, decision: MonitorActionDecision, check: Check): Promise; + handleIncident(monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise; resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise; getIncidentsByTeam( teamId: string, @@ -54,7 +54,7 @@ export class IncidentService implements IIncidentService { this.notificationMessageBuilder = notificationMessageBuilder; } - handleIncident = async (monitor: Monitor, code: number, decision: MonitorActionDecision, check: Check): Promise => { + handleIncident = async (monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise => { if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) { return null; } @@ -65,7 +65,7 @@ export class IncidentService implements IIncidentService { if (activeIncident) { return activeIncident; } else { - let statusCode = code; + let statusCode = check.statusCode; let message: string | undefined; // For threshold breaches, use 9999 status code and build descriptive message diff --git a/server/src/worker/reactors/reactor.incident.ts b/server/src/worker/reactors/reactor.incident.ts index 24ca60e7b..390be7fd8 100644 --- a/server/src/worker/reactors/reactor.incident.ts +++ b/server/src/worker/reactors/reactor.incident.ts @@ -8,6 +8,6 @@ export class IncidentReactor implements IMonitorReactor { constructor(private incidentService: IIncidentService) {} react = async (evaluation: MonitorEvaluation) => { - await this.incidentService.handleIncident(evaluation.monitor, evaluation.statusChange.code, evaluation.decision, evaluation.check); + await this.incidentService.handleIncident(evaluation.monitor, evaluation.decision, evaluation.check); }; } From 7abc74ac98a6d1508812f87bab5f6b24e3731cdf Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 13:54:55 -0700 Subject: [PATCH 15/29] udpate tests --- .../integration/incidentLifecycle.test.ts | 44 ++++++++++++------- .../unit/services/incidentService.test.ts | 20 ++++----- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/server/test/integration/incidentLifecycle.test.ts b/server/test/integration/incidentLifecycle.test.ts index 037713b36..ff2ff2c53 100644 --- a/server/test/integration/incidentLifecycle.test.ts +++ b/server/test/integration/incidentLifecycle.test.ts @@ -7,6 +7,7 @@ import type { IUsersRepository } from "../../src/domain/users/user.repository.in import type { INotificationMessageBuilder } from "../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../src/domain/monitors/monitor.type.ts"; import type { MonitorActionDecision } from "../../src/worker/worker.interface.ts"; +import type { Check } from "../../src/domain/checks/check.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -30,6 +31,17 @@ const makeDecision = (overrides?: Partial): MonitorAction ...overrides, }); +const makeCheck = (statusCode: number): Check => ({ + id: "check-1", + metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" }, + status: statusCode < 400, + statusCode, + responseTime: 100, + message: statusCode < 400 ? "OK" : "Service Unavailable", + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", +}); + // ── Test suite ─────────────────────────────────────────────────────────────── describe("Incident lifecycle (integration)", () => { @@ -53,7 +65,7 @@ describe("Incident lifecycle (integration)", () => { const monitor = makeMonitor({ status: "down" }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }); - const incident = await service.handleIncident(monitor, 503, decision); + const incident = await service.handleIncident(monitor, decision, makeCheck(503)); expect(incident).not.toBeNull(); expect(incident!.monitorId).toBe("mon-1"); @@ -74,8 +86,8 @@ describe("Incident lifecycle (integration)", () => { const monitor = makeMonitor({ status: "down" }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }); - const first = await service.handleIncident(monitor, 503, decision); - const second = await service.handleIncident(monitor, 503, decision); + const first = await service.handleIncident(monitor, decision, makeCheck(503)); + const second = await service.handleIncident(monitor, decision, makeCheck(503)); expect(second!.id).toBe(first!.id); expect(repo.getAll()).toHaveLength(1); @@ -86,10 +98,10 @@ describe("Incident lifecycle (integration)", () => { it("auto-resolves an incident when monitor recovers", async () => { const monitor = makeMonitor({ status: "down" }); const createDecision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }); - const created = await service.handleIncident(monitor, 503, createDecision); + const created = await service.handleIncident(monitor, createDecision, makeCheck(503)); const resolveDecision = makeDecision({ shouldResolveIncident: true }); - const resolved = await service.handleIncident(monitor, 200, resolveDecision); + const resolved = await service.handleIncident(monitor, resolveDecision, makeCheck(200)); expect(resolved).not.toBeNull(); expect(resolved!.id).toBe(created!.id); @@ -104,7 +116,7 @@ describe("Incident lifecycle (integration)", () => { const monitor = makeMonitor({ status: "up" }); const decision = makeDecision({ shouldResolveIncident: true }); - const result = await service.handleIncident(monitor, 200, decision); + const result = await service.handleIncident(monitor, decision, makeCheck(200)); expect(result).toBeNull(); }); @@ -114,7 +126,7 @@ describe("Incident lifecycle (integration)", () => { it("manually resolves an active incident with comment", async () => { const monitor = makeMonitor({ status: "down" }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }); - const created = await service.handleIncident(monitor, 500, decision); + const created = await service.handleIncident(monitor, decision, makeCheck(500)); const resolved = await service.resolveIncident(created!.id, "user-1", "team-1", "Root cause identified", "user@test.com"); @@ -129,7 +141,7 @@ describe("Incident lifecycle (integration)", () => { it("throws when manually resolving an already-resolved incident", async () => { const monitor = makeMonitor({ status: "down" }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }); - const created = await service.handleIncident(monitor, 500, decision); + const created = await service.handleIncident(monitor, decision, makeCheck(500)); await service.resolveIncident(created!.id, "user-1", "team-1"); @@ -146,7 +158,7 @@ describe("Incident lifecycle (integration)", () => { ]); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - const incident = await service.handleIncident(monitor, 200, decision, { monitorId: "mon-1" } as any); + const incident = await service.handleIncident(monitor, decision, makeCheck(200)); expect(incident!.statusCode).toBe(9999); expect(incident!.message).toBe("CPU: 92% (threshold: 80%), MEMORY: 88% (threshold: 85%)"); @@ -158,7 +170,7 @@ describe("Incident lifecycle (integration)", () => { const monitor = makeMonitor({ status: "up" }); const decision = makeDecision(); - const result = await service.handleIncident(monitor, 200, decision); + const result = await service.handleIncident(monitor, decision, makeCheck(200)); expect(result).toBeNull(); expect(repo.getAll()).toHaveLength(0); @@ -170,19 +182,19 @@ describe("Incident lifecycle (integration)", () => { const monitor = makeMonitor({ status: "down" }); // First outage - const first = await service.handleIncident(monitor, 503, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" })); + const first = await service.handleIncident(monitor, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), makeCheck(503)); expect(first!.status).toBe(true); // Manually resolved await service.resolveIncident(first!.id, "user-1", "team-1", "Restarted server"); // Second outage — new incident since previous was resolved - const second = await service.handleIncident(monitor, 502, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" })); + const second = await service.handleIncident(monitor, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), makeCheck(502)); expect(second!.id).not.toBe(first!.id); expect(second!.statusCode).toBe(502); // Auto-resolve - const resolved = await service.handleIncident(monitor, 200, makeDecision({ shouldResolveIncident: true })); + const resolved = await service.handleIncident(monitor, makeDecision({ shouldResolveIncident: true }), makeCheck(200)); expect(resolved!.id).toBe(second!.id); expect(resolved!.status).toBe(false); expect(resolved!.resolutionType).toBe("automatic"); @@ -200,14 +212,14 @@ describe("Incident lifecycle (integration)", () => { const monitorB = makeMonitor({ id: "mon-b", status: "down" }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }); - const incidentA = await service.handleIncident(monitorA, 500, decision); - const incidentB = await service.handleIncident(monitorB, 502, decision); + const incidentA = await service.handleIncident(monitorA, decision, makeCheck(500)); + const incidentB = await service.handleIncident(monitorB, decision, makeCheck(502)); expect(incidentA!.id).not.toBe(incidentB!.id); expect(repo.getAll()).toHaveLength(2); // Resolving monitor A does not affect monitor B - const resolved = await service.handleIncident(monitorA, 200, makeDecision({ shouldResolveIncident: true })); + const resolved = await service.handleIncident(monitorA, makeDecision({ shouldResolveIncident: true }), makeCheck(200)); expect(resolved!.id).toBe(incidentA!.id); const bStillActive = await repo.findActiveByMonitorId("mon-b", "team-1"); diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index f5f978327..44d0a23cf 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -110,7 +110,7 @@ describe("IncidentService", () => { describe("handleIncident", () => { it("returns null when neither create nor resolve is requested", async () => { const { service } = createService(); - const result = await service.handleIncident(makeMonitor(), 200, makeDecision(), makeCheck()); + const result = await service.handleIncident(makeMonitor(), makeDecision(), makeCheck()); expect(result).toBeNull(); }); @@ -121,9 +121,8 @@ describe("IncidentService", () => { const result = await service.handleIncident( makeMonitor(), - 500, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), - makeCheck() + makeCheck({ statusCode: 500 }) ); expect(result).toBe(existing); @@ -138,9 +137,8 @@ describe("IncidentService", () => { const result = await service.handleIncident( makeMonitor(), - 500, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), - makeCheck() + makeCheck({ statusCode: 500 }) ); expect(result).toBe(created); @@ -166,7 +164,7 @@ describe("IncidentService", () => { const monitor = makeMonitor({ type: "hardware" }); const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(monitor, 200, decision, check); + await service.handleIncident(monitor, decision, check); expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check); expect(incidentsRepository.create).toHaveBeenCalledWith( @@ -185,7 +183,7 @@ describe("IncidentService", () => { (notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([]); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), 200, decision, makeCheck()); + await service.handleIncident(makeMonitor(), decision, makeCheck()); expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" })); }); @@ -207,7 +205,7 @@ describe("IncidentService", () => { containerSummary: { total: 3, running: 2, stopped: 1, unhealthy: 1 }, }); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(monitor, 200, decision, check); + await service.handleIncident(monitor, decision, check); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" }) @@ -226,7 +224,7 @@ describe("IncidentService", () => { ]); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), 200, decision, makeCheck()); + await service.handleIncident(makeMonitor(), decision, makeCheck()); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ message: "CPU: 95% (threshold: 80%), MEMORY: 90% (threshold: 80%)" }) @@ -240,7 +238,7 @@ describe("IncidentService", () => { (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(active); (incidentsRepository.updateById as jest.Mock).mockResolvedValue(resolved); - const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true }), makeCheck()); + const result = await service.handleIncident(makeMonitor(), makeDecision({ shouldResolveIncident: true }), makeCheck()); expect(result).toBe(resolved); expect(incidentsRepository.updateById).toHaveBeenCalledWith( @@ -254,7 +252,7 @@ describe("IncidentService", () => { const { service, incidentsRepository } = createService(); (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); - const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true }), makeCheck()); + const result = await service.handleIncident(makeMonitor(), makeDecision({ shouldResolveIncident: true }), makeCheck()); expect(result).toBeNull(); }); From b0ed75382ebd1f533da86177e0d55996c451911e Mon Sep 17 00:00:00 2001 From: Malena Aguiar Date: Wed, 23 Sep 2026 17:57:28 -0600 Subject: [PATCH 16/29] feat(dashboard): add code refactors --- client/src/Features/UI/uiSlice.ts | 12 ++++- .../components/cards/CurrentlyDownCard.tsx | 7 ++- .../{LowestUptimeCard.tsx => UptimeCard.tsx} | 44 ++++++++++++++++--- client/src/Pages/Dashboard/index.tsx | 4 +- client/src/Types/Dashboard.ts | 5 ++- client/src/locales/en.json | 5 ++- 6 files changed, 61 insertions(+), 16 deletions(-) rename client/src/Pages/Dashboard/components/cards/{LowestUptimeCard.tsx => UptimeCard.tsx} (52%) diff --git a/client/src/Features/UI/uiSlice.ts b/client/src/Features/UI/uiSlice.ts index 55e83333c..aad3b63f5 100644 --- a/client/src/Features/UI/uiSlice.ts +++ b/client/src/Features/UI/uiSlice.ts @@ -1,4 +1,8 @@ -import { dashboardCardKeys, type DashboardCardKey } from "@/Types/Dashboard"; +import { + dashboardCardKeys, + type DashboardCardKey, + type DashboardSortOrder, +} from "@/Types/Dashboard"; import { createSlice, type PayloadAction } from "@reduxjs/toolkit"; type ThemeMode = "light" | "dark"; @@ -38,6 +42,7 @@ interface UIState { starPromptOpen: boolean; chartType: ChartType; dashboardVisibleCards: DashboardCardKey[]; + dashboardUptimeSortOrder: DashboardSortOrder; } const initialMode: ThemeMode = window?.matchMedia?.("(prefers-color-scheme: dark)") @@ -78,6 +83,7 @@ const initialState: UIState = { starPromptOpen: true, chartType: "histogram", dashboardVisibleCards: [...dashboardCardKeys], + dashboardUptimeSortOrder: "ascending", }; const uiSlice = createSlice({ @@ -124,6 +130,9 @@ const uiSlice = createSlice({ setDashboardVisibleCards: (state, action: PayloadAction) => { state.dashboardVisibleCards = action.payload; }, + setDashboardUptimeSortOrder: (state, action: PayloadAction) => { + state.dashboardUptimeSortOrder = action.payload; + }, }, }); @@ -141,4 +150,5 @@ export const { setStarPromptOpen, setChartType, setDashboardVisibleCards, + setDashboardUptimeSortOrder, } = uiSlice.actions; diff --git a/client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx b/client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx index 478b2efa3..f7e7cdd8b 100644 --- a/client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx +++ b/client/src/Pages/Dashboard/components/cards/CurrentlyDownCard.tsx @@ -4,9 +4,8 @@ import { useTheme } from "@mui/material/styles"; import { useTranslation } from "react-i18next"; import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard"; import { Dot } from "@/Components/design-elements"; -import { getStatusColor } from "@/Utils/MonitorUtils"; import { typographyLevels } from "@/Utils/Theme/Palette"; -import { LAYOUT, SPACING } from "@/Utils/Theme/constants"; +import { LAYOUT } from "@/Utils/Theme/constants"; import type { Monitor } from "@/Types/Monitor"; interface CurrentlyDownCardProps { @@ -32,7 +31,7 @@ export const CurrentlyDownCard = ({ monitors }: CurrentlyDownCardProps) => { {t("pages.dashboard.currentlyDown.empty")} ) : ( - + {downMonitors.map((monitor) => ( { alignItems="center" gap={theme.spacing(LAYOUT.XS)} > - + { +export const UptimeCard = ({ monitors }: UptimeCardProps) => { const theme = useTheme(); const { t } = useTranslation(); + const dispatch = useDispatch(); + const sortOrder = useSelector((state: RootState) => state.ui.dashboardUptimeSortOrder); + const direction = sortOrder === "ascending" ? 1 : -1; const ranked = monitors .filter((monitor) => monitor.status !== "paused") .filter( (monitor): monitor is Monitor & { uptimePercentage: number } => typeof monitor.uptimePercentage === "number" ) - .sort((a, b) => a.uptimePercentage - b.uptimePercentage) + .sort((a, b) => direction * (a.uptimePercentage - b.uptimePercentage)) .slice(0, TOP_N); return ( - + + dispatch( + setDashboardUptimeSortOrder( + sortOrder === "ascending" ? "descending" : "ascending" + ) + ) + } + > + + + } + > {ranked.length === 0 ? ( { {t("pages.dashboard.noData")} ) : ( - + {ranked.map((monitor) => { const paletteKey = getUptimePercentageColor(monitor.uptimePercentage); const color = theme.palette[paletteKey].main; @@ -48,7 +78,7 @@ export const LowestUptimeCard = ({ monitors }: LowestUptimeCardProps) => { key={monitor.id} label={monitor.name} value={formatPercentage(monitor.uptimePercentage)} - fillRatio={1 - monitor.uptimePercentage} + fillRatio={monitor.uptimePercentage} valueColor={color} /> ); diff --git a/client/src/Pages/Dashboard/index.tsx b/client/src/Pages/Dashboard/index.tsx index 1656b9709..fac8d15fc 100644 --- a/client/src/Pages/Dashboard/index.tsx +++ b/client/src/Pages/Dashboard/index.tsx @@ -15,7 +15,7 @@ import type { RootState, AppDispatch } from "@/store"; import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard"; import { CurrentlyDownCard } from "@/Pages/Dashboard/components/cards/CurrentlyDownCard"; -import { LowestUptimeCard } from "@/Pages/Dashboard/components/cards/LowestUptimeCard"; +import { UptimeCard } from "@/Pages/Dashboard/components/cards/UptimeCard"; import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard"; import { EditCardsModal } from "@/Pages/Dashboard/components/EditCardsModal"; import { type DashboardCardKey, dashboardCardKeys } from "@/Types/Dashboard"; @@ -51,7 +51,7 @@ const Dashboard = () => { const cardComponents: Record = { monitorStatus: , currentlyDown: , - lowestUptime: , + uptime: , monitorsByType: , }; diff --git a/client/src/Types/Dashboard.ts b/client/src/Types/Dashboard.ts index 6e418f5ed..4390402f6 100644 --- a/client/src/Types/Dashboard.ts +++ b/client/src/Types/Dashboard.ts @@ -1,7 +1,10 @@ export const dashboardCardKeys = [ "monitorStatus", "currentlyDown", - "lowestUptime", + "uptime", "monitorsByType", ] as const; export type DashboardCardKey = (typeof dashboardCardKeys)[number]; + +export const dashboardSortOrders = ["ascending", "descending"] as const; +export type DashboardSortOrder = (typeof dashboardSortOrders)[number]; diff --git a/client/src/locales/en.json b/client/src/locales/en.json index ba0e3e897..19878f77f 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -1925,9 +1925,12 @@ "cards": { "monitorStatus": "Monitor status", "currentlyDown": "Currently down", - "lowestUptime": "Lowest uptime", + "uptime": "Monitor uptime", "monitorsByType": "Monitors by type" }, + "uptimeCard": { + "toggleSortOrder": "Toggle sort order" + }, "currentlyDown": { "empty": "No monitors are down at this moment." }, From 21fb320f3a98aae54bae62f1ecbf797d40e242aa Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 11:46:04 -0700 Subject: [PATCH 17/29] add hardware metric key types --- server/src/domain/monitors/monitor.type.ts | 5 +++++ .../domain/notifications/notification.type.ts | 4 +++- server/src/service/statusService.ts | 19 +++++++++++-------- server/src/types/network.ts | 9 ++------- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index b033c560a..07b0ef2e4 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -240,3 +240,8 @@ export type GamesMap = Record; export type MonitorScheduleFields = Pick; export type DockerContainerLogsResult = DockerLogPage; + +export const HardwareMetricKeys = ["cpu", "memory", "disk", "temp"] as const; +export type HardwareMetricKey = (typeof HardwareMetricKeys)[number]; +export type HardwareBreaches = Record; +export type HardwareCounters = Record; diff --git a/server/src/domain/notifications/notification.type.ts b/server/src/domain/notifications/notification.type.ts index 7f2c7df1e..0082f2098 100644 --- a/server/src/domain/notifications/notification.type.ts +++ b/server/src/domain/notifications/notification.type.ts @@ -1,3 +1,5 @@ +import { HardwareMetricKey } from "@/domain/monitors/monitor.type.js"; + export const NotificationChannels = [ "email", "slack", @@ -85,7 +87,7 @@ export interface MonitorInfo { } export interface ThresholdBreach { - metric: "cpu" | "memory" | "disk" | "temp"; + metric: HardwareMetricKey; currentValue: number; threshold: number; unit: string; diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 64a6f2e61..0ceeb9540 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -1,7 +1,14 @@ import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js"; -import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js"; +import { + HardwareBreaches, + HardwareCounters, + HardwareMetricKeys, + MonitorStatuses, + type Monitor, + type MonitorStatus, +} from "@/domain/monitors/monitor.type.js"; import type { StatusChangeResult } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { ILogger } from "@/utils/logger.js"; @@ -12,10 +19,6 @@ import { findContainerBreaches } from "@/domain/docker/docker-alert.js"; const SERVICE_NAME = "StatusService"; const HARDWARE_ALERT_COUNTER_START = 5; -const HARDWARE_METRIC_KEYS = ["cpu", "memory", "disk", "temp"] as const; -type HardwareMetricKey = (typeof HARDWARE_METRIC_KEYS)[number]; -type HardwareBreaches = Record; -type HardwareCounters = Record; export interface IStatusService { updateRunningStats(check: Check, monitor: Monitor): Promise; @@ -111,7 +114,7 @@ export class StatusService implements IStatusService { // Update counters: decrement (floored at 0) if breached, reset to start otherwise. const nextCounters = { ...counters }; - for (const key of HARDWARE_METRIC_KEYS) { + for (const key of HardwareMetricKeys) { nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START; } @@ -123,8 +126,8 @@ export class StatusService implements IStatusService { if (!reachabilityDown) { // A counter can only reach zero via the decrement path, which only runs when that // metric is currently breaching — so anyCounterZero already implies anyBreached. - const anyCounterZero = HARDWARE_METRIC_KEYS.some((k) => nextCounters[k] === 0); - const allNormal = HARDWARE_METRIC_KEYS.every((k) => !breaches[k]); + const anyCounterZero = HardwareMetricKeys.some((k) => nextCounters[k] === 0); + const allNormal = HardwareMetricKeys.every((k) => !breaches[k]); if (anyCounterZero && currentStatus !== "breached") { nextStatus = "breached"; diff --git a/server/src/types/network.ts b/server/src/types/network.ts index a2cf78dbe..3ca2f107f 100644 --- a/server/src/types/network.ts +++ b/server/src/types/network.ts @@ -10,7 +10,7 @@ import type { ILighthouseAudit, } from "@/domain/checks/check.type.js"; import { DockerContainerInfo, DockerContainerLogs, DockerContainerSummary } from "@/domain/docker/docker.type.js"; -import type { DnsRecordType, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js"; +import type { DnsRecordType, HardwareBreaches, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js"; import type { QueryResult } from "gamedig"; @@ -184,12 +184,7 @@ export type StatusChangeResult = { prevStatus: MonitorStatus; code: number; timestamp: number; - thresholdBreaches?: { - cpu: boolean; - memory: boolean; - disk: boolean; - temp: boolean; - }; + thresholdBreaches?: HardwareBreaches; }; export type MonitorStatusResponseOverrides = Partial, "monitorId" | "teamId" | "type">>; From 91de9e7fb7984e6481b7be3b39c7d387964295d4 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 12:25:22 -0700 Subject: [PATCH 18/29] pass threshold breaches on rather than reconsructing --- .../src/domain/incidents/incident.service.ts | 6 +- .../notification.message-builder.ts | 111 ++++++------------ server/src/worker/worker.interface.ts | 9 +- server/src/worker/worker.pipeline.ts | 1 + 4 files changed, 39 insertions(+), 88 deletions(-) diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index c4208aafb..4adbf9b75 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -71,7 +71,7 @@ export class IncidentService implements IIncidentService { // For threshold breaches, use 9999 status code and build descriptive message if (decision.incidentReason === "threshold_breach") { statusCode = 9999; - message = this.buildThresholdBreachMessage(monitor, check); + message = this.buildThresholdBreachMessage(monitor, check, decision); } const incident = { @@ -96,14 +96,14 @@ export class IncidentService implements IIncidentService { return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); }; - private buildThresholdBreachMessage(monitor: Monitor, check: Check): string { + private buildThresholdBreachMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision): string { if (monitor.type === "docker") { const containers = check.containers ?? []; const breaches = findContainerBreaches(monitor, containers); return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert"; } - const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check); + const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check, decision.thresholdBreaches); if (breaches.length === 0) { return "Threshold breach detected"; diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 11d11c069..f53f9d609 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -1,4 +1,4 @@ -import type { Monitor } from "@/domain/monitors/monitor.type.js"; +import { HardwareMetricKeys, type HardwareBreaches, type HardwareMetricKey, type Monitor } from "@/domain/monitors/monitor.type.js"; import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { NotificationMessage, @@ -12,7 +12,7 @@ import type { Check } from "@/domain/checks/check.type.js"; export interface INotificationMessageBuilder { buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage; - extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[]; + extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[]; } const SERVICE_NAME = "NotificationMessageBuilder"; @@ -23,7 +23,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage { const type = this.determineNotificationType(decision, monitor); const severity = this.determineSeverity(type); - const content = this.buildContent(type, monitor, check); + const content = this.buildContent(type, monitor, check, decision); return { type, @@ -82,14 +82,14 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildContent(type: NotificationType, monitor: Monitor, check: Check): NotificationContent { + private buildContent(type: NotificationType, monitor: Monitor, check: Check, decision: MonitorActionDecision): NotificationContent { switch (type) { case "monitor_down": return this.buildMonitorDownContent(monitor, check); case "monitor_up": return this.buildMonitorUpContent(monitor); case "threshold_breach": - return this.buildThresholdBreachContent(monitor, check); + return this.buildThresholdBreachContent(monitor, check, decision.thresholdBreaches); case "threshold_resolved": return this.buildThresholdResolvedContent(monitor); case "container_breach": @@ -137,12 +137,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildThresholdBreachContent(monitor: Monitor, check: Check): NotificationContent { + private buildThresholdBreachContent(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): NotificationContent { const title = `Threshold Exceeded: ${monitor.name}`; const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`; const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`]; - const thresholds = this.extractThresholdBreaches(monitor, check); + const thresholds = this.extractThresholdBreaches(monitor, check, thresholdBreaches); return { title, @@ -191,83 +191,38 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - public extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[] { + private describeBreach(metric: HardwareMetricKey, monitor: Monitor, check: Check): ThresholdBreach { + switch (metric) { + case "cpu": { + const currentValue = (check.cpu?.usage_percent ?? 0) * 100; + return { metric, currentValue, threshold: monitor.cpuAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` }; + } + case "memory": { + const currentValue = (check.memory?.usage_percent ?? 0) * 100; + return { metric, currentValue, threshold: monitor.memoryAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` }; + } + case "disk": { + const currentValue = Math.max(0, ...(check.disk ?? []).map((disk) => disk?.usage_percent ?? 0)) * 100; + return { metric, currentValue, threshold: monitor.diskAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` }; + } + case "temp": { + const currentValue = Math.max(0, ...(check.cpu?.temperature ?? [])); + return { metric, currentValue, threshold: monitor.tempAlertThreshold, unit: "°C", formattedValue: `${currentValue.toFixed(1)}°C` }; + } + } + } + + public extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] { const breaches: ThresholdBreach[] = []; // Check if this is a hardware monitor with threshold data - if (monitor.type !== "hardware") { + if (monitor.type !== "hardware" || thresholdBreaches === undefined) { return breaches; } - // Note: usage_percent values in hardware payload are decimals (0-1) - if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && check.cpu?.usage_percent !== undefined) { - const cpuUsageDecimal = check.cpu.usage_percent; - const cpuPercent = cpuUsageDecimal * 100; - const threshold = monitor.cpuAlertThreshold; - if (cpuPercent > threshold) { - breaches.push({ - metric: "cpu", - currentValue: cpuPercent, - threshold, - unit: "%", - formattedValue: `${cpuPercent.toFixed(1)}%`, - }); - } - } - - // Memory threshold breach - if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && check.memory?.usage_percent !== undefined) { - const memoryUsageDecimal = check.memory.usage_percent; - const memoryPercent = memoryUsageDecimal * 100; - const threshold = monitor.memoryAlertThreshold; - if (memoryPercent > threshold) { - breaches.push({ - metric: "memory", - currentValue: memoryPercent, - threshold, - unit: "%", - formattedValue: `${memoryPercent.toFixed(1)}%`, - }); - } - } - - // Disk threshold breach - if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(check.disk)) { - // Find the highest disk usage - let maxDiskUsageDecimal = 0; - for (const disk of check.disk) { - if (disk.usage_percent !== undefined && disk.usage_percent > maxDiskUsageDecimal) { - maxDiskUsageDecimal = disk.usage_percent; - } - } - const maxDiskPercent = maxDiskUsageDecimal * 100; - const threshold = monitor.diskAlertThreshold; - if (maxDiskPercent > threshold) { - breaches.push({ - metric: "disk", - currentValue: maxDiskPercent, - threshold, - unit: "%", - formattedValue: `${maxDiskPercent.toFixed(1)}%`, - }); - } - } - - // Temperature threshold breach - if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && check.cpu?.temperature) { - // Temperature is an array in cpu.temperature - const temps = Array.isArray(check.cpu.temperature) ? check.cpu.temperature : [check.cpu.temperature]; - const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t))); - const threshold = monitor.tempAlertThreshold; - if (maxTemp > threshold) { - breaches.push({ - metric: "temp", - currentValue: maxTemp, - threshold, - unit: "°C", - formattedValue: `${maxTemp.toFixed(1)}°C`, - }); - } + for (const metric of HardwareMetricKeys) { + if (!thresholdBreaches[metric]) continue; + breaches.push(this.describeBreach(metric, monitor, check)); } return breaches; diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 4b5bba2c5..92f8eb20c 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,4 +1,4 @@ -import { Monitor } from "@/domain/monitors/monitor.type.js"; +import { HardwareBreaches, Monitor } from "@/domain/monitors/monitor.type.js"; import { Check } from "@/domain/checks/check.type.js"; import { Job, JobType } from "@/domain/jobs/job.type.js"; import { StatusChangeResult } from "@/types/network.js"; @@ -13,12 +13,7 @@ export interface MonitorActionDecision { shouldSendNotification: boolean; incidentReason: "status_down" | "threshold_breach" | null; notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null; - thresholdBreaches?: { - cpu?: boolean; - memory?: boolean; - disk?: boolean; - temp?: boolean; - }; + thresholdBreaches?: HardwareBreaches; } export type MonitorEvaluation = { diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index 1faef2a11..ceac5293f 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -320,6 +320,7 @@ export class WorkerPipeline implements IWorkerPipeline { decision.incidentReason = "status_down"; decision.notificationReason = "status_change"; } else if (monitor.status === "breached") { + decision.thresholdBreaches = statusChange.thresholdBreaches; decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; From 6b75f9bf7ce4c23d11fb27c75feebd231a758517 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 12:41:33 -0700 Subject: [PATCH 19/29] remove statusChange from decision, no longer needed --- server/src/worker/worker.interface.ts | 2 -- server/src/worker/worker.pipeline.ts | 48 ++++++++------------------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 92f8eb20c..6e9ec3d49 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,7 +1,6 @@ import { HardwareBreaches, Monitor } from "@/domain/monitors/monitor.type.js"; import { Check } from "@/domain/checks/check.type.js"; import { Job, JobType } from "@/domain/jobs/job.type.js"; -import { StatusChangeResult } from "@/types/network.js"; import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; @@ -19,7 +18,6 @@ export interface MonitorActionDecision { export type MonitorEvaluation = { monitor: Monitor; check: Check; - statusChange: StatusChangeResult; // from statusService.updateMonitorStatus decision: MonitorActionDecision; }; diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index ceac5293f..f1447fbd7 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -3,7 +3,6 @@ import { ICheckService } from "@/domain/checks/check.service.js"; import { Check } from "@/domain/checks/check.type.js"; import { IDockerLogsService } from "@/domain/docker/docker-log.service.js"; import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js"; -import type { EgressStatus } from "@/domain/egress/egress.type.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { PendingCheck } from "@/domain/jobs/job.type.js"; import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; @@ -21,7 +20,13 @@ import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js"; const SERVICE_NAME = "WorkerPipeline"; - +const NO_ACTION: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, +}; // The three entry points: the queue calls the two handlers, the buffer calls ingestChecks. // produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer. export interface IWorkerPipeline { @@ -128,7 +133,7 @@ export class WorkerPipeline implements IWorkerPipeline { // Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the // target. Null means the egress check is disabled (or failed internally) and the check is treated as usual. - const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null; + const egressStatus = this.isTransportFailure(monitor, status) ? await this.deps.egressService.assessAfterFailure() : null; if (egressStatus !== null) { check.egressStatus = egressStatus; } @@ -171,20 +176,6 @@ export class WorkerPipeline implements IWorkerPipeline { return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status); } - // The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown. - private async assessEgress(monitorId: string): Promise { - try { - return await this.deps.egressService.assessAfterFailure(); - } catch (error: unknown) { - this.deps.logger.warn({ - message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "assessEgress", - }); - return null; - } - } - // **************************************************************** // Stage 2: Ingestion // Called by the buffer, checks are inserted and evaluation jobs created @@ -252,7 +243,7 @@ export class WorkerPipeline implements IWorkerPipeline { }); return; } - current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check + current = evaluation.monitor; // fresh statusWindow/status/counters for the next check } // Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever @@ -277,18 +268,11 @@ export class WorkerPipeline implements IWorkerPipeline { service: SERVICE_NAME, method: "evaluateCheck", }); - const decision: MonitorActionDecision = { - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }; + return { monitor, check, - statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: check.statusCode, timestamp: Date.now() }, - decision, + decision: { ...NO_ACTION }, }; }; @@ -300,18 +284,12 @@ export class WorkerPipeline implements IWorkerPipeline { const statusChange = await this.deps.statusService.updateMonitorStatus(check, monitor); const decision = this.decide(statusChange); - return { monitor: statusChange.monitor, check, statusChange, decision }; + return { monitor: statusChange.monitor, check, decision }; }; private decide = (statusChange: StatusChangeResult): MonitorActionDecision => { const { monitor, statusChanged, prevStatus } = statusChange; - const decision: MonitorActionDecision = { - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }; + const decision: MonitorActionDecision = { ...NO_ACTION }; if (!statusChanged) return decision; if (monitor.status === "down") { From 11c89d150aea382057d90002f847c136b3494bf4 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 12:58:28 -0700 Subject: [PATCH 20/29] update tests --- .../unit/services/incidentService.test.ts | 5 +- .../notificationMessageBuilder.test.ts | 395 +++++------------- .../test/unit/services/workerPipeline.test.ts | 66 ++- 3 files changed, 152 insertions(+), 314 deletions(-) diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index 44d0a23cf..fa1edea07 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -163,10 +163,11 @@ describe("IncidentService", () => { const monitor = makeMonitor({ type: "hardware" }); const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } }); - const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false }; + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach", thresholdBreaches }); await service.handleIncident(monitor, decision, check); - expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check); + expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check, thresholdBreaches); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ statusCode: 9999, diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index 3937e3bcf..199d80101 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, beforeEach } from "@jest/globals"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; +import type { HardwareBreaches, Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; @@ -119,7 +119,10 @@ describe("NotificationMessageBuilder", () => { it("builds a threshold_breach message", () => { const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); - const decision = makeDecision({ notificationReason: "threshold_breach" }); + const decision = makeDecision({ + notificationReason: "threshold_breach", + thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false }, + }); const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); const msg = builder.buildMessage(monitor, check, decision, "https://app.example.com"); @@ -346,21 +349,26 @@ describe("NotificationMessageBuilder", () => { }); describe("threshold_breach", () => { - it("includes threshold breaches in content", () => { + it("includes the breaches flagged on the decision in content", () => { + const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); + const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); + const decision = makeDecision({ + notificationReason: "threshold_breach", + thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false }, + }); + + const msg = builder.buildMessage(monitor, check, decision, ""); + + expect(msg.content.thresholds).toEqual([expect.objectContaining({ metric: "cpu", currentValue: 90, threshold: 80 })]); + }); + + it("leaves thresholds empty when the decision carries no breaches", () => { const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); const msg = builder.buildMessage(monitor, check, makeDecision({ notificationReason: "threshold_breach" }), ""); - expect(msg.content.thresholds).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - metric: "cpu", - currentValue: 90, - threshold: 80, - }), - ]) - ); + expect(msg.content.thresholds).toEqual([]); }); }); @@ -480,346 +488,151 @@ describe("NotificationMessageBuilder", () => { // ── extractThresholdBreaches ───────────────────────────────────────── describe("extractThresholdBreaches", () => { - it("returns empty array for non-hardware monitor", () => { - const monitor = makeMonitor({ type: "http" }); - const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches).toEqual([]); - }); - - it("returns empty array when the check carries no hardware data", () => { - const monitor = makeMonitor({ + const none: HardwareBreaches = { cpu: false, memory: false, disk: false, temp: false }; + const hardwareMonitor = (overrides?: Partial) => + makeMonitor({ type: "hardware", cpuAlertThreshold: 80, - memoryAlertThreshold: 80, - diskAlertThreshold: 80, - tempAlertThreshold: 80, + memoryAlertThreshold: 70, + diskAlertThreshold: 85, + tempAlertThreshold: 65, + ...overrides, }); - const check = makeCheck(); - const breaches = builder.extractThresholdBreaches(monitor, check); + it("returns empty when no breaches are supplied", () => { + const check = makeCheck({ cpu: { usage_percent: 0.99, temperature: [99] } }); - expect(breaches).toEqual([]); + expect(builder.extractThresholdBreaches(hardwareMonitor(), check, undefined)).toEqual([]); }); - // ── CPU ────────────────────────────────────────────────────────── + it("returns empty when every flag is false, whatever the check says", () => { + const check = makeCheck({ + cpu: { usage_percent: 0.99, temperature: [99] }, + memory: { usage_percent: 0.99 }, + disk: [{ usage_percent: 0.99 }], + }); - describe("cpu threshold", () => { - it("detects CPU breach when usage exceeds threshold", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); + expect(builder.extractThresholdBreaches(hardwareMonitor(), check, none)).toEqual([]); + }); + + it("returns empty for a non-hardware monitor even when flags are set", () => { + const check = makeCheck({ cpu: { usage_percent: 0.99 } }); + + expect(builder.extractThresholdBreaches(makeMonitor({ type: "http" }), check, { ...none, cpu: true })).toEqual([]); + }); + + // The flags are the verdict. The builder formats what it is told is breaching and never re-runs the comparison. + it("reports a flagged metric without re-testing it against the threshold", () => { + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } }); + + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); + + expect(breaches).toEqual([{ metric: "cpu", currentValue: 50, threshold: 80, unit: "%", formattedValue: "50.0%" }]); + }); + + describe("cpu", () => { + it("formats cpu usage as a percentage when flagged", () => { const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "cpu", - currentValue: 90, - threshold: 80, - unit: "%", - formattedValue: "90.0%", - }) - ); + expect(breaches).toEqual([{ metric: "cpu", currentValue: 90, threshold: 80, unit: "%", formattedValue: "90.0%" }]); }); - it("does not report CPU breach when usage is below threshold", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } }); + it("ignores cpu when not flagged", () => { + const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpuAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: undefined }); - const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpuAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: null as any }); - const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpu usage_percent is undefined", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: undefined } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpu object is undefined", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const check = makeCheck({ cpu: undefined }); - - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true }); expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); }); }); - // ── Memory ─────────────────────────────────────────────────────── - - describe("memory threshold", () => { - it("detects memory breach when usage exceeds threshold", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); + describe("memory", () => { + it("formats memory usage as a percentage when flagged", () => { const check = makeCheck({ memory: { usage_percent: 0.85 } }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "memory", - currentValue: 85, - threshold: 70, - unit: "%", - formattedValue: "85.0%", - }) - ); + expect(breaches).toEqual([{ metric: "memory", currentValue: 85, threshold: 70, unit: "%", formattedValue: "85.0%" }]); }); - it("does not report memory breach when usage is below threshold", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 90 }); - const check = makeCheck({ memory: { usage_percent: 0.5 } }); + it("ignores memory when not flagged", () => { + const check = makeCheck({ memory: { usage_percent: 0.85 } }); - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memoryAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: undefined }); - const check = makeCheck({ memory: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memoryAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: null as any }); - const check = makeCheck({ memory: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memory usage_percent is undefined", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); - const check = makeCheck({ memory: { usage_percent: undefined } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memory object is undefined", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); - const check = makeCheck({ memory: undefined }); - - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); }); }); - // ── Disk ───────────────────────────────────────────────────────── - - describe("disk threshold", () => { - it("detects disk breach using highest usage across multiple disks", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 }); + describe("disk", () => { + it("formats the highest usage across disks when flagged", () => { const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.95 }, { usage_percent: 0.7 }] }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "disk", - currentValue: 95, - threshold: 80, - formattedValue: "95.0%", - }) - ); + expect(breaches).toEqual([{ metric: "disk", currentValue: 95, threshold: 85, unit: "%", formattedValue: "95.0%" }]); }); - it("does not report disk breach when all disks are below threshold", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 90 }); - const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.6 }] }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disk check when diskAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: undefined }); - const check = makeCheck({ disk: [{ usage_percent: 0.99 }] }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disk check when diskAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: null as any }); - const check = makeCheck({ disk: [{ usage_percent: 0.99 }] }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disk check when disk is not an array", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 }); - const check = makeCheck({ disk: "not-an-array" as any }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disks with undefined usage_percent", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 50 }); + it("treats a disk with no usage figure as zero when finding the highest", () => { const check = makeCheck({ disk: [{ usage_percent: undefined }, { usage_percent: 0.3 }] }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true }); + + expect(breaches).toEqual([expect.objectContaining({ metric: "disk", currentValue: 30 })]); + }); + + it("ignores disk when not flagged", () => { + const check = makeCheck({ disk: [{ usage_percent: 0.95 }] }); + + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true }); expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); }); }); - // ── Temperature ────────────────────────────────────────────────── - - describe("temperature threshold", () => { - it("detects temperature breach from array of temperatures", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); + describe("temperature", () => { + it("formats the highest core temperature when flagged", () => { const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [65, 75, 68] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "temp", - currentValue: 75, - threshold: 70, - unit: "°C", - formattedValue: "75.0°C", - }) - ); + expect(breaches).toEqual([{ metric: "temp", currentValue: 75, threshold: 65, unit: "°C", formattedValue: "75.0°C" }]); }); - it("handles single temperature value (non-array)", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: 80 as any } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "temp", - currentValue: 80, - threshold: 70, - }) - ); - }); - - it("does not report temperature breach when below threshold", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [60, 65] } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - // A temperature exactly AT the threshold must not report a breach here, matching - // StatusService.computeHardwareStatus's own strict `>` comparison (cpu/memory/disk - // already agree with this file on `>`; temp alone used to diverge with `>=`, so an - // incident-driving check at exactly the threshold never fired but the notification - // content for a breach some OTHER metric triggered could still list temp as breaching). - it("does not report temperature breach when exactly at threshold", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [80] } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when tempAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined }); + it("ignores temperature when not flagged", () => { const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when tempAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: null as any }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when cpu.temperature is falsy", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: null as any } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when cpu object is undefined", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const check = makeCheck({ cpu: undefined }); - - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); }); - // ── Combined ───────────────────────────────────────────────────── - - describe("combined thresholds", () => { - it("detects multiple breaches simultaneously", () => { - const monitor = makeMonitor({ - type: "hardware", - cpuAlertThreshold: 80, - memoryAlertThreshold: 70, - diskAlertThreshold: 85, - tempAlertThreshold: 65, - }); + describe("combined", () => { + it("reports every flagged metric in cpu, memory, disk, temp order", () => { const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [70] }, memory: { usage_percent: 0.85 }, disk: [{ usage_percent: 0.95 }], }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { cpu: true, memory: true, disk: true, temp: true }); - const metrics = breaches.map((b) => b.metric); - expect(metrics).toContain("cpu"); - expect(metrics).toContain("memory"); - expect(metrics).toContain("disk"); - expect(metrics).toContain("temp"); + expect(breaches.map((b) => b.metric)).toEqual(["cpu", "memory", "disk", "temp"]); + }); + + it("reports only the flagged subset", () => { + const check = makeCheck({ + cpu: { usage_percent: 0.9, temperature: [70] }, + memory: { usage_percent: 0.85 }, + disk: [{ usage_percent: 0.95 }], + }); + + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true, temp: true }); + + expect(breaches.map((b) => b.metric)).toEqual(["memory", "temp"]); }); }); }); diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index 2e1335f88..cf4ae4293 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -518,24 +518,6 @@ describe("WorkerPipeline", () => { expect(check.egressStatus).toBe("ok"); }); - it("buffers the check when the egress assessment rejects", async () => { - const check: Record = { id: "check-1" }; - const { pipeline, deps } = createPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) }, - }); - - const result = await pipeline.produce(makeMonitor()); - - expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check); - expect(check).not.toHaveProperty("egressStatus"); - expect(result).toEqual({ status: failingStatus, check }); - expect(deps.logger.warn).toHaveBeenCalledWith( - expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "WorkerPipeline", method: "assessEgress" }) - ); - }); - it("leaves the field unset on a failing check when the egress check is disabled", async () => { const check: Record = { id: "check-1" }; const { pipeline, deps } = createPipeline({ @@ -734,7 +716,6 @@ describe("WorkerPipeline", () => { expect(result).toMatchObject({ monitor: statusChange.monitor, check, - statusChange, decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, @@ -777,8 +758,7 @@ describe("WorkerPipeline", () => { expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled(); expect(result.monitor).toBe(monitor); - expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 }); - expect(result.statusChange.timestamp).toEqual(expect.any(Number)); + expect(result).not.toHaveProperty("statusChange"); expect(result.decision).toEqual({ shouldCreateIncident: false, shouldResolveIncident: false, @@ -865,5 +845,49 @@ describe("WorkerPipeline", () => { const decision = await decide(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" })); expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false }); }); + + it("carries the threshold breaches onto a breached decision", async () => { + const thresholdBreaches = { cpu: true, memory: false, disk: true, temp: false }; + const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up", thresholdBreaches })); + expect(decision.thresholdBreaches).toEqual(thresholdBreaches); + }); + + it("leaves threshold breaches off a down decision even when the status change carries them", async () => { + const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false }; + const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500, thresholdBreaches })); + expect(decision).not.toHaveProperty("thresholdBreaches"); + }); + + it("leaves threshold breaches off a no-op decision", async () => { + const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false }; + const decision = await decide(makeStatusChange({ status: "breached", statusChanged: false, prevStatus: "breached", thresholdBreaches })); + expect(decision).not.toHaveProperty("thresholdBreaches"); + }); + + // Each decision starts from a shared template. A transition must not leak its flags into the next decision + // the same pipeline makes, whether that is a no-op evaluation or a degraded-egress short-circuit. + it("starts every decision clean after a transition on the same pipeline", async () => { + const updateMonitorStatus = jest + .fn() + .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 })) + .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "down", code: 500 })); + const { pipeline } = createPipeline({ statusService: { updateMonitorStatus } }); + const failing = makeCheck({ status: false, statusCode: 500, message: "Error" }); + + const first = await pipeline.evaluateCheck(failing, makeMonitor()); + const second = await pipeline.evaluateCheck(failing, makeMonitor({ status: "down" })); + const degraded = await pipeline.evaluateCheck(makeCheck({ ...failing, egressStatus: "degraded" }), makeMonitor({ status: "down" })); + + expect(first.decision.shouldCreateIncident).toBe(true); + const clean = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }; + expect(second.decision).toEqual(clean); + expect(degraded.decision).toEqual(clean); + }); }); }); From f2c23636fe848b5347cf6c49a6acd4b13ebdeb78 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 13:20:03 -0700 Subject: [PATCH 21/29] type only imports, add empty guard for threshold breaches do Docker monitors don't get this --- server/src/domain/notifications/notification.type.ts | 2 +- server/src/service/statusService.ts | 4 ++-- server/src/worker/worker.pipeline.ts | 2 +- server/test/unit/services/workerPipeline.test.ts | 7 +++++++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/server/src/domain/notifications/notification.type.ts b/server/src/domain/notifications/notification.type.ts index 0082f2098..b696bf752 100644 --- a/server/src/domain/notifications/notification.type.ts +++ b/server/src/domain/notifications/notification.type.ts @@ -1,4 +1,4 @@ -import { HardwareMetricKey } from "@/domain/monitors/monitor.type.js"; +import type { HardwareMetricKey } from "@/domain/monitors/monitor.type.js"; export const NotificationChannels = [ "email", diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 0ceeb9540..48b75f1da 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -2,10 +2,10 @@ import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.re import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js"; import { - HardwareBreaches, - HardwareCounters, HardwareMetricKeys, MonitorStatuses, + type HardwareBreaches, + type HardwareCounters, type Monitor, type MonitorStatus, } from "@/domain/monitors/monitor.type.js"; diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index f1447fbd7..3d703d3eb 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -298,7 +298,7 @@ export class WorkerPipeline implements IWorkerPipeline { decision.incidentReason = "status_down"; decision.notificationReason = "status_change"; } else if (monitor.status === "breached") { - decision.thresholdBreaches = statusChange.thresholdBreaches; + if (statusChange.thresholdBreaches) decision.thresholdBreaches = statusChange.thresholdBreaches; decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index cf4ae4293..bbb389573 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -864,6 +864,13 @@ describe("WorkerPipeline", () => { expect(decision).not.toHaveProperty("thresholdBreaches"); }); + // StatusService only sets thresholdBreaches for hardware monitors; a docker breach must not leave the key behind holding undefined. + it("leaves threshold breaches off a breached decision when the status change carries none", async () => { + const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" })); + expect(decision).toMatchObject({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + expect(decision).not.toHaveProperty("thresholdBreaches"); + }); + // Each decision starts from a shared template. A transition must not leak its flags into the next decision // the same pipeline makes, whether that is a no-op evaluation or a degraded-egress short-circuit. it("starts every decision clean after a transition on the same pipeline", async () => { From 4e4d3c3971f09db28348f51f2f52d7bd816bc24f Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 13:41:58 -0700 Subject: [PATCH 22/29] remove code and timestamp from stautschangeresult, no longer used --- server/src/service/statusService.ts | 6 +----- server/src/types/network.ts | 2 -- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 48b75f1da..92aabf65f 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -160,7 +160,7 @@ export class StatusService implements IStatusService { updateMonitorStatus = async (check: Check, monitor: Monitor): Promise => { try { - const { status, statusCode } = check; + const { status } = check; // Update running stats await this.tryUpdateRunningStats(check, monitor); @@ -204,8 +204,6 @@ export class StatusService implements IStatusService { monitor: updated, statusChanged, prevStatus, - code: statusCode, - timestamp: Date.now(), }; } @@ -279,8 +277,6 @@ export class StatusService implements IStatusService { monitor: updated, statusChanged, prevStatus, - code: statusCode, - timestamp: new Date().getTime(), thresholdBreaches, }; } catch (error: unknown) { diff --git a/server/src/types/network.ts b/server/src/types/network.ts index 3ca2f107f..a1b23c26c 100644 --- a/server/src/types/network.ts +++ b/server/src/types/network.ts @@ -182,8 +182,6 @@ export type StatusChangeResult = { monitor: Monitor; statusChanged: boolean; prevStatus: MonitorStatus; - code: number; - timestamp: number; thresholdBreaches?: HardwareBreaches; }; From 187a0fc3d49f4d128bb8ca675ba1c536c4065309 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 13:43:40 -0700 Subject: [PATCH 23/29] remove extra guard --- server/src/worker/reactors/reactor.notification.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/worker/reactors/reactor.notification.ts b/server/src/worker/reactors/reactor.notification.ts index 91b9b7cd4..be65475e0 100644 --- a/server/src/worker/reactors/reactor.notification.ts +++ b/server/src/worker/reactors/reactor.notification.ts @@ -7,7 +7,6 @@ export class NotificationReactor implements IMonitorReactor { constructor(private notificationService: INotificationsService) {} react = async (evaluation: MonitorEvaluation) => { - if (!evaluation.decision.shouldSendNotification) return; await this.notificationService.handleNotifications(evaluation.monitor, evaluation.check, evaluation.decision); }; } From 4bbfe2e889cca16dc9debd686ffae5345691b428 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 13:49:54 -0700 Subject: [PATCH 24/29] remove extra write on short window --- server/src/service/statusService.ts | 115 ++++++++++++---------------- 1 file changed, 49 insertions(+), 66 deletions(-) diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 92aabf65f..459701bfb 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -188,79 +188,62 @@ export class StatusService implements IStatusService { statusChanged = newStatus === "down"; } - // Not enough data points yet — record the check and return - if (projectedWindow.length < monitor.statusWindowSize) { - const updated = await this.monitorsRepository.updateStatusWindowAndChecks( - monitor.id, - monitor.teamId, - check.status, - checkSnapshot, - monitor.statusWindowSize, - MAX_RECENT_CHECKS, - patch - ); - - return { - monitor: updated, - statusChanged, - prevStatus, - }; - } - - // First evaluate reachability status changes, which apply to all monitor types - // and take precedence over hardware breaches. - const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold); - if (reachabilityResult.transitioned) { - newStatus = reachabilityResult.nextStatus; - statusChanged = true; - } - - // Evaluate hardware threshold breaches let thresholdBreaches: HardwareBreaches | undefined; - if (monitor.type === "hardware") { - const hardware = this.computeHardwareStatus({ - currentStatus: newStatus, - reachabilityDown: newStatus === "down", - metrics: check, - thresholds: { - cpu: monitor.cpuAlertThreshold, - memory: monitor.memoryAlertThreshold, - disk: monitor.diskAlertThreshold, - temp: monitor.tempAlertThreshold, - }, - counters: { - cpu: monitor.cpuAlertCounter, - memory: monitor.memoryAlertCounter, - disk: monitor.diskAlertCounter, - temp: monitor.tempAlertCounter, - }, - }); - patch.cpuAlertCounter = hardware.nextCounters.cpu; - patch.memoryAlertCounter = hardware.nextCounters.memory; - patch.diskAlertCounter = hardware.nextCounters.disk; - patch.tempAlertCounter = hardware.nextCounters.temp; - thresholdBreaches = hardware.breaches; - if (hardware.transitioned) { - newStatus = hardware.nextStatus; + if (projectedWindow.length >= monitor.statusWindowSize) { + // First evaluate reachability status changes, which apply to all monitor types + // and take precedence over hardware breaches. + const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold); + if (reachabilityResult.transitioned) { + newStatus = reachabilityResult.nextStatus; statusChanged = true; } - } - if (monitor.type === "docker" && check.containers) { - const breaches = findContainerBreaches(monitor, check.containers); - const docker = this.computeDockerStatus({ - currentStatus: newStatus, - reachabilityDown: newStatus === "down", - breaching: breaches.length > 0, - }); - if (docker.transitioned) { - newStatus = docker.nextStatus; - statusChanged = true; + // Evaluate hardware threshold breaches + if (monitor.type === "hardware") { + const hardware = this.computeHardwareStatus({ + currentStatus: newStatus, + reachabilityDown: newStatus === "down", + metrics: check, + thresholds: { + cpu: monitor.cpuAlertThreshold, + memory: monitor.memoryAlertThreshold, + disk: monitor.diskAlertThreshold, + temp: monitor.tempAlertThreshold, + }, + counters: { + cpu: monitor.cpuAlertCounter, + memory: monitor.memoryAlertCounter, + disk: monitor.diskAlertCounter, + temp: monitor.tempAlertCounter, + }, + }); + + patch.cpuAlertCounter = hardware.nextCounters.cpu; + patch.memoryAlertCounter = hardware.nextCounters.memory; + patch.diskAlertCounter = hardware.nextCounters.disk; + patch.tempAlertCounter = hardware.nextCounters.temp; + thresholdBreaches = hardware.breaches; + if (hardware.transitioned) { + newStatus = hardware.nextStatus; + statusChanged = true; + } } - } - patch.status = newStatus; + if (monitor.type === "docker" && check.containers) { + const breaches = findContainerBreaches(monitor, check.containers); + const docker = this.computeDockerStatus({ + currentStatus: newStatus, + reachabilityDown: newStatus === "down", + breaching: breaches.length > 0, + }); + if (docker.transitioned) { + newStatus = docker.nextStatus; + statusChanged = true; + } + } + patch.status = newStatus; + } // Single atomic write: push arrays + set status/counters const updated = await this.monitorsRepository.updateStatusWindowAndChecks( From cb47439e9f28a8cdc11998ce8187ec85b726a384 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 13:55:17 -0700 Subject: [PATCH 25/29] remove double logging --- server/src/service/statusService.ts | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 459701bfb..b51b9698b 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -21,7 +21,6 @@ const SERVICE_NAME = "StatusService"; const HARDWARE_ALERT_COUNTER_START = 5; export interface IStatusService { - updateRunningStats(check: Check, monitor: Monitor): Promise; updateMonitorStatus(check: Check, monitor: Monitor): Promise; } @@ -37,32 +36,19 @@ export class StatusService implements IStatusService { this.monitorStatsRepository = monitorStatsRepository; } - async updateRunningStats(check: Check, monitor: Monitor) { + private tryUpdateRunningStats = async (check: Check, monitor: Monitor) => { try { await this.monitorStatsRepository.updateByMonitorId(monitor.id, { status: check.status === true, responseTime: check.responseTime ?? 0, now: Date.now(), }); - return true; } catch (error: unknown) { - this.logger.error({ - service: SERVICE_NAME, - message: error instanceof Error ? error.message : "Unknown error", - method: "updateRunningStats", - stack: error instanceof Error ? error.stack : undefined, - }); - return false; - } - } - - private tryUpdateRunningStats = async (check: Check, monitor: Monitor) => { - const statsOk = await this.updateRunningStats(check, monitor); - if (!statsOk) { this.logger.warn({ service: SERVICE_NAME, - method: "updateMonitorStatus", - message: `Stats update failed for monitor ${monitor.id}`, + message: error instanceof Error ? error.message : "Unknown error", + method: "tryUpdateRunningStats", + stack: error instanceof Error ? error.stack : undefined, }); } }; From 02e6333c3c713cdac76c1105c14d1122b7dbb331 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 14:00:54 -0700 Subject: [PATCH 26/29] narrow notificaitonReason type --- .../src/domain/notifications/notification.message-builder.ts | 2 +- server/src/domain/notifications/notification.type.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index f53f9d609..55b2ae1ef 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -39,7 +39,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { clientHost, metadata: { teamId: monitor.teamId, - notificationReason: decision.notificationReason || "status_change", + notificationReason: decision.notificationReason, }, }; } diff --git a/server/src/domain/notifications/notification.type.ts b/server/src/domain/notifications/notification.type.ts index b696bf752..34899386d 100644 --- a/server/src/domain/notifications/notification.type.ts +++ b/server/src/domain/notifications/notification.type.ts @@ -1,4 +1,5 @@ import type { HardwareMetricKey } from "@/domain/monitors/monitor.type.js"; +import { MonitorActionDecision } from "@/worker/worker.interface.js"; export const NotificationChannels = [ "email", @@ -119,6 +120,6 @@ export interface NotificationMessage { clientHost: string; metadata: { teamId: string; - notificationReason: string; + notificationReason: MonitorActionDecision["notificationReason"]; }; } From 18f38818e2d48c77fc448e836c6a16d1b0225385 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 14:03:34 -0700 Subject: [PATCH 27/29] update tests --- server/test/helpers/heartbeatTestHarness.ts | 10 +- .../evaluateCursorMultiWorker.test.ts | 2 +- .../notificationMessageBuilder.test.ts | 9 -- .../test/unit/services/statusService.test.ts | 117 ++++++------------ .../test/unit/services/workerPipeline.test.ts | 12 +- 5 files changed, 55 insertions(+), 95 deletions(-) diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index f99be611c..d1166f238 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -10,6 +10,8 @@ import { InMemoryMonitorsRepository } from "./InMemoryMonitorsRepository.ts"; import { InMemoryIncidentsRepository } from "./InMemoryIncidentsRepository.ts"; import { createMockLogger } from "./createMockLogger.ts"; import type { Monitor } from "../../src/domain/monitors/monitor.type.ts"; +import type { Check } from "../../src/domain/checks/check.type.ts"; +import type { MonitorActionDecision } from "../../src/worker/worker.interface.ts"; import type { MonitorStatusResponse } from "../../src/types/network.ts"; import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts"; import type { EgressStatus } from "../../src/domain/egress/egress.type.ts"; @@ -117,7 +119,13 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { nextEgressStatus = status; }; - const notificationReactor = new NotificationReactor(notificationsService as any); + // The real NotificationsService ignores decisions with shouldSendNotification false; the stub is reached only for + // the ones it would act on, so tests can assert on handleNotifications calls as "notifications sent". + const guardedNotificationsService = { + handleNotifications: (monitor: Monitor, check: Check, decision: MonitorActionDecision) => + decision.shouldSendNotification ? notificationsService.handleNotifications(monitor, check, decision) : Promise.resolve(false), + }; + const notificationReactor = new NotificationReactor(guardedNotificationsService as any); const incidentReactor = new IncidentReactor(incidentService as any); const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); diff --git a/server/test/integration/evaluateCursorMultiWorker.test.ts b/server/test/integration/evaluateCursorMultiWorker.test.ts index a3fe75b04..5f40d2eef 100644 --- a/server/test/integration/evaluateCursorMultiWorker.test.ts +++ b/server/test/integration/evaluateCursorMultiWorker.test.ts @@ -133,7 +133,7 @@ describe("Evaluate cursor with more than one processing worker", () => { statusService: { updateMonitorStatus: async (check: Check, monitor: Monitor) => { evaluatedCheckIds.push(check.id); - return { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 }; + return { monitor, statusChanged: false, prevStatus: monitor.status }; }, } as any, dispatcher: { dispatch: jest.fn().mockResolvedValue(undefined) } as any, diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index 199d80101..472542e09 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -144,15 +144,6 @@ describe("NotificationMessageBuilder", () => { expect(msg.severity).toBe("success"); expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor"); }); - - it("uses notificationReason from decision, falling back to status_change", () => { - const monitor = makeMonitor({ status: "down" }); - const decision = makeDecision({ notificationReason: null }); - - const msg = builder.buildMessage(monitor, makeCheck(), decision, "https://app.example.com"); - - expect(msg.metadata.notificationReason).toBe("status_change"); - }); }); // ── determineNotificationType (via buildMessage) ───────────────────── diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 50005312d..031df1a46 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -131,92 +131,86 @@ const makeExistingStats = (overrides?: Record) => ({ // ── Tests ──────────────────────────────────────────────────────────────────── describe("StatusService", () => { - // ── updateRunningStats ─────────────────────────────────────────────────── + // ── running stats (via updateMonitorStatus) ────────────────────────────── - describe("updateRunningStats", () => { - // The service now delegates the entire running-stats computation to the repository's - // atomic `updateByMonitorId`, so the service-level tests only verify forwarding and error - // handling. The math (running average, uptimePercentage, timeOfLastFailure state machine) - // is a repository-level concern and is exercised by the repository tests. + describe("running stats", () => { + // The service delegates the running-stats computation to the repository's atomic `updateByMonitorId`, + // so these cases only verify what is forwarded and that a failure never blocks the status update. + // The math (running average, uptimePercentage, timeOfLastFailure) is exercised by the repository tests. + + const run = async (check: Check) => { + const monitor = makeMonitor(); + const deps = createService(); + (deps.monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (deps.monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); + const result = await deps.service.updateMonitorStatus(check, monitor); + return { ...deps, result }; + }; it("forwards a successful check to updateByMonitorId", async () => { - const { service, monitorStatsRepository } = createService(); - (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); + const { monitorStatsRepository } = await run(makeCheck({ responseTime: 50 })); - const result = await service.updateRunningStats(makeCheck({ responseTime: 50 }), makeMonitor()); - - expect(result).toBe(true); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith( "mon-1", - expect.objectContaining({ - status: true, - responseTime: 50, - now: expect.any(Number), - }) + expect.objectContaining({ status: true, responseTime: 50, now: expect.any(Number) }) ); }); it("forwards a failed check to updateByMonitorId", async () => { - const { service, monitorStatsRepository } = createService(); - (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); + const { monitorStatsRepository } = await run(makeCheck({ status: false, responseTime: 100 })); - const result = await service.updateRunningStats(makeCheck({ status: false, responseTime: 100 }), makeMonitor()); - - expect(result).toBe(true); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false, responseTime: 100 })); }); it("passes 0 when responseTime is undefined", async () => { - const { service, monitorStatsRepository } = createService(); - (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - - await service.updateRunningStats(makeCheck({ responseTime: undefined as any }), makeMonitor()); + const { monitorStatsRepository } = await run(makeCheck({ responseTime: undefined as any })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 })); }); it("forwards responseTime of 0 as 0 (falsy but defined)", async () => { - const { service, monitorStatsRepository } = createService(); - (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - - await service.updateRunningStats(makeCheck({ responseTime: 0 }), makeMonitor()); + const { monitorStatsRepository } = await run(makeCheck({ responseTime: 0 })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 })); }); it("coerces a non-true status to false (e.g. undefined)", async () => { - const { service, monitorStatsRepository } = createService(); - (monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); - - await service.updateRunningStats(makeCheck({ status: undefined as any }), makeMonitor()); + const { monitorStatsRepository } = await run(makeCheck({ status: undefined as any })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false })); }); - it("returns false and logs error when updateByMonitorId throws", async () => { - const { service, logger, monitorStatsRepository } = createService(); - (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("db write failed")); + it("warns once and still updates the status when updateByMonitorId throws", async () => { + const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" }); + const { service, logger, monitorsRepository, monitorStatsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down")); - const result = await service.updateRunningStats(makeCheck(), makeMonitor()); + const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); - expect(result).toBe(false); - expect(logger.error).toHaveBeenCalledWith( + expect(result.monitor.status).toBe("up"); + expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( expect.objectContaining({ service: "StatusService", - method: "updateRunningStats", - message: "db write failed", + method: "tryUpdateRunningStats", + message: "stats db down", + stack: expect.any(String), }) ); + expect(logger.error).not.toHaveBeenCalled(); }); - it("logs error with 'Unknown error' for non-Error exceptions", async () => { - const { service, logger, monitorStatsRepository } = createService(); + it("warns with 'Unknown error' for non-Error rejections", async () => { + const monitor = makeMonitor(); + const { service, logger, monitorsRepository, monitorStatsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue("string error"); - const result = await service.updateRunningStats(makeCheck(), makeMonitor()); + await service.updateMonitorStatus(makeCheck(), monitor); - expect(result).toBe(false); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined })); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined })); }); }); @@ -640,37 +634,6 @@ describe("StatusService", () => { expect(result.monitor.status).toBe("down"); }); - it("logs a warning but still succeeds when running-stats update fails mid-flight", async () => { - const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" }); - const { service, logger, monitorsRepository, monitorStatsRepository } = createService(); - (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down")); - - const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); - - expect(result.monitor.status).toBe("up"); - expect(logger.warn).toHaveBeenCalledWith( - expect.objectContaining({ - service: "StatusService", - method: "updateMonitorStatus", - message: expect.stringContaining("Stats update failed"), - }) - ); - }); - - it("returns code and timestamp in result", async () => { - const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5 }); - const { service, monitorsRepository } = createService(); - (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - - const result = await service.updateMonitorStatus(makeCheck({ statusCode: 201 }), monitor); - - expect(result.code).toBe(201); - expect(result.timestamp).toBeGreaterThan(0); - }); - // ── Hardware threshold breach tests ────────────────────────────────── describe("hardware threshold breaches", () => { diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index bbb389573..fbaf2eb73 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -64,8 +64,6 @@ const makeStatusChange = (overrides: Partial & { status?: st monitor: { id: "m1", status } as any, statusChanged: false, prevStatus: "up", - code: 200, - timestamp: 0, ...rest, } as StatusChangeResult; }; @@ -706,7 +704,7 @@ describe("WorkerPipeline", () => { it("returns an evaluation that opens an incident on a down transition", async () => { const check = makeCheck({ status: false, statusCode: 500, message: "Error" }); - const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 }); + const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up" }); const { pipeline } = createPipeline({ statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, }); @@ -802,7 +800,7 @@ describe("WorkerPipeline", () => { }); it("creates incident and notifies when monitor goes down", async () => { - const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 })); + const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up" })); expect(decision).toMatchObject({ shouldCreateIncident: true, shouldSendNotification: true, @@ -854,7 +852,7 @@ describe("WorkerPipeline", () => { it("leaves threshold breaches off a down decision even when the status change carries them", async () => { const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false }; - const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500, thresholdBreaches })); + const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", thresholdBreaches })); expect(decision).not.toHaveProperty("thresholdBreaches"); }); @@ -876,8 +874,8 @@ describe("WorkerPipeline", () => { it("starts every decision clean after a transition on the same pipeline", async () => { const updateMonitorStatus = jest .fn() - .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 })) - .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "down", code: 500 })); + .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up" })) + .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "down" })); const { pipeline } = createPipeline({ statusService: { updateMonitorStatus } }); const failing = makeCheck({ status: false, statusCode: 500, message: "Error" }); From 544502c6e56720d37f856d5a567f48dfc6fdb936 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 14:19:13 -0700 Subject: [PATCH 28/29] move formatting --- .../src/domain/incidents/incident.service.ts | 21 ++------------ .../notification.message-builder.ts | 28 +++++++++++++++---- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index 4adbf9b75..84ef104bd 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -1,4 +1,3 @@ -const SERVICE_NAME = "incidentService"; import type { Monitor } from "@/domain/monitors/monitor.type.js"; import type { Check } from "@/domain/checks/check.type.js"; import { AppError } from "@/utils/AppError.js"; @@ -12,8 +11,8 @@ import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; import type { ILogger } from "@/utils/logger.js"; import { DateRange } from "@/types/query.js"; -import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js"; +const SERVICE_NAME = "incidentService"; export interface IIncidentService { handleIncident(monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise; resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise; @@ -71,7 +70,7 @@ export class IncidentService implements IIncidentService { // For threshold breaches, use 9999 status code and build descriptive message if (decision.incidentReason === "threshold_breach") { statusCode = 9999; - message = this.buildThresholdBreachMessage(monitor, check, decision); + message = this.notificationMessageBuilder.buildThresholdBreachMessage(monitor, check, decision.thresholdBreaches); } const incident = { @@ -96,22 +95,6 @@ export class IncidentService implements IIncidentService { return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); }; - private buildThresholdBreachMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision): string { - if (monitor.type === "docker") { - const containers = check.containers ?? []; - const breaches = findContainerBreaches(monitor, containers); - return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert"; - } - - const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check, decision.thresholdBreaches); - - if (breaches.length === 0) { - return "Threshold breach detected"; - } - - return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", "); - } - resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => { try { if (!incidentId) { diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 55b2ae1ef..605868ddb 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -12,7 +12,7 @@ import type { Check } from "@/domain/checks/check.type.js"; export interface INotificationMessageBuilder { buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage; - extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[]; + buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string; } const SERVICE_NAME = "NotificationMessageBuilder"; @@ -167,11 +167,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } private buildContainerBreachContent(monitor: Monitor, check: Check): NotificationContent { - const containers = check.containers ?? []; - const breaches = findContainerBreaches(monitor, containers); + const lines = this.describeContainerBreaches(monitor, check); const title = `Container Alert: ${monitor.name}`; - const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`; - const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)]; + const summary = `${lines.length} container(s) on "${monitor.name}" need attention.`; + const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...lines]; return { title, summary, details, timestamp: new Date() }; } @@ -212,7 +211,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - public extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] { + private extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] { const breaches: ThresholdBreach[] = []; // Check if this is a hardware monitor with threshold data @@ -227,4 +226,21 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return breaches; } + + private describeContainerBreaches(monitor: Monitor, check: Check): string[] { + return findContainerBreaches(monitor, check.containers ?? []).map(describeContainerBreach); + } + + public buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string { + if (monitor.type === "docker") { + const lines = this.describeContainerBreaches(monitor, check); + return lines.length > 0 ? lines.join(", ") : "Container alert"; + } + + const breaches = this.extractThresholdBreaches(monitor, check, thresholdBreaches); + if (breaches.length === 0) { + return "Threshold breach detected"; + } + return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", "); + } } From 5d77b8804f267f0e6590854d9bd24dc261f203c4 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 24 Sep 2026 14:22:31 -0700 Subject: [PATCH 29/29] update tests --- server/test/helpers/heartbeatTestHarness.ts | 4 +- .../heartbeatJobThresholdBreach.test.ts | 4 +- .../integration/incidentLifecycle.test.ts | 7 +- .../unit/services/incidentService.test.ts | 67 ++----------------- .../notificationMessageBuilder.test.ts | 47 +++++++++++++ .../services/notificationsService.test.ts | 2 +- 6 files changed, 59 insertions(+), 72 deletions(-) diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index d1166f238..c0f7f1529 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -63,7 +63,7 @@ export interface HeartbeatTestHarness { networkService: { requestStatus: jest.Mock }; bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock }; maintenanceWindowsRepo: { findByMonitorId: jest.Mock }; - messageBuilder: { extractThresholdBreaches: jest.Mock }; + messageBuilder: { buildThresholdBreachMessage: jest.Mock }; egressService: { assessAfterFailure: jest.Mock }; heartbeatJob: (monitor: Monitor) => Promise; setNextResponse: (status: boolean, code: number) => void; @@ -79,7 +79,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { const statusService = new StatusService(logger, monitorsRepo as any, createStubMonitorStatsRepo() as any); - const messageBuilder = { extractThresholdBreaches: jest.fn().mockReturnValue([]) }; + const messageBuilder = { buildThresholdBreachMessage: jest.fn().mockReturnValue("") }; const incidentService = new IncidentService(logger, incidentsRepo, monitorsRepo as any, { findById: jest.fn() } as any, messageBuilder as any); const notificationsService = { handleNotifications: jest.fn().mockResolvedValue(true) }; diff --git a/server/test/integration/heartbeatJobThresholdBreach.test.ts b/server/test/integration/heartbeatJobThresholdBreach.test.ts index ccb4065fa..2615c030b 100644 --- a/server/test/integration/heartbeatJobThresholdBreach.test.ts +++ b/server/test/integration/heartbeatJobThresholdBreach.test.ts @@ -49,7 +49,7 @@ describe("Heartbeat job: hardware threshold breach", () => { h.setNextResponseFull(makeHardwareResponse(0.9)); // Configure message builder for when incident is created - h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]); + h.messageBuilder.buildThresholdBreachMessage.mockReturnValue("CPU: 90% (threshold: 80%)"); // 4 heartbeats: counter goes 5→4→3→2→1, status still "up" for (let i = 0; i < 4; i++) { @@ -191,7 +191,7 @@ describe("Heartbeat job: hardware threshold breach", () => { h.monitorsRepo.seed(monitor); h.setNextResponseFull(makeHardwareResponse(0.9)); - h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]); + h.messageBuilder.buildThresholdBreachMessage.mockReturnValue("CPU: 90% (threshold: 80%)"); // 5 heartbeats to trigger breach, then 5 more while still breaching for (let i = 0; i < 10; i++) { diff --git a/server/test/integration/incidentLifecycle.test.ts b/server/test/integration/incidentLifecycle.test.ts index ff2ff2c53..64e723183 100644 --- a/server/test/integration/incidentLifecycle.test.ts +++ b/server/test/integration/incidentLifecycle.test.ts @@ -55,7 +55,7 @@ describe("Incident lifecycle (integration)", () => { repo = new InMemoryIncidentsRepository(); monitorsRepo = { findById: jest.fn() } as unknown as jest.Mocked; usersRepo = { findById: jest.fn() } as unknown as jest.Mocked; - messageBuilder = { extractThresholdBreaches: jest.fn() } as unknown as jest.Mocked; + messageBuilder = { buildThresholdBreachMessage: jest.fn() } as unknown as jest.Mocked; service = new IncidentService(createMockLogger() as any, repo, monitorsRepo, usersRepo, messageBuilder); }); @@ -152,10 +152,7 @@ describe("Incident lifecycle (integration)", () => { it("creates a threshold breach incident with statusCode 9999 and descriptive message", async () => { const monitor = makeMonitor({ status: "breached", type: "hardware" }); - (messageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([ - { metric: "cpu", formattedValue: "92%", threshold: 80, unit: "%" }, - { metric: "memory", formattedValue: "88%", threshold: 85, unit: "%" }, - ]); + (messageBuilder.buildThresholdBreachMessage as jest.Mock).mockReturnValue("CPU: 92% (threshold: 80%), MEMORY: 88% (threshold: 85%)"); const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); const incident = await service.handleIncident(monitor, decision, makeCheck(200)); diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index fa1edea07..8639542c4 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -36,7 +36,7 @@ const createUsersRepo = () => const createMessageBuilder = () => ({ - extractThresholdBreaches: jest.fn(), + buildThresholdBreachMessage: jest.fn(), }) as unknown as jest.Mocked; const createService = (overrides?: { @@ -131,7 +131,7 @@ describe("IncidentService", () => { it("creates a new incident when shouldCreateIncident and no active incident exists", async () => { const created = makeIncident(); - const { service, incidentsRepository } = createService(); + const { service, incidentsRepository, notificationMessageBuilder } = createService(); (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); (incidentsRepository.create as jest.Mock).mockResolvedValue(created); @@ -150,6 +150,7 @@ describe("IncidentService", () => { statusCode: 500, }) ); + expect(notificationMessageBuilder.buildThresholdBreachMessage).not.toHaveBeenCalled(); }); it("uses status code 9999 and builds message for threshold_breach incidents", async () => { @@ -157,9 +158,7 @@ describe("IncidentService", () => { const { service, incidentsRepository, notificationMessageBuilder } = createService(); (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); (incidentsRepository.create as jest.Mock).mockResolvedValue(created); - (notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([ - { metric: "cpu", formattedValue: "95%", threshold: 80, unit: "%" }, - ]); + (notificationMessageBuilder.buildThresholdBreachMessage as jest.Mock).mockReturnValue("CPU: 95% (threshold: 80%)"); const monitor = makeMonitor({ type: "hardware" }); const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } }); @@ -167,7 +166,7 @@ describe("IncidentService", () => { const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach", thresholdBreaches }); await service.handleIncident(monitor, decision, check); - expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check, thresholdBreaches); + expect(notificationMessageBuilder.buildThresholdBreachMessage).toHaveBeenCalledWith(monitor, check, thresholdBreaches); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ statusCode: 9999, @@ -176,62 +175,6 @@ describe("IncidentService", () => { ); }); - it("uses fallback message when extractThresholdBreaches returns empty array", async () => { - const created = makeIncident(); - const { service, incidentsRepository, notificationMessageBuilder } = createService(); - (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); - (incidentsRepository.create as jest.Mock).mockResolvedValue(created); - (notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([]); - - const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), decision, makeCheck()); - - expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" })); - }); - - it("builds the container list for docker threshold_breach incidents", async () => { - const created = makeIncident(); - const { service, incidentsRepository, notificationMessageBuilder } = createService(); - (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); - (incidentsRepository.create as jest.Mock).mockResolvedValue(created); - - const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }); - const check = makeCheck({ - metadata: { monitorId: "mon-1", teamId: "team-1", type: "docker" }, - containers: [ - { id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 }, - { id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" }, - { id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" }, - ], - containerSummary: { total: 3, running: 2, stopped: 1, unhealthy: 1 }, - }); - const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(monitor, decision, check); - - expect(incidentsRepository.create).toHaveBeenCalledWith( - expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" }) - ); - expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled(); - }); - - it("joins multiple threshold breaches with commas", async () => { - const created = makeIncident(); - const { service, incidentsRepository, notificationMessageBuilder } = createService(); - (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); - (incidentsRepository.create as jest.Mock).mockResolvedValue(created); - (notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([ - { metric: "cpu", formattedValue: "95%", threshold: 80, unit: "%" }, - { metric: "memory", formattedValue: "90%", threshold: 80, unit: "%" }, - ]); - - const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); - await service.handleIncident(makeMonitor(), decision, makeCheck()); - - expect(incidentsRepository.create).toHaveBeenCalledWith( - expect.objectContaining({ message: "CPU: 95% (threshold: 80%), MEMORY: 90% (threshold: 80%)" }) - ); - }); - it("resolves active incident when shouldResolveIncident", async () => { const active = makeIncident(); const resolved = makeIncident({ status: false, endTime: "123", resolutionType: "automatic" }); diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index 472542e09..7560b1168 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -627,4 +627,51 @@ describe("NotificationMessageBuilder", () => { }); }); }); + + // ── buildThresholdBreachMessage ───────────────────────────────────────── + + describe("buildThresholdBreachMessage", () => { + const none: HardwareBreaches = { cpu: false, memory: false, disk: false, temp: false }; + const hardwareMonitor = () => makeMonitor({ type: "hardware", cpuAlertThreshold: 80, memoryAlertThreshold: 80 }); + + it("describes a single flagged metric", () => { + const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] } }); + + const message = builder.buildThresholdBreachMessage(hardwareMonitor(), check, { ...none, cpu: true }); + + expect(message).toBe("CPU: 95.0% (threshold: 80%)"); + }); + + it("joins several flagged metrics with a comma", () => { + const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] }, memory: { usage_percent: 0.9 } }); + + const message = builder.buildThresholdBreachMessage(hardwareMonitor(), check, { ...none, cpu: true, memory: true }); + + expect(message).toBe("CPU: 95.0% (threshold: 80%), MEMORY: 90.0% (threshold: 80%)"); + }); + + it("falls back when no metric is flagged", () => { + const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] } }); + + expect(builder.buildThresholdBreachMessage(hardwareMonitor(), check, none)).toBe("Threshold breach detected"); + expect(builder.buildThresholdBreachMessage(hardwareMonitor(), check, undefined)).toBe("Threshold breach detected"); + }); + + it("lists the breaching containers for a docker monitor", () => { + const check = makeCheck({ + status: true, + containers: makeContainers([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]), + }); + + const message = builder.buildThresholdBreachMessage(makeDockerMonitor(), check, undefined); + + expect(message).toBe("db: stopped (exit code 1), api: unhealthy"); + }); + + it("falls back when no container is breaching", () => { + const check = makeCheck({ status: true, containers: makeContainers([{ name: "web" }]) }); + + expect(builder.buildThresholdBreachMessage(makeDockerMonitor(), check, undefined)).toBe("Container alert"); + }); + }); }); diff --git a/server/test/unit/services/notificationsService.test.ts b/server/test/unit/services/notificationsService.test.ts index 7abba9d7f..c621fe0e4 100644 --- a/server/test/unit/services/notificationsService.test.ts +++ b/server/test/unit/services/notificationsService.test.ts @@ -33,7 +33,7 @@ const createSettingsService = (clientHost = "https://app.example.com") => ({ const createMessageBuilder = () => ({ buildMessage: jest.fn().mockReturnValue({ type: "monitor_down", content: { title: "Down" } }), - extractThresholdBreaches: jest.fn(), + buildThresholdBreachMessage: jest.fn(), }); const createService = (overrides?: Record) => {