diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index c4208aafb..4adbf9b75 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -71,7 +71,7 @@ export class IncidentService implements IIncidentService { // For threshold breaches, use 9999 status code and build descriptive message if (decision.incidentReason === "threshold_breach") { statusCode = 9999; - message = this.buildThresholdBreachMessage(monitor, check); + message = this.buildThresholdBreachMessage(monitor, check, decision); } const incident = { @@ -96,14 +96,14 @@ export class IncidentService implements IIncidentService { return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); }; - private buildThresholdBreachMessage(monitor: Monitor, check: Check): string { + private buildThresholdBreachMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision): string { if (monitor.type === "docker") { const containers = check.containers ?? []; const breaches = findContainerBreaches(monitor, containers); return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert"; } - const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check); + const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check, decision.thresholdBreaches); if (breaches.length === 0) { return "Threshold breach detected"; diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index b033c560a..07b0ef2e4 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -240,3 +240,8 @@ export type GamesMap = Record; export type MonitorScheduleFields = Pick; export type DockerContainerLogsResult = DockerLogPage; + +export const HardwareMetricKeys = ["cpu", "memory", "disk", "temp"] as const; +export type HardwareMetricKey = (typeof HardwareMetricKeys)[number]; +export type HardwareBreaches = Record; +export type HardwareCounters = Record; diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 11d11c069..f53f9d609 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -1,4 +1,4 @@ -import type { Monitor } from "@/domain/monitors/monitor.type.js"; +import { HardwareMetricKeys, type HardwareBreaches, type HardwareMetricKey, type Monitor } from "@/domain/monitors/monitor.type.js"; import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { NotificationMessage, @@ -12,7 +12,7 @@ import type { Check } from "@/domain/checks/check.type.js"; export interface INotificationMessageBuilder { buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage; - extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[]; + extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[]; } const SERVICE_NAME = "NotificationMessageBuilder"; @@ -23,7 +23,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage { const type = this.determineNotificationType(decision, monitor); const severity = this.determineSeverity(type); - const content = this.buildContent(type, monitor, check); + const content = this.buildContent(type, monitor, check, decision); return { type, @@ -82,14 +82,14 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { } } - private buildContent(type: NotificationType, monitor: Monitor, check: Check): NotificationContent { + private buildContent(type: NotificationType, monitor: Monitor, check: Check, decision: MonitorActionDecision): NotificationContent { switch (type) { case "monitor_down": return this.buildMonitorDownContent(monitor, check); case "monitor_up": return this.buildMonitorUpContent(monitor); case "threshold_breach": - return this.buildThresholdBreachContent(monitor, check); + return this.buildThresholdBreachContent(monitor, check, decision.thresholdBreaches); case "threshold_resolved": return this.buildThresholdResolvedContent(monitor); case "container_breach": @@ -137,12 +137,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private buildThresholdBreachContent(monitor: Monitor, check: Check): NotificationContent { + private buildThresholdBreachContent(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): NotificationContent { const title = `Threshold Exceeded: ${monitor.name}`; const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`; const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`]; - const thresholds = this.extractThresholdBreaches(monitor, check); + const thresholds = this.extractThresholdBreaches(monitor, check, thresholdBreaches); return { title, @@ -191,83 +191,38 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - public extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[] { + private describeBreach(metric: HardwareMetricKey, monitor: Monitor, check: Check): ThresholdBreach { + switch (metric) { + case "cpu": { + const currentValue = (check.cpu?.usage_percent ?? 0) * 100; + return { metric, currentValue, threshold: monitor.cpuAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` }; + } + case "memory": { + const currentValue = (check.memory?.usage_percent ?? 0) * 100; + return { metric, currentValue, threshold: monitor.memoryAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` }; + } + case "disk": { + const currentValue = Math.max(0, ...(check.disk ?? []).map((disk) => disk?.usage_percent ?? 0)) * 100; + return { metric, currentValue, threshold: monitor.diskAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` }; + } + case "temp": { + const currentValue = Math.max(0, ...(check.cpu?.temperature ?? [])); + return { metric, currentValue, threshold: monitor.tempAlertThreshold, unit: "°C", formattedValue: `${currentValue.toFixed(1)}°C` }; + } + } + } + + public extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] { const breaches: ThresholdBreach[] = []; // Check if this is a hardware monitor with threshold data - if (monitor.type !== "hardware") { + if (monitor.type !== "hardware" || thresholdBreaches === undefined) { return breaches; } - // Note: usage_percent values in hardware payload are decimals (0-1) - if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && check.cpu?.usage_percent !== undefined) { - const cpuUsageDecimal = check.cpu.usage_percent; - const cpuPercent = cpuUsageDecimal * 100; - const threshold = monitor.cpuAlertThreshold; - if (cpuPercent > threshold) { - breaches.push({ - metric: "cpu", - currentValue: cpuPercent, - threshold, - unit: "%", - formattedValue: `${cpuPercent.toFixed(1)}%`, - }); - } - } - - // Memory threshold breach - if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && check.memory?.usage_percent !== undefined) { - const memoryUsageDecimal = check.memory.usage_percent; - const memoryPercent = memoryUsageDecimal * 100; - const threshold = monitor.memoryAlertThreshold; - if (memoryPercent > threshold) { - breaches.push({ - metric: "memory", - currentValue: memoryPercent, - threshold, - unit: "%", - formattedValue: `${memoryPercent.toFixed(1)}%`, - }); - } - } - - // Disk threshold breach - if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(check.disk)) { - // Find the highest disk usage - let maxDiskUsageDecimal = 0; - for (const disk of check.disk) { - if (disk.usage_percent !== undefined && disk.usage_percent > maxDiskUsageDecimal) { - maxDiskUsageDecimal = disk.usage_percent; - } - } - const maxDiskPercent = maxDiskUsageDecimal * 100; - const threshold = monitor.diskAlertThreshold; - if (maxDiskPercent > threshold) { - breaches.push({ - metric: "disk", - currentValue: maxDiskPercent, - threshold, - unit: "%", - formattedValue: `${maxDiskPercent.toFixed(1)}%`, - }); - } - } - - // Temperature threshold breach - if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && check.cpu?.temperature) { - // Temperature is an array in cpu.temperature - const temps = Array.isArray(check.cpu.temperature) ? check.cpu.temperature : [check.cpu.temperature]; - const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t))); - const threshold = monitor.tempAlertThreshold; - if (maxTemp > threshold) { - breaches.push({ - metric: "temp", - currentValue: maxTemp, - threshold, - unit: "°C", - formattedValue: `${maxTemp.toFixed(1)}°C`, - }); - } + for (const metric of HardwareMetricKeys) { + if (!thresholdBreaches[metric]) continue; + breaches.push(this.describeBreach(metric, monitor, check)); } return breaches; diff --git a/server/src/domain/notifications/notification.type.ts b/server/src/domain/notifications/notification.type.ts index 7f2c7df1e..b696bf752 100644 --- a/server/src/domain/notifications/notification.type.ts +++ b/server/src/domain/notifications/notification.type.ts @@ -1,3 +1,5 @@ +import type { HardwareMetricKey } from "@/domain/monitors/monitor.type.js"; + export const NotificationChannels = [ "email", "slack", @@ -85,7 +87,7 @@ export interface MonitorInfo { } export interface ThresholdBreach { - metric: "cpu" | "memory" | "disk" | "temp"; + metric: HardwareMetricKey; currentValue: number; threshold: number; unit: string; diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 64a6f2e61..48b75f1da 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -1,7 +1,14 @@ import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js"; -import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js"; +import { + HardwareMetricKeys, + MonitorStatuses, + type HardwareBreaches, + type HardwareCounters, + type Monitor, + type MonitorStatus, +} from "@/domain/monitors/monitor.type.js"; import type { StatusChangeResult } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { ILogger } from "@/utils/logger.js"; @@ -12,10 +19,6 @@ import { findContainerBreaches } from "@/domain/docker/docker-alert.js"; const SERVICE_NAME = "StatusService"; const HARDWARE_ALERT_COUNTER_START = 5; -const HARDWARE_METRIC_KEYS = ["cpu", "memory", "disk", "temp"] as const; -type HardwareMetricKey = (typeof HARDWARE_METRIC_KEYS)[number]; -type HardwareBreaches = Record; -type HardwareCounters = Record; export interface IStatusService { updateRunningStats(check: Check, monitor: Monitor): Promise; @@ -111,7 +114,7 @@ export class StatusService implements IStatusService { // Update counters: decrement (floored at 0) if breached, reset to start otherwise. const nextCounters = { ...counters }; - for (const key of HARDWARE_METRIC_KEYS) { + for (const key of HardwareMetricKeys) { nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START; } @@ -123,8 +126,8 @@ export class StatusService implements IStatusService { if (!reachabilityDown) { // A counter can only reach zero via the decrement path, which only runs when that // metric is currently breaching — so anyCounterZero already implies anyBreached. - const anyCounterZero = HARDWARE_METRIC_KEYS.some((k) => nextCounters[k] === 0); - const allNormal = HARDWARE_METRIC_KEYS.every((k) => !breaches[k]); + const anyCounterZero = HardwareMetricKeys.some((k) => nextCounters[k] === 0); + const allNormal = HardwareMetricKeys.every((k) => !breaches[k]); if (anyCounterZero && currentStatus !== "breached") { nextStatus = "breached"; diff --git a/server/src/types/network.ts b/server/src/types/network.ts index a2cf78dbe..3ca2f107f 100644 --- a/server/src/types/network.ts +++ b/server/src/types/network.ts @@ -10,7 +10,7 @@ import type { ILighthouseAudit, } from "@/domain/checks/check.type.js"; import { DockerContainerInfo, DockerContainerLogs, DockerContainerSummary } from "@/domain/docker/docker.type.js"; -import type { DnsRecordType, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js"; +import type { DnsRecordType, HardwareBreaches, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js"; import type { QueryResult } from "gamedig"; @@ -184,12 +184,7 @@ export type StatusChangeResult = { prevStatus: MonitorStatus; code: number; timestamp: number; - thresholdBreaches?: { - cpu: boolean; - memory: boolean; - disk: boolean; - temp: boolean; - }; + thresholdBreaches?: HardwareBreaches; }; export type MonitorStatusResponseOverrides = Partial, "monitorId" | "teamId" | "type">>; diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 4b5bba2c5..6e9ec3d49 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,7 +1,6 @@ -import { Monitor } from "@/domain/monitors/monitor.type.js"; +import { HardwareBreaches, Monitor } from "@/domain/monitors/monitor.type.js"; import { Check } from "@/domain/checks/check.type.js"; import { Job, JobType } from "@/domain/jobs/job.type.js"; -import { StatusChangeResult } from "@/types/network.js"; import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; @@ -13,18 +12,12 @@ export interface MonitorActionDecision { shouldSendNotification: boolean; incidentReason: "status_down" | "threshold_breach" | null; notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null; - thresholdBreaches?: { - cpu?: boolean; - memory?: boolean; - disk?: boolean; - temp?: boolean; - }; + thresholdBreaches?: HardwareBreaches; } export type MonitorEvaluation = { monitor: Monitor; check: Check; - statusChange: StatusChangeResult; // from statusService.updateMonitorStatus decision: MonitorActionDecision; }; diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts index 1faef2a11..3d703d3eb 100644 --- a/server/src/worker/worker.pipeline.ts +++ b/server/src/worker/worker.pipeline.ts @@ -3,7 +3,6 @@ import { ICheckService } from "@/domain/checks/check.service.js"; import { Check } from "@/domain/checks/check.type.js"; import { IDockerLogsService } from "@/domain/docker/docker-log.service.js"; import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js"; -import type { EgressStatus } from "@/domain/egress/egress.type.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { PendingCheck } from "@/domain/jobs/job.type.js"; import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; @@ -21,7 +20,13 @@ import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js"; const SERVICE_NAME = "WorkerPipeline"; - +const NO_ACTION: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, +}; // The three entry points: the queue calls the two handlers, the buffer calls ingestChecks. // produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer. export interface IWorkerPipeline { @@ -128,7 +133,7 @@ export class WorkerPipeline implements IWorkerPipeline { // Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the // target. Null means the egress check is disabled (or failed internally) and the check is treated as usual. - const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null; + const egressStatus = this.isTransportFailure(monitor, status) ? await this.deps.egressService.assessAfterFailure() : null; if (egressStatus !== null) { check.egressStatus = egressStatus; } @@ -171,20 +176,6 @@ export class WorkerPipeline implements IWorkerPipeline { return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status); } - // The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown. - private async assessEgress(monitorId: string): Promise { - try { - return await this.deps.egressService.assessAfterFailure(); - } catch (error: unknown) { - this.deps.logger.warn({ - message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "assessEgress", - }); - return null; - } - } - // **************************************************************** // Stage 2: Ingestion // Called by the buffer, checks are inserted and evaluation jobs created @@ -252,7 +243,7 @@ export class WorkerPipeline implements IWorkerPipeline { }); return; } - current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check + current = evaluation.monitor; // fresh statusWindow/status/counters for the next check } // Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever @@ -277,18 +268,11 @@ export class WorkerPipeline implements IWorkerPipeline { service: SERVICE_NAME, method: "evaluateCheck", }); - const decision: MonitorActionDecision = { - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }; + return { monitor, check, - statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: check.statusCode, timestamp: Date.now() }, - decision, + decision: { ...NO_ACTION }, }; }; @@ -300,18 +284,12 @@ export class WorkerPipeline implements IWorkerPipeline { const statusChange = await this.deps.statusService.updateMonitorStatus(check, monitor); const decision = this.decide(statusChange); - return { monitor: statusChange.monitor, check, statusChange, decision }; + return { monitor: statusChange.monitor, check, decision }; }; private decide = (statusChange: StatusChangeResult): MonitorActionDecision => { const { monitor, statusChanged, prevStatus } = statusChange; - const decision: MonitorActionDecision = { - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }; + const decision: MonitorActionDecision = { ...NO_ACTION }; if (!statusChanged) return decision; if (monitor.status === "down") { @@ -320,6 +298,7 @@ export class WorkerPipeline implements IWorkerPipeline { decision.incidentReason = "status_down"; decision.notificationReason = "status_change"; } else if (monitor.status === "breached") { + if (statusChange.thresholdBreaches) decision.thresholdBreaches = statusChange.thresholdBreaches; decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index 44d0a23cf..fa1edea07 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -163,10 +163,11 @@ describe("IncidentService", () => { const monitor = makeMonitor({ type: "hardware" }); const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } }); - const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false }; + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach", thresholdBreaches }); await service.handleIncident(monitor, decision, check); - expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check); + expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check, thresholdBreaches); expect(incidentsRepository.create).toHaveBeenCalledWith( expect.objectContaining({ statusCode: 9999, diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index 3937e3bcf..199d80101 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, beforeEach } from "@jest/globals"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; +import type { HardwareBreaches, Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; @@ -119,7 +119,10 @@ describe("NotificationMessageBuilder", () => { it("builds a threshold_breach message", () => { const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); - const decision = makeDecision({ notificationReason: "threshold_breach" }); + const decision = makeDecision({ + notificationReason: "threshold_breach", + thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false }, + }); const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); const msg = builder.buildMessage(monitor, check, decision, "https://app.example.com"); @@ -346,21 +349,26 @@ describe("NotificationMessageBuilder", () => { }); describe("threshold_breach", () => { - it("includes threshold breaches in content", () => { + it("includes the breaches flagged on the decision in content", () => { + const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); + const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); + const decision = makeDecision({ + notificationReason: "threshold_breach", + thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false }, + }); + + const msg = builder.buildMessage(monitor, check, decision, ""); + + expect(msg.content.thresholds).toEqual([expect.objectContaining({ metric: "cpu", currentValue: 90, threshold: 80 })]); + }); + + it("leaves thresholds empty when the decision carries no breaches", () => { const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 }); const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); const msg = builder.buildMessage(monitor, check, makeDecision({ notificationReason: "threshold_breach" }), ""); - expect(msg.content.thresholds).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - metric: "cpu", - currentValue: 90, - threshold: 80, - }), - ]) - ); + expect(msg.content.thresholds).toEqual([]); }); }); @@ -480,346 +488,151 @@ describe("NotificationMessageBuilder", () => { // ── extractThresholdBreaches ───────────────────────────────────────── describe("extractThresholdBreaches", () => { - it("returns empty array for non-hardware monitor", () => { - const monitor = makeMonitor({ type: "http" }); - const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches).toEqual([]); - }); - - it("returns empty array when the check carries no hardware data", () => { - const monitor = makeMonitor({ + const none: HardwareBreaches = { cpu: false, memory: false, disk: false, temp: false }; + const hardwareMonitor = (overrides?: Partial) => + makeMonitor({ type: "hardware", cpuAlertThreshold: 80, - memoryAlertThreshold: 80, - diskAlertThreshold: 80, - tempAlertThreshold: 80, + memoryAlertThreshold: 70, + diskAlertThreshold: 85, + tempAlertThreshold: 65, + ...overrides, }); - const check = makeCheck(); - const breaches = builder.extractThresholdBreaches(monitor, check); + it("returns empty when no breaches are supplied", () => { + const check = makeCheck({ cpu: { usage_percent: 0.99, temperature: [99] } }); - expect(breaches).toEqual([]); + expect(builder.extractThresholdBreaches(hardwareMonitor(), check, undefined)).toEqual([]); }); - // ── CPU ────────────────────────────────────────────────────────── + it("returns empty when every flag is false, whatever the check says", () => { + const check = makeCheck({ + cpu: { usage_percent: 0.99, temperature: [99] }, + memory: { usage_percent: 0.99 }, + disk: [{ usage_percent: 0.99 }], + }); - describe("cpu threshold", () => { - it("detects CPU breach when usage exceeds threshold", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); + expect(builder.extractThresholdBreaches(hardwareMonitor(), check, none)).toEqual([]); + }); + + it("returns empty for a non-hardware monitor even when flags are set", () => { + const check = makeCheck({ cpu: { usage_percent: 0.99 } }); + + expect(builder.extractThresholdBreaches(makeMonitor({ type: "http" }), check, { ...none, cpu: true })).toEqual([]); + }); + + // The flags are the verdict. The builder formats what it is told is breaching and never re-runs the comparison. + it("reports a flagged metric without re-testing it against the threshold", () => { + const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } }); + + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); + + expect(breaches).toEqual([{ metric: "cpu", currentValue: 50, threshold: 80, unit: "%", formattedValue: "50.0%" }]); + }); + + describe("cpu", () => { + it("formats cpu usage as a percentage when flagged", () => { const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "cpu", - currentValue: 90, - threshold: 80, - unit: "%", - formattedValue: "90.0%", - }) - ); + expect(breaches).toEqual([{ metric: "cpu", currentValue: 90, threshold: 80, unit: "%", formattedValue: "90.0%" }]); }); - it("does not report CPU breach when usage is below threshold", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } }); + it("ignores cpu when not flagged", () => { + const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpuAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: undefined }); - const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpuAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: null as any }); - const check = makeCheck({ cpu: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpu usage_percent is undefined", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: undefined } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); - }); - - it("skips CPU check when cpu object is undefined", () => { - const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 }); - const check = makeCheck({ cpu: undefined }); - - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true }); expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined(); }); }); - // ── Memory ─────────────────────────────────────────────────────── - - describe("memory threshold", () => { - it("detects memory breach when usage exceeds threshold", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); + describe("memory", () => { + it("formats memory usage as a percentage when flagged", () => { const check = makeCheck({ memory: { usage_percent: 0.85 } }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "memory", - currentValue: 85, - threshold: 70, - unit: "%", - formattedValue: "85.0%", - }) - ); + expect(breaches).toEqual([{ metric: "memory", currentValue: 85, threshold: 70, unit: "%", formattedValue: "85.0%" }]); }); - it("does not report memory breach when usage is below threshold", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 90 }); - const check = makeCheck({ memory: { usage_percent: 0.5 } }); + it("ignores memory when not flagged", () => { + const check = makeCheck({ memory: { usage_percent: 0.85 } }); - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memoryAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: undefined }); - const check = makeCheck({ memory: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memoryAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: null as any }); - const check = makeCheck({ memory: { usage_percent: 0.99 } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memory usage_percent is undefined", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); - const check = makeCheck({ memory: { usage_percent: undefined } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); - }); - - it("skips memory check when memory object is undefined", () => { - const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 }); - const check = makeCheck({ memory: undefined }); - - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); expect(breaches.find((b) => b.metric === "memory")).toBeUndefined(); }); }); - // ── Disk ───────────────────────────────────────────────────────── - - describe("disk threshold", () => { - it("detects disk breach using highest usage across multiple disks", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 }); + describe("disk", () => { + it("formats the highest usage across disks when flagged", () => { const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.95 }, { usage_percent: 0.7 }] }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "disk", - currentValue: 95, - threshold: 80, - formattedValue: "95.0%", - }) - ); + expect(breaches).toEqual([{ metric: "disk", currentValue: 95, threshold: 85, unit: "%", formattedValue: "95.0%" }]); }); - it("does not report disk breach when all disks are below threshold", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 90 }); - const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.6 }] }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disk check when diskAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: undefined }); - const check = makeCheck({ disk: [{ usage_percent: 0.99 }] }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disk check when diskAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: null as any }); - const check = makeCheck({ disk: [{ usage_percent: 0.99 }] }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disk check when disk is not an array", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 }); - const check = makeCheck({ disk: "not-an-array" as any }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); - }); - - it("skips disks with undefined usage_percent", () => { - const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 50 }); + it("treats a disk with no usage figure as zero when finding the highest", () => { const check = makeCheck({ disk: [{ usage_percent: undefined }, { usage_percent: 0.3 }] }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true }); + + expect(breaches).toEqual([expect.objectContaining({ metric: "disk", currentValue: 30 })]); + }); + + it("ignores disk when not flagged", () => { + const check = makeCheck({ disk: [{ usage_percent: 0.95 }] }); + + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true }); expect(breaches.find((b) => b.metric === "disk")).toBeUndefined(); }); }); - // ── Temperature ────────────────────────────────────────────────── - - describe("temperature threshold", () => { - it("detects temperature breach from array of temperatures", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); + describe("temperature", () => { + it("formats the highest core temperature when flagged", () => { const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [65, 75, 68] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true }); - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "temp", - currentValue: 75, - threshold: 70, - unit: "°C", - formattedValue: "75.0°C", - }) - ); + expect(breaches).toEqual([{ metric: "temp", currentValue: 75, threshold: 65, unit: "°C", formattedValue: "75.0°C" }]); }); - it("handles single temperature value (non-array)", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: 80 as any } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches).toContainEqual( - expect.objectContaining({ - metric: "temp", - currentValue: 80, - threshold: 70, - }) - ); - }); - - it("does not report temperature breach when below threshold", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [60, 65] } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - // A temperature exactly AT the threshold must not report a breach here, matching - // StatusService.computeHardwareStatus's own strict `>` comparison (cpu/memory/disk - // already agree with this file on `>`; temp alone used to diverge with `>=`, so an - // incident-driving check at exactly the threshold never fired but the notification - // content for a breach some OTHER metric triggered could still list temp as breaching). - it("does not report temperature breach when exactly at threshold", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [80] } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when tempAlertThreshold is undefined", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined }); + it("ignores temperature when not flagged", () => { const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } }); - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when tempAlertThreshold is null", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: null as any }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when cpu.temperature is falsy", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: null as any } }); - - const breaches = builder.extractThresholdBreaches(monitor, check); - - expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); - }); - - it("skips temperature check when cpu object is undefined", () => { - const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 }); - const check = makeCheck({ cpu: undefined }); - - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true }); expect(breaches.find((b) => b.metric === "temp")).toBeUndefined(); }); }); - // ── Combined ───────────────────────────────────────────────────── - - describe("combined thresholds", () => { - it("detects multiple breaches simultaneously", () => { - const monitor = makeMonitor({ - type: "hardware", - cpuAlertThreshold: 80, - memoryAlertThreshold: 70, - diskAlertThreshold: 85, - tempAlertThreshold: 65, - }); + describe("combined", () => { + it("reports every flagged metric in cpu, memory, disk, temp order", () => { const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [70] }, memory: { usage_percent: 0.85 }, disk: [{ usage_percent: 0.95 }], }); - const breaches = builder.extractThresholdBreaches(monitor, check); + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { cpu: true, memory: true, disk: true, temp: true }); - const metrics = breaches.map((b) => b.metric); - expect(metrics).toContain("cpu"); - expect(metrics).toContain("memory"); - expect(metrics).toContain("disk"); - expect(metrics).toContain("temp"); + expect(breaches.map((b) => b.metric)).toEqual(["cpu", "memory", "disk", "temp"]); + }); + + it("reports only the flagged subset", () => { + const check = makeCheck({ + cpu: { usage_percent: 0.9, temperature: [70] }, + memory: { usage_percent: 0.85 }, + disk: [{ usage_percent: 0.95 }], + }); + + const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true, temp: true }); + + expect(breaches.map((b) => b.metric)).toEqual(["memory", "temp"]); }); }); }); diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index 2e1335f88..bbb389573 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -518,24 +518,6 @@ describe("WorkerPipeline", () => { expect(check.egressStatus).toBe("ok"); }); - it("buffers the check when the egress assessment rejects", async () => { - const check: Record = { id: "check-1" }; - const { pipeline, deps } = createPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) }, - }); - - const result = await pipeline.produce(makeMonitor()); - - expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check); - expect(check).not.toHaveProperty("egressStatus"); - expect(result).toEqual({ status: failingStatus, check }); - expect(deps.logger.warn).toHaveBeenCalledWith( - expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "WorkerPipeline", method: "assessEgress" }) - ); - }); - it("leaves the field unset on a failing check when the egress check is disabled", async () => { const check: Record = { id: "check-1" }; const { pipeline, deps } = createPipeline({ @@ -734,7 +716,6 @@ describe("WorkerPipeline", () => { expect(result).toMatchObject({ monitor: statusChange.monitor, check, - statusChange, decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, @@ -777,8 +758,7 @@ describe("WorkerPipeline", () => { expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled(); expect(result.monitor).toBe(monitor); - expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 }); - expect(result.statusChange.timestamp).toEqual(expect.any(Number)); + expect(result).not.toHaveProperty("statusChange"); expect(result.decision).toEqual({ shouldCreateIncident: false, shouldResolveIncident: false, @@ -865,5 +845,56 @@ describe("WorkerPipeline", () => { const decision = await decide(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" })); expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false }); }); + + it("carries the threshold breaches onto a breached decision", async () => { + const thresholdBreaches = { cpu: true, memory: false, disk: true, temp: false }; + const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up", thresholdBreaches })); + expect(decision.thresholdBreaches).toEqual(thresholdBreaches); + }); + + it("leaves threshold breaches off a down decision even when the status change carries them", async () => { + const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false }; + const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500, thresholdBreaches })); + expect(decision).not.toHaveProperty("thresholdBreaches"); + }); + + it("leaves threshold breaches off a no-op decision", async () => { + const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false }; + const decision = await decide(makeStatusChange({ status: "breached", statusChanged: false, prevStatus: "breached", thresholdBreaches })); + expect(decision).not.toHaveProperty("thresholdBreaches"); + }); + + // StatusService only sets thresholdBreaches for hardware monitors; a docker breach must not leave the key behind holding undefined. + it("leaves threshold breaches off a breached decision when the status change carries none", async () => { + const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" })); + expect(decision).toMatchObject({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + expect(decision).not.toHaveProperty("thresholdBreaches"); + }); + + // Each decision starts from a shared template. A transition must not leak its flags into the next decision + // the same pipeline makes, whether that is a no-op evaluation or a degraded-egress short-circuit. + it("starts every decision clean after a transition on the same pipeline", async () => { + const updateMonitorStatus = jest + .fn() + .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 })) + .mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "down", code: 500 })); + const { pipeline } = createPipeline({ statusService: { updateMonitorStatus } }); + const failing = makeCheck({ status: false, statusCode: 500, message: "Error" }); + + const first = await pipeline.evaluateCheck(failing, makeMonitor()); + const second = await pipeline.evaluateCheck(failing, makeMonitor({ status: "down" })); + const degraded = await pipeline.evaluateCheck(makeCheck({ ...failing, egressStatus: "degraded" }), makeMonitor({ status: "down" })); + + expect(first.decision.shouldCreateIncident).toBe(true); + const clean = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }; + expect(second.decision).toEqual(clean); + expect(degraded.decision).toEqual(clean); + }); }); });