Merge pull request #3991 from bluewave-labs/fix/threshold-source-of-truth

fix: threshold source of truth
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-09-24 13:33:12 -07:00
commit 6267e08a51
11 files changed
+232 -455

No files matched your search

@@ -71,7 +71,7 @@ export class IncidentService implements IIncidentService {
// For threshold breaches, use 9999 status code and build descriptive message
if (decision.incidentReason === "threshold_breach") {
statusCode = 9999;
message = this.buildThresholdBreachMessage(monitor, check);
message = this.buildThresholdBreachMessage(monitor, check, decision);
}
const incident = {
@@ -96,14 +96,14 @@ export class IncidentService implements IIncidentService {
return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
};
private buildThresholdBreachMessage(monitor: Monitor, check: Check): string {
private buildThresholdBreachMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision): string {
if (monitor.type === "docker") {
const containers = check.containers ?? [];
const breaches = findContainerBreaches(monitor, containers);
return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert";
}
const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check);
const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check, decision.thresholdBreaches);
if (breaches.length === 0) {
return "Threshold breach detected";
@@ -240,3 +240,8 @@ export type GamesMap = Record<string, Game>;
export type MonitorScheduleFields = Pick<Monitor, "id" | "type" | "isActive" | "interval" | "geoCheckEnabled" | "geoCheckInterval">;
export type DockerContainerLogsResult = DockerLogPage;
export const HardwareMetricKeys = ["cpu", "memory", "disk", "temp"] as const;
export type HardwareMetricKey = (typeof HardwareMetricKeys)[number];
export type HardwareBreaches = Record<HardwareMetricKey, boolean>;
export type HardwareCounters = Record<HardwareMetricKey, number>;
@@ -1,4 +1,4 @@
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import { HardwareMetricKeys, type HardwareBreaches, type HardwareMetricKey, type Monitor } from "@/domain/monitors/monitor.type.js";
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
import type {
NotificationMessage,
@@ -12,7 +12,7 @@ import type { Check } from "@/domain/checks/check.type.js";
export interface INotificationMessageBuilder {
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage;
extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[];
extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[];
}
const SERVICE_NAME = "NotificationMessageBuilder";
@@ -23,7 +23,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage {
const type = this.determineNotificationType(decision, monitor);
const severity = this.determineSeverity(type);
const content = this.buildContent(type, monitor, check);
const content = this.buildContent(type, monitor, check, decision);
return {
type,
@@ -82,14 +82,14 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
}
}
private buildContent(type: NotificationType, monitor: Monitor, check: Check): NotificationContent {
private buildContent(type: NotificationType, monitor: Monitor, check: Check, decision: MonitorActionDecision): NotificationContent {
switch (type) {
case "monitor_down":
return this.buildMonitorDownContent(monitor, check);
case "monitor_up":
return this.buildMonitorUpContent(monitor);
case "threshold_breach":
return this.buildThresholdBreachContent(monitor, check);
return this.buildThresholdBreachContent(monitor, check, decision.thresholdBreaches);
case "threshold_resolved":
return this.buildThresholdResolvedContent(monitor);
case "container_breach":
@@ -137,12 +137,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}
private buildThresholdBreachContent(monitor: Monitor, check: Check): NotificationContent {
private buildThresholdBreachContent(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): NotificationContent {
const title = `Threshold Exceeded: ${monitor.name}`;
const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`;
const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`];
const thresholds = this.extractThresholdBreaches(monitor, check);
const thresholds = this.extractThresholdBreaches(monitor, check, thresholdBreaches);
return {
title,
@@ -191,83 +191,38 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}
public extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[] {
private describeBreach(metric: HardwareMetricKey, monitor: Monitor, check: Check): ThresholdBreach {
switch (metric) {
case "cpu": {
const currentValue = (check.cpu?.usage_percent ?? 0) * 100;
return { metric, currentValue, threshold: monitor.cpuAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
}
case "memory": {
const currentValue = (check.memory?.usage_percent ?? 0) * 100;
return { metric, currentValue, threshold: monitor.memoryAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
}
case "disk": {
const currentValue = Math.max(0, ...(check.disk ?? []).map((disk) => disk?.usage_percent ?? 0)) * 100;
return { metric, currentValue, threshold: monitor.diskAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
}
case "temp": {
const currentValue = Math.max(0, ...(check.cpu?.temperature ?? []));
return { metric, currentValue, threshold: monitor.tempAlertThreshold, unit: "°C", formattedValue: `${currentValue.toFixed(1)}°C` };
}
}
}
public extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] {
const breaches: ThresholdBreach[] = [];
// Check if this is a hardware monitor with threshold data
if (monitor.type !== "hardware") {
if (monitor.type !== "hardware" || thresholdBreaches === undefined) {
return breaches;
}
// Note: usage_percent values in hardware payload are decimals (0-1)
if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && check.cpu?.usage_percent !== undefined) {
const cpuUsageDecimal = check.cpu.usage_percent;
const cpuPercent = cpuUsageDecimal * 100;
const threshold = monitor.cpuAlertThreshold;
if (cpuPercent > threshold) {
breaches.push({
metric: "cpu",
currentValue: cpuPercent,
threshold,
unit: "%",
formattedValue: `${cpuPercent.toFixed(1)}%`,
});
}
}
// Memory threshold breach
if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && check.memory?.usage_percent !== undefined) {
const memoryUsageDecimal = check.memory.usage_percent;
const memoryPercent = memoryUsageDecimal * 100;
const threshold = monitor.memoryAlertThreshold;
if (memoryPercent > threshold) {
breaches.push({
metric: "memory",
currentValue: memoryPercent,
threshold,
unit: "%",
formattedValue: `${memoryPercent.toFixed(1)}%`,
});
}
}
// Disk threshold breach
if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(check.disk)) {
// Find the highest disk usage
let maxDiskUsageDecimal = 0;
for (const disk of check.disk) {
if (disk.usage_percent !== undefined && disk.usage_percent > maxDiskUsageDecimal) {
maxDiskUsageDecimal = disk.usage_percent;
}
}
const maxDiskPercent = maxDiskUsageDecimal * 100;
const threshold = monitor.diskAlertThreshold;
if (maxDiskPercent > threshold) {
breaches.push({
metric: "disk",
currentValue: maxDiskPercent,
threshold,
unit: "%",
formattedValue: `${maxDiskPercent.toFixed(1)}%`,
});
}
}
// Temperature threshold breach
if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && check.cpu?.temperature) {
// Temperature is an array in cpu.temperature
const temps = Array.isArray(check.cpu.temperature) ? check.cpu.temperature : [check.cpu.temperature];
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
const threshold = monitor.tempAlertThreshold;
if (maxTemp > threshold) {
breaches.push({
metric: "temp",
currentValue: maxTemp,
threshold,
unit: "°C",
formattedValue: `${maxTemp.toFixed(1)}°C`,
});
}
for (const metric of HardwareMetricKeys) {
if (!thresholdBreaches[metric]) continue;
breaches.push(this.describeBreach(metric, monitor, check));
}
return breaches;
@@ -1,3 +1,5 @@
import type { HardwareMetricKey } from "@/domain/monitors/monitor.type.js";
export const NotificationChannels = [
"email",
"slack",
@@ -85,7 +87,7 @@ export interface MonitorInfo {
}
export interface ThresholdBreach {
metric: "cpu" | "memory" | "disk" | "temp";
metric: HardwareMetricKey;
currentValue: number;
threshold: number;
unit: string;
+11 -8
View File
@@ -1,7 +1,14 @@
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js";
import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js";
import {
HardwareMetricKeys,
MonitorStatuses,
type HardwareBreaches,
type HardwareCounters,
type Monitor,
type MonitorStatus,
} from "@/domain/monitors/monitor.type.js";
import type { StatusChangeResult } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
@@ -12,10 +19,6 @@ import { findContainerBreaches } from "@/domain/docker/docker-alert.js";
const SERVICE_NAME = "StatusService";
const HARDWARE_ALERT_COUNTER_START = 5;
const HARDWARE_METRIC_KEYS = ["cpu", "memory", "disk", "temp"] as const;
type HardwareMetricKey = (typeof HARDWARE_METRIC_KEYS)[number];
type HardwareBreaches = Record<HardwareMetricKey, boolean>;
type HardwareCounters = Record<HardwareMetricKey, number>;
export interface IStatusService {
updateRunningStats(check: Check, monitor: Monitor): Promise<boolean>;
@@ -111,7 +114,7 @@ export class StatusService implements IStatusService {
// Update counters: decrement (floored at 0) if breached, reset to start otherwise.
const nextCounters = { ...counters };
for (const key of HARDWARE_METRIC_KEYS) {
for (const key of HardwareMetricKeys) {
nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START;
}
@@ -123,8 +126,8 @@ export class StatusService implements IStatusService {
if (!reachabilityDown) {
// A counter can only reach zero via the decrement path, which only runs when that
// metric is currently breaching — so anyCounterZero already implies anyBreached.
const anyCounterZero = HARDWARE_METRIC_KEYS.some((k) => nextCounters[k] === 0);
const allNormal = HARDWARE_METRIC_KEYS.every((k) => !breaches[k]);
const anyCounterZero = HardwareMetricKeys.some((k) => nextCounters[k] === 0);
const allNormal = HardwareMetricKeys.every((k) => !breaches[k]);
if (anyCounterZero && currentStatus !== "breached") {
nextStatus = "breached";
+2 -7
View File
@@ -10,7 +10,7 @@ import type {
ILighthouseAudit,
} from "@/domain/checks/check.type.js";
import { DockerContainerInfo, DockerContainerLogs, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { DnsRecordType, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js";
import type { DnsRecordType, HardwareBreaches, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js";
import type { QueryResult } from "gamedig";
@@ -184,12 +184,7 @@ export type StatusChangeResult = {
prevStatus: MonitorStatus;
code: number;
timestamp: number;
thresholdBreaches?: {
cpu: boolean;
memory: boolean;
disk: boolean;
temp: boolean;
};
thresholdBreaches?: HardwareBreaches;
};
export type MonitorStatusResponseOverrides<T> = Partial<Omit<MonitorStatusResponse<T>, "monitorId" | "teamId" | "type">>;
+2 -9
View File
@@ -1,7 +1,6 @@
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { HardwareBreaches, Monitor } from "@/domain/monitors/monitor.type.js";
import { Check } from "@/domain/checks/check.type.js";
import { Job, JobType } from "@/domain/jobs/job.type.js";
import { StatusChangeResult } from "@/types/network.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
import type { QueueMode } from "@/domain/app-settings/app-settings.type.js";
@@ -13,18 +12,12 @@ export interface MonitorActionDecision {
shouldSendNotification: boolean;
incidentReason: "status_down" | "threshold_breach" | null;
notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null;
thresholdBreaches?: {
cpu?: boolean;
memory?: boolean;
disk?: boolean;
temp?: boolean;
};
thresholdBreaches?: HardwareBreaches;
}
export type MonitorEvaluation = {
monitor: Monitor;
check: Check;
statusChange: StatusChangeResult; // from statusService.updateMonitorStatus
decision: MonitorActionDecision;
};
+14 -35
View File
@@ -3,7 +3,6 @@ import { ICheckService } from "@/domain/checks/check.service.js";
import { Check } from "@/domain/checks/check.type.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
@@ -21,7 +20,13 @@ import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js";
const SERVICE_NAME = "WorkerPipeline";
const NO_ACTION: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
// The three entry points: the queue calls the two handlers, the buffer calls ingestChecks.
// produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer.
export interface IWorkerPipeline {
@@ -128,7 +133,7 @@ export class WorkerPipeline implements IWorkerPipeline {
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
const egressStatus = this.isTransportFailure(monitor, status) ? await this.deps.egressService.assessAfterFailure() : null;
if (egressStatus !== null) {
check.egressStatus = egressStatus;
}
@@ -171,20 +176,6 @@ export class WorkerPipeline implements IWorkerPipeline {
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
}
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
try {
return await this.deps.egressService.assessAfterFailure();
} catch (error: unknown) {
this.deps.logger.warn({
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "assessEgress",
});
return null;
}
}
// ****************************************************************
// Stage 2: Ingestion
// Called by the buffer, checks are inserted and evaluation jobs created
@@ -252,7 +243,7 @@ export class WorkerPipeline implements IWorkerPipeline {
});
return;
}
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
current = evaluation.monitor; // fresh statusWindow/status/counters for the next check
}
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
@@ -277,18 +268,11 @@ export class WorkerPipeline implements IWorkerPipeline {
service: SERVICE_NAME,
method: "evaluateCheck",
});
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
return {
monitor,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: check.statusCode, timestamp: Date.now() },
decision,
decision: { ...NO_ACTION },
};
};
@@ -300,18 +284,12 @@ export class WorkerPipeline implements IWorkerPipeline {
const statusChange = await this.deps.statusService.updateMonitorStatus(check, monitor);
const decision = this.decide(statusChange);
return { monitor: statusChange.monitor, check, statusChange, decision };
return { monitor: statusChange.monitor, check, decision };
};
private decide = (statusChange: StatusChangeResult): MonitorActionDecision => {
const { monitor, statusChanged, prevStatus } = statusChange;
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
const decision: MonitorActionDecision = { ...NO_ACTION };
if (!statusChanged) return decision;
if (monitor.status === "down") {
@@ -320,6 +298,7 @@ export class WorkerPipeline implements IWorkerPipeline {
decision.incidentReason = "status_down";
decision.notificationReason = "status_change";
} else if (monitor.status === "breached") {
if (statusChange.thresholdBreaches) decision.thresholdBreaches = statusChange.thresholdBreaches;
decision.shouldCreateIncident = true;
decision.shouldSendNotification = true;
decision.incidentReason = "threshold_breach";
@@ -163,10 +163,11 @@ describe("IncidentService", () => {
const monitor = makeMonitor({ type: "hardware" });
const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } });
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false };
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach", thresholdBreaches });
await service.handleIncident(monitor, decision, check);
expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check);
expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check, thresholdBreaches);
expect(incidentsRepository.create).toHaveBeenCalledWith(
expect.objectContaining({
statusCode: 9999,
@@ -1,6 +1,6 @@
import { describe, expect, it, beforeEach } from "@jest/globals";
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { HardwareBreaches, Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
@@ -119,7 +119,10 @@ describe("NotificationMessageBuilder", () => {
it("builds a threshold_breach message", () => {
const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 });
const decision = makeDecision({ notificationReason: "threshold_breach" });
const decision = makeDecision({
notificationReason: "threshold_breach",
thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false },
});
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const msg = builder.buildMessage(monitor, check, decision, "https://app.example.com");
@@ -346,21 +349,26 @@ describe("NotificationMessageBuilder", () => {
});
describe("threshold_breach", () => {
it("includes threshold breaches in content", () => {
it("includes the breaches flagged on the decision in content", () => {
const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 });
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const decision = makeDecision({
notificationReason: "threshold_breach",
thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false },
});
const msg = builder.buildMessage(monitor, check, decision, "");
expect(msg.content.thresholds).toEqual([expect.objectContaining({ metric: "cpu", currentValue: 90, threshold: 80 })]);
});
it("leaves thresholds empty when the decision carries no breaches", () => {
const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 });
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const msg = builder.buildMessage(monitor, check, makeDecision({ notificationReason: "threshold_breach" }), "");
expect(msg.content.thresholds).toEqual(
expect.arrayContaining([
expect.objectContaining({
metric: "cpu",
currentValue: 90,
threshold: 80,
}),
])
);
expect(msg.content.thresholds).toEqual([]);
});
});
@@ -480,346 +488,151 @@ describe("NotificationMessageBuilder", () => {
// ── extractThresholdBreaches ─────────────────────────────────────────
describe("extractThresholdBreaches", () => {
it("returns empty array for non-hardware monitor", () => {
const monitor = makeMonitor({ type: "http" });
const check = makeCheck({ cpu: { usage_percent: 0.99 } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches).toEqual([]);
});
it("returns empty array when the check carries no hardware data", () => {
const monitor = makeMonitor({
const none: HardwareBreaches = { cpu: false, memory: false, disk: false, temp: false };
const hardwareMonitor = (overrides?: Partial<Monitor>) =>
makeMonitor({
type: "hardware",
cpuAlertThreshold: 80,
memoryAlertThreshold: 80,
diskAlertThreshold: 80,
tempAlertThreshold: 80,
memoryAlertThreshold: 70,
diskAlertThreshold: 85,
tempAlertThreshold: 65,
...overrides,
});
const check = makeCheck();
const breaches = builder.extractThresholdBreaches(monitor, check);
it("returns empty when no breaches are supplied", () => {
const check = makeCheck({ cpu: { usage_percent: 0.99, temperature: [99] } });
expect(breaches).toEqual([]);
expect(builder.extractThresholdBreaches(hardwareMonitor(), check, undefined)).toEqual([]);
});
// ── CPU ──────────────────────────────────────────────────────────
it("returns empty when every flag is false, whatever the check says", () => {
const check = makeCheck({
cpu: { usage_percent: 0.99, temperature: [99] },
memory: { usage_percent: 0.99 },
disk: [{ usage_percent: 0.99 }],
});
describe("cpu threshold", () => {
it("detects CPU breach when usage exceeds threshold", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
expect(builder.extractThresholdBreaches(hardwareMonitor(), check, none)).toEqual([]);
});
it("returns empty for a non-hardware monitor even when flags are set", () => {
const check = makeCheck({ cpu: { usage_percent: 0.99 } });
expect(builder.extractThresholdBreaches(makeMonitor({ type: "http" }), check, { ...none, cpu: true })).toEqual([]);
});
// The flags are the verdict. The builder formats what it is told is breaching and never re-runs the comparison.
it("reports a flagged metric without re-testing it against the threshold", () => {
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } });
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
expect(breaches).toEqual([{ metric: "cpu", currentValue: 50, threshold: 80, unit: "%", formattedValue: "50.0%" }]);
});
describe("cpu", () => {
it("formats cpu usage as a percentage when flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "cpu",
currentValue: 90,
threshold: 80,
unit: "%",
formattedValue: "90.0%",
})
);
expect(breaches).toEqual([{ metric: "cpu", currentValue: 90, threshold: 80, unit: "%", formattedValue: "90.0%" }]);
});
it("does not report CPU breach when usage is below threshold", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } });
it("ignores cpu when not flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpuAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: undefined });
const check = makeCheck({ cpu: { usage_percent: 0.99 } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpuAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: null as any });
const check = makeCheck({ cpu: { usage_percent: 0.99 } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpu usage_percent is undefined", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
const check = makeCheck({ cpu: { usage_percent: undefined } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpu object is undefined", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
const check = makeCheck({ cpu: undefined });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true });
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
});
// ── Memory ───────────────────────────────────────────────────────
describe("memory threshold", () => {
it("detects memory breach when usage exceeds threshold", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 });
describe("memory", () => {
it("formats memory usage as a percentage when flagged", () => {
const check = makeCheck({ memory: { usage_percent: 0.85 } });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true });
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "memory",
currentValue: 85,
threshold: 70,
unit: "%",
formattedValue: "85.0%",
})
);
expect(breaches).toEqual([{ metric: "memory", currentValue: 85, threshold: 70, unit: "%", formattedValue: "85.0%" }]);
});
it("does not report memory breach when usage is below threshold", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 90 });
const check = makeCheck({ memory: { usage_percent: 0.5 } });
it("ignores memory when not flagged", () => {
const check = makeCheck({ memory: { usage_percent: 0.85 } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memoryAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: undefined });
const check = makeCheck({ memory: { usage_percent: 0.99 } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memoryAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: null as any });
const check = makeCheck({ memory: { usage_percent: 0.99 } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memory usage_percent is undefined", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 });
const check = makeCheck({ memory: { usage_percent: undefined } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memory object is undefined", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 });
const check = makeCheck({ memory: undefined });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
});
// ── Disk ─────────────────────────────────────────────────────────
describe("disk threshold", () => {
it("detects disk breach using highest usage across multiple disks", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 });
describe("disk", () => {
it("formats the highest usage across disks when flagged", () => {
const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.95 }, { usage_percent: 0.7 }] });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true });
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "disk",
currentValue: 95,
threshold: 80,
formattedValue: "95.0%",
})
);
expect(breaches).toEqual([{ metric: "disk", currentValue: 95, threshold: 85, unit: "%", formattedValue: "95.0%" }]);
});
it("does not report disk breach when all disks are below threshold", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 90 });
const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.6 }] });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
it("skips disk check when diskAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: undefined });
const check = makeCheck({ disk: [{ usage_percent: 0.99 }] });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
it("skips disk check when diskAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: null as any });
const check = makeCheck({ disk: [{ usage_percent: 0.99 }] });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
it("skips disk check when disk is not an array", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 });
const check = makeCheck({ disk: "not-an-array" as any });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
it("skips disks with undefined usage_percent", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 50 });
it("treats a disk with no usage figure as zero when finding the highest", () => {
const check = makeCheck({ disk: [{ usage_percent: undefined }, { usage_percent: 0.3 }] });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true });
expect(breaches).toEqual([expect.objectContaining({ metric: "disk", currentValue: 30 })]);
});
it("ignores disk when not flagged", () => {
const check = makeCheck({ disk: [{ usage_percent: 0.95 }] });
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true });
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
});
// ── Temperature ──────────────────────────────────────────────────
describe("temperature threshold", () => {
it("detects temperature breach from array of temperatures", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
describe("temperature", () => {
it("formats the highest core temperature when flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [65, 75, 68] } });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true });
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "temp",
currentValue: 75,
threshold: 70,
unit: "°C",
formattedValue: "75.0°C",
})
);
expect(breaches).toEqual([{ metric: "temp", currentValue: 75, threshold: 65, unit: "°C", formattedValue: "75.0°C" }]);
});
it("handles single temperature value (non-array)", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: 80 as any } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "temp",
currentValue: 80,
threshold: 70,
})
);
});
it("does not report temperature breach when below threshold", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 });
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [60, 65] } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
// A temperature exactly AT the threshold must not report a breach here, matching
// StatusService.computeHardwareStatus's own strict `>` comparison (cpu/memory/disk
// already agree with this file on `>`; temp alone used to diverge with `>=`, so an
// incident-driving check at exactly the threshold never fired but the notification
// content for a breach some OTHER metric triggered could still list temp as breaching).
it("does not report temperature breach when exactly at threshold", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 });
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [80] } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when tempAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined });
it("ignores temperature when not flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when tempAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: null as any });
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when cpu.temperature is falsy", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: null as any } });
const breaches = builder.extractThresholdBreaches(monitor, check);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when cpu object is undefined", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
const check = makeCheck({ cpu: undefined });
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
});
// ── Combined ─────────────────────────────────────────────────────
describe("combined thresholds", () => {
it("detects multiple breaches simultaneously", () => {
const monitor = makeMonitor({
type: "hardware",
cpuAlertThreshold: 80,
memoryAlertThreshold: 70,
diskAlertThreshold: 85,
tempAlertThreshold: 65,
});
describe("combined", () => {
it("reports every flagged metric in cpu, memory, disk, temp order", () => {
const check = makeCheck({
cpu: { usage_percent: 0.9, temperature: [70] },
memory: { usage_percent: 0.85 },
disk: [{ usage_percent: 0.95 }],
});
const breaches = builder.extractThresholdBreaches(monitor, check);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { cpu: true, memory: true, disk: true, temp: true });
const metrics = breaches.map((b) => b.metric);
expect(metrics).toContain("cpu");
expect(metrics).toContain("memory");
expect(metrics).toContain("disk");
expect(metrics).toContain("temp");
expect(breaches.map((b) => b.metric)).toEqual(["cpu", "memory", "disk", "temp"]);
});
it("reports only the flagged subset", () => {
const check = makeCheck({
cpu: { usage_percent: 0.9, temperature: [70] },
memory: { usage_percent: 0.85 },
disk: [{ usage_percent: 0.95 }],
});
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true, temp: true });
expect(breaches.map((b) => b.metric)).toEqual(["memory", "temp"]);
});
});
});
@@ -518,24 +518,6 @@ describe("WorkerPipeline", () => {
expect(check.egressStatus).toBe("ok");
});
it("buffers the check when the egress assessment rejects", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockRejectedValue(new Error("settings unavailable")) },
});
const result = await pipeline.produce(makeMonitor());
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
expect(check).not.toHaveProperty("egressStatus");
expect(result).toEqual({ status: failingStatus, check });
expect(deps.logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "WorkerPipeline", method: "assessEgress" })
);
});
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
@@ -734,7 +716,6 @@ describe("WorkerPipeline", () => {
expect(result).toMatchObject({
monitor: statusChange.monitor,
check,
statusChange,
decision: expect.objectContaining({
shouldCreateIncident: true,
shouldSendNotification: true,
@@ -777,8 +758,7 @@ describe("WorkerPipeline", () => {
expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled();
expect(result.monitor).toBe(monitor);
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
expect(result).not.toHaveProperty("statusChange");
expect(result.decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
@@ -865,5 +845,56 @@ describe("WorkerPipeline", () => {
const decision = await decide(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false });
});
it("carries the threshold breaches onto a breached decision", async () => {
const thresholdBreaches = { cpu: true, memory: false, disk: true, temp: false };
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up", thresholdBreaches }));
expect(decision.thresholdBreaches).toEqual(thresholdBreaches);
});
it("leaves threshold breaches off a down decision even when the status change carries them", async () => {
const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false };
const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500, thresholdBreaches }));
expect(decision).not.toHaveProperty("thresholdBreaches");
});
it("leaves threshold breaches off a no-op decision", async () => {
const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false };
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: false, prevStatus: "breached", thresholdBreaches }));
expect(decision).not.toHaveProperty("thresholdBreaches");
});
// StatusService only sets thresholdBreaches for hardware monitors; a docker breach must not leave the key behind holding undefined.
it("leaves threshold breaches off a breached decision when the status change carries none", async () => {
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
expect(decision).not.toHaveProperty("thresholdBreaches");
});
// Each decision starts from a shared template. A transition must not leak its flags into the next decision
// the same pipeline makes, whether that is a no-op evaluation or a degraded-egress short-circuit.
it("starts every decision clean after a transition on the same pipeline", async () => {
const updateMonitorStatus = jest
.fn<any>()
.mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 }))
.mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "down", code: 500 }));
const { pipeline } = createPipeline({ statusService: { updateMonitorStatus } });
const failing = makeCheck({ status: false, statusCode: 500, message: "Error" });
const first = await pipeline.evaluateCheck(failing, makeMonitor());
const second = await pipeline.evaluateCheck(failing, makeMonitor({ status: "down" }));
const degraded = await pipeline.evaluateCheck(makeCheck({ ...failing, egressStatus: "degraded" }), makeMonitor({ status: "down" }));
expect(first.decision.shouldCreateIncident).toBe(true);
const clean = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
expect(second.decision).toEqual(clean);
expect(degraded.decision).toEqual(clean);
});
});
});