import { describe, expect, it, jest, beforeEach } from "@jest/globals"; import { StatusService } from "../../../src/service/statusService.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts"; import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── const createMonitorsRepo = () => { const findById = jest.fn(); const updateStatusWindowAndChecks = jest .fn() .mockImplementation( async ( _id: unknown, _tid: unknown, status: boolean, checkSnapshot: unknown, windowSize: number, maxRecentChecks: number, statusPatch?: Partial ) => { // updateMonitorStatus no longer reads the monitor itself — tests seed it via // findById.mockResolvedValue, so the mock resolves it here as the post-write monitor. const m: any = await findById(); m.statusWindow = m.statusWindow || []; m.statusWindow.push(status); while (m.statusWindow.length > windowSize) { m.statusWindow.shift(); } m.recentChecks = m.recentChecks || []; m.recentChecks.push(checkSnapshot); while (m.recentChecks.length > maxRecentChecks) { m.recentChecks.shift(); } if (statusPatch) { Object.assign(m, statusPatch); } return { ...m }; } ); return { findById, updateById: jest.fn(), updateStatusWindowAndChecks, } as unknown as jest.Mocked; }; const createMonitorStatsRepo = () => ({ findByMonitorId: jest.fn(), create: jest.fn(), updateByMonitorId: jest.fn(), }) as unknown as jest.Mocked; const createService = (overrides?: { logger?: ReturnType; monitorsRepository?: ReturnType; monitorStatsRepository?: ReturnType; }) => { const logger = overrides?.logger ?? createMockLogger(); const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo(); const monitorStatsRepository = overrides?.monitorStatsRepository ?? createMonitorStatsRepo(); const service = new StatusService(logger as any, monitorsRepository, monitorStatsRepository); return { service, logger, monitorsRepository, monitorStatsRepository }; }; const makeMonitor = (overrides?: Partial): Monitor => ({ id: "mon-1", userId: "user-1", teamId: "team-1", name: "Test Monitor", type: "http", url: "https://example.com", isActive: true, interval: 60000, status: "up", statusWindow: [], statusWindowSize: 5, statusWindowThreshold: 80, recentChecks: [], cpuAlertThreshold: 80, memoryAlertThreshold: 80, diskAlertThreshold: 80, tempAlertThreshold: 80, cpuAlertCounter: 5, memoryAlertCounter: 5, diskAlertCounter: 5, tempAlertCounter: 5, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", ...overrides, }) as Monitor; const makeCheck = (overrides?: Partial): Check => ({ id: "check-1", metadata: { monitorId: "mon-1", teamId: "team-1" }, status: true, responseTime: 100, statusCode: 200, message: "OK", createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", ...overrides, }) as Check; const makeExistingStats = (overrides?: Record) => ({ id: "stats-1", monitorId: "mon-1", avgResponseTime: 100, maxResponseTime: 200, totalChecks: 10, totalUpChecks: 9, totalDownChecks: 1, uptimePercentage: 0.9, lastResponseTime: 90, lastCheckTimestamp: 1000, timeOfLastFailure: 500, createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", ...overrides, }); // ── Tests ──────────────────────────────────────────────────────────────────── describe("StatusService", () => { // ── running stats (via updateMonitorStatus) ────────────────────────────── describe("running stats", () => { // The service delegates the running-stats computation to the repository's atomic `updateByMonitorId`, // so these cases only verify what is forwarded and that a failure never blocks the status update. // The math (running average, uptimePercentage, timeOfLastFailure) is exercised by the repository tests. const run = async (check: Check) => { const monitor = makeMonitor(); const deps = createService(); (deps.monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (deps.monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({}); const result = await deps.service.updateMonitorStatus(check, monitor); return { ...deps, result }; }; it("forwards a successful check to updateByMonitorId", async () => { const { monitorStatsRepository } = await run(makeCheck({ responseTime: 50 })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith( "mon-1", expect.objectContaining({ status: true, responseTime: 50, now: expect.any(Number) }) ); }); it("forwards a failed check to updateByMonitorId", async () => { const { monitorStatsRepository } = await run(makeCheck({ status: false, responseTime: 100 })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false, responseTime: 100 })); }); it("passes 0 when responseTime is undefined", async () => { const { monitorStatsRepository } = await run(makeCheck({ responseTime: undefined as any })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 })); }); it("forwards responseTime of 0 as 0 (falsy but defined)", async () => { const { monitorStatsRepository } = await run(makeCheck({ responseTime: 0 })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 })); }); it("coerces a non-true status to false (e.g. undefined)", async () => { const { monitorStatsRepository } = await run(makeCheck({ status: undefined as any })); expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false })); }); it("warns once and still updates the status when updateByMonitorId throws", async () => { const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" }); const { service, logger, monitorsRepository, monitorStatsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down")); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledTimes(1); expect(logger.warn).toHaveBeenCalledTimes(1); expect(logger.warn).toHaveBeenCalledWith( expect.objectContaining({ service: "StatusService", method: "tryUpdateRunningStats", message: "stats db down", stack: expect.any(String), }) ); expect(logger.error).not.toHaveBeenCalled(); }); it("warns with 'Unknown error' for non-Error rejections", async () => { const monitor = makeMonitor(); const { service, logger, monitorsRepository, monitorStatsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue("string error"); await service.updateMonitorStatus(makeCheck(), monitor); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined })); }); }); // ── updateMonitorStatus ────────────────────────────────────────────────── describe("updateMonitorStatus", () => { it("returns early with no status change when statusWindow is not full", async () => { const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.prevStatus).toBe("up"); }); it("uses the prefetched monitor and skips findById when one is passed", async () => { const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" }); const { service, monitorsRepository } = createService(); (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(monitorsRepository.findById).not.toHaveBeenCalled(); expect(result.prevStatus).toBe("up"); expect(result.monitor).toBe(monitor); }); it("pushes to statusWindow and trims to statusWindowSize", async () => { const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); await service.updateMonitorStatus(makeCheck({ status: false }), monitor); // Atomic push should have been called with the correct status and window size expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith( "mon-1", "team-1", false, expect.any(Object), 5, MAX_RECENT_CHECKS, expect.objectContaining({ status: expect.any(String) }) ); }); it("pushes check snapshot to recentChecks and trims to MAX_RECENT_CHECKS", async () => { const existingChecks = Array.from({ length: MAX_RECENT_CHECKS }, (_, i) => ({ id: `old-${i}` })); const monitor = makeMonitor({ recentChecks: existingChecks as any, statusWindow: [], statusWindowSize: 5 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); await service.updateMonitorStatus(makeCheck({ id: "new-check" }), monitor); expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS); expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" })); }); it("marks status as down when failure threshold is met", async () => { // 5/5 failures = 100% >= 80% threshold const monitor = makeMonitor({ statusWindow: [false, false, false, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "up", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); }); it("recovers to up when failure rate drops below threshold", async () => { // 1/5 failures = 20% < 80% threshold, and monitor was down const monitor = makeMonitor({ statusWindow: [true, true, true, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "down", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); }); it("does not change status when already up and below threshold", async () => { const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, statusWindowThreshold: 80, status: "up", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("keeps status 'up' on a sub-threshold failed check (regression: #3438)", async () => { // Monitor is up, window has one failure already, incoming check fails. // 2/5 = 40% < 80% threshold — must NOT silently write status='down', // otherwise the next successful check fires a spurious 'Recovered' notification. const monitor = makeMonitor({ statusWindow: [true, true, true, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "up", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("does not change status when already down and still above threshold", async () => { const monitor = makeMonitor({ statusWindow: [false, false, false, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "down", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); }); it("initializes statusWindow when undefined", async () => { const monitor = makeMonitor({ statusWindow: undefined as any, statusWindowSize: 5 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); await service.updateMonitorStatus(makeCheck(), monitor); expect(monitor.statusWindow).toEqual([true]); }); it("initializes recentChecks when undefined", async () => { const monitor = makeMonitor({ recentChecks: undefined as any, statusWindowSize: 5 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); await service.updateMonitorStatus(makeCheck(), monitor); expect(monitor.recentChecks).toHaveLength(1); }); it("throws AppError when repository throws", async () => { const monitor = makeMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error")); await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Failed to update monitor"); }); it("throws AppError with 'Unknown error' for non-Error exceptions", async () => { const monitor = makeMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error"); await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Unknown error"); }); it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => { // Monitor is down, window is [false,false,false,false]. A single successful check makes it // [false,false,false,false,true] → 4/5 = 80%, still at threshold, must NOT silently recover. // Pre-fix, newStatus was seeded "up" from the raw check and fell through to be persisted, // causing the next failing check to fire a spurious second 'Down' notification. const monitor = makeMonitor({ statusWindow: [false, false, false, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "down", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); }); it("triggers 'down' transition exactly at the failure threshold boundary", async () => { // 4/5 = 80%, threshold is 80, comparison is >= so this must trip. const monitor = makeMonitor({ statusWindow: [false, false, false, true], statusWindowSize: 5, statusWindowThreshold: 80, status: "up", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); }); it("resumes from 'initializing' to 'up' on a passing check with a full window (regression: pause/resume)", async () => { // When a monitor is paused and resumed, togglePauseById sets status='initializing' // but does NOT reset statusWindow. On the next check the window is already full, // so the warmup early-return is skipped. Pre-fix, newStatus was seeded from // monitor.status ('initializing') and computeReachability never transitioned // out of 'initializing' on a passing window, so the monitor was stuck forever. const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5, statusWindowThreshold: 80, status: "initializing", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); it("resumes from 'initializing' to 'down' on a failing check with a full window (regression: pause/resume mirror)", async () => { // Mirror of the pause/resume regression with a failing check: should flip to 'down' // immediately rather than stay stuck in 'initializing'. const monitor = makeMonitor({ statusWindow: [false, false, false, false, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "initializing", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); }); it("resolves a legacy boolean status ('true') to 'up' on a passing check (regression: pre-upgrade monitors stuck forever)", async () => { // Old Checkmate versions stored monitor.status as a boolean. Pre-fix, a monitor // carrying status='true' never matched the initializing override and // computeReachability only transitions to 'up' from 'down', so the legacy value // was written back unchanged on every evaluation — stuck forever while the UI // rendered it as 'initializing'. const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5, statusWindowThreshold: 80, status: "true" as MonitorStatus, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); expect(result.statusChanged).toBe(false); // was effectively up already — no notification storm }); it("resolves a legacy boolean status ('false') to 'down' on a failing check (regression mirror)", async () => { const monitor = makeMonitor({ statusWindow: [false, false, false, false, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "false" as MonitorStatus, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); expect(result.statusChanged).toBe(true); // down must surface so an incident is opened }); it("during warmup, sub-threshold raw checks no longer flip stored status (regression: down-during-init incident not resolving)", async () => { // Once the monitor has left 'initializing', the warmup branch must NOT silently // flip status from raw checks. Otherwise a down check during warmup writes // status='down' without statusChanged, and the next up check leaks status='up' // the same way — by the time the window fills, monitor.status is already 'up' // and computeReachability cannot fire down→up to resolve the incident. const monitor = makeMonitor({ statusWindow: [true, true], statusWindowSize: 5, status: "up" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); expect(monitor.statusWindow).toEqual([true, true, false]); }); it("creates incident on initial down check when statusWindowSize is 1 (regression: alert on initial down skips warmup branch)", async () => { // statusWindowSize=1 means the very first check skips the warmup branch entirely // (1 < 1 is false) and lands in the full-window branch. The 'initializing' override // must still surface statusChanged=true on a down result, otherwise computeReachability // sees currentStatus='down' and refuses to transition, and no incident is opened. const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 1, statusWindowThreshold: 60, status: "initializing" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); expect(result.prevStatus).toBe("initializing"); }); it("creates incident on initial down check for a brand-new monitor (regression: alert on initial down)", async () => { // Brand-new monitor: status='initializing', empty statusWindow. First check is down. // Must surface statusChanged=true with monitor.status='down' so the helper opens an incident. const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "initializing" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); expect(result.prevStatus).toBe("initializing"); }); it("leaves 'maintenance' for 'up' on a passing check after the window closes (empty statusWindow)", async () => { // On entry to a maintenance window the producer clears statusWindow, so the first // check after the window closes sees an empty window and lands in the warmup // early-return branch. A passing check must move the monitor out of 'maintenance' // to 'up' without flagging a status change (no incident). const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); expect(result.prevStatus).toBe("maintenance"); }); it("leaves 'maintenance' for 'down' on a failing check after the window closes (empty statusWindow)", async () => { // Mirror of the recovery case: a failing first check after the window closes must // flip the monitor straight to 'down' and surface statusChanged so an incident opens, // rather than leave it stuck in 'maintenance'. const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); expect(result.prevStatus).toBe("maintenance"); }); it("unsticks from 'maintenance' to 'up' on a passing check even with a full window (regression: stuck in maintenance)", async () => { // Guards the up-front override directly: even if a full window survives (e.g. the // entry-clear is bypassed), computeReachability only transitions out to 'up' from // 'down', so without the override the monitor would stay stuck in 'maintenance'. const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5, statusWindowThreshold: 80, status: "maintenance", }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); it("does not flip status on subsequent checks during warmup once 'initializing' has been left (regression: down→up incident resolves once window fills)", async () => { // After a 'initializing'→'down' transition on check #1, monitor.status is now 'down' // and statusWindow has one false. Subsequent up checks during warmup must leave // monitor.status='down' so the sliding window can detect the down→up transition // once the window fills. const monitor = makeMonitor({ statusWindow: [false], statusWindowSize: 5, status: "down" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); }); // ── Hardware threshold breach tests ────────────────────────────────── describe("hardware threshold breaches", () => { const makeHardwareMonitor = (overrides?: Partial) => makeMonitor({ type: "hardware", statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up", cpuAlertThreshold: 80, memoryAlertThreshold: 80, diskAlertThreshold: 80, tempAlertThreshold: 80, cpuAlertCounter: 5, memoryAlertCounter: 5, diskAlertCounter: 5, tempAlertCounter: 5, ...overrides, }); const makeHardwareCheck = (metrics: Pick, overrides?: Partial) => makeCheck({ status: true, ...metrics, ...overrides, }); it("detects CPU threshold breach and decrements counter", async () => { const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.cpu).toBe(true); expect(result.thresholdBreaches?.memory).toBe(false); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(0); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("breached"); }); it("detects memory threshold breach", async () => { const monitor = makeHardwareMonitor({ memoryAlertCounter: 1 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.memory).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.memoryAlertCounter).toBe(0); expect(result.monitor.status).toBe("breached"); }); it("detects disk threshold breach", async () => { const monitor = makeHardwareMonitor({ diskAlertCounter: 1 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.diskAlertCounter).toBe(0); expect(result.monitor.status).toBe("breached"); }); it("detects temperature threshold breach", async () => { const monitor = makeHardwareMonitor({ tempAlertCounter: 1 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.temp).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.tempAlertCounter).toBe(0); expect(result.monitor.status).toBe("breached"); }); it("resets counters to 5 when thresholds are not breached", async () => { const monitor = makeHardwareMonitor({ cpuAlertCounter: 2, memoryAlertCounter: 2, diskAlertCounter: 2, tempAlertCounter: 2, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {}, }); await service.updateMonitorStatus(check, monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(5); expect(patch.memoryAlertCounter).toBe(5); expect(patch.diskAlertCounter).toBe(5); expect(patch.tempAlertCounter).toBe(5); }); it("stays breached without statusChanged when already breached and counter still at 0", async () => { const monitor = makeHardwareMonitor({ status: "breached", cpuAlertCounter: 1, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.monitor.status).toBe("breached"); expect(result.statusChanged).toBe(false); }); it("recovers from breached to up when all thresholds return to normal", async () => { const monitor = makeHardwareMonitor({ status: "breached" }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {}, }); const result = await service.updateMonitorStatus(check, monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); }); it("does not override down status with breached", async () => { // Monitor is down due to failure threshold, hardware breaches should not override const monitor = makeHardwareMonitor({ statusWindow: [false, false, false, false], statusWindowSize: 5, statusWindowThreshold: 80, status: "up", cpuAlertCounter: 1, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false }); const result = await service.updateMonitorStatus(check, monitor); expect(result.monitor.status).toBe("down"); }); it("handles missing cpu usage_percent (returns -1)", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.cpu).toBe(false); }); it("handles missing memory usage_percent", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); it("handles empty temperature array", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); it("handles undefined temperature", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); it("reports no breaches when the check carries no hardware metrics", async () => { // A stored hardware check with no metrics (cpu/memory/disk/host all absent) must not // throw and must not report any breach. const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck(), monitor); expect(result.thresholdBreaches).toEqual({ cpu: false, memory: false, disk: false, temp: false }); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("does not set thresholdBreaches for non-hardware monitors", async () => { const monitor = makeMonitor({ type: "http", statusWindow: [true, true, true, true], statusWindowSize: 5, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); it("handles null entry in disk array", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); it("handles disk with undefined usage_percent entries", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); it("handles nullish cpu in metrics", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.cpu).toBe(false); expect(result.thresholdBreaches?.temp).toBe(false); }); it("handles nullish memory in metrics", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); it("handles nullish disk in metrics", async () => { const monitor = makeHardwareMonitor(); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} }); const result = await service.updateMonitorStatus(check, monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); it("decrements breach counter without changing status while counter > 0", async () => { // Counter at 3 → decrements to 2, status must stay 'up' and statusChanged must be false. // Only counter→0 may flip to 'breached'. const monitor = makeHardwareMonitor({ cpuAlertCounter: 3 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }); const result = await service.updateMonitorStatus(check, monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(2); expect(result.thresholdBreaches?.cpu).toBe(true); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("transitions from 'breached' to 'down' when the reachability threshold trips", async () => { // A hardware monitor currently in 'breached' state starts failing its reachability // checks enough to cross the statusWindow threshold — 'down' must take precedence. const monitor = makeHardwareMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowSize: 5, statusWindowThreshold: 80, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false }); const result = await service.updateMonitorStatus(check, monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); }); it("counters do not go below 0", async () => { const monitor = makeHardwareMonitor({ cpuAlertCounter: 0, memoryAlertCounter: 0, diskAlertCounter: 0, tempAlertCounter: 0, }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const check = makeHardwareCheck({ cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {}, }); await service.updateMonitorStatus(check, monitor); expect(monitor.cpuAlertCounter).toBe(0); expect(monitor.memoryAlertCounter).toBe(0); expect(monitor.diskAlertCounter).toBe(0); expect(monitor.tempAlertCounter).toBe(0); }); }); }); // ── Pure helpers ───────────────────────────────────────────────────────── // These test the private state-machine helpers directly, without repository // or logger mocks. The helpers are accessed via `service as any` because // they're private implementation details; exposing them publicly would // widen the API surface just for testability. describe("docker container alerts", () => { const makeDockerMonitor = (overrides?: Partial) => makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true, ...overrides, }); const makeContainers = (containers: Partial[]): DockerContainerInfo[] => containers.map((container, index) => ({ id: `container-${index}`, name: `container-${index}`, image: "nginx:latest", state: "running", status: "Up 3 hours", health: "none", ...container, })); const makeDockerCheck = (containers: DockerContainerInfo[] | null, overrides?: Partial) => makeCheck({ status: true, ...(containers ? { containers } : {}), ...overrides, }); const run = async (monitor: Monitor, check: Check) => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const result = await service.updateMonitorStatus(check, monitor); return { result, monitorsRepository }; }; const stopped = { name: "db", state: "exited" as const, exitCode: 1 }; const unhealthy = { name: "api", health: "unhealthy" as const }; it("goes to breached on the first check with a stopped container", async () => { const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("breached"); }); it("goes to breached on the first check with an unhealthy container", async () => { const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([unhealthy]))); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("breached"); }); it("stays up on a stopped container when both switches are off", async () => { const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false }); const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped, unhealthy]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("stays up on a stopped container when only the unhealthy switch is on", async () => { const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("stays up on an exited container with code 0", async () => { const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "job", state: "exited", exitCode: 0 }]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("stays breached without a transition while a container is still down", async () => { const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("breached"); }); it("stays up without a transition while every container is clean", async () => { const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "web" }]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("recovers from breached to up on the first clean check", async () => { const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([{ name: "web" }]))); expect(result.statusChanged).toBe(true); expect(result.prevStatus).toBe("breached"); expect(result.monitor.status).toBe("up"); }); it("recovers from breached to up when the breaching switch is turned off", async () => { const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false }); const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); }); it("does not override down with breached when the host is unreachable", async () => { const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); const check = makeDockerCheck(null, { status: false, statusCode: 5000 }); const { result } = await run(monitor, check); expect(result.monitor.status).toBe("down"); }); it("transitions from breached to down when the reachability threshold trips", async () => { const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); const check = makeDockerCheck(null, { status: false, statusCode: 5000 }); const { result } = await run(monitor, check); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); }); it("does not evaluate containers while the status window is short", async () => { const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 }); const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped]))); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); }); it("does not evaluate a failed check with a null payload", async () => { const monitor = makeDockerMonitor({ status: "breached" }); const check = makeDockerCheck(null, { status: false, statusCode: 5000 }); const { result } = await run(monitor, check); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("breached"); }); it("does not touch the hardware counters or thresholdBreaches", async () => { const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped]))); expect(result.thresholdBreaches).toBeUndefined(); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBeUndefined(); expect(patch.status).toBe("breached"); }); }); describe("computeReachability (pure)", () => { const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => { const { service } = createService(); return (service as any).computeReachability(currentStatus, window, threshold) as { nextStatus: "up" | "down"; transitioned: boolean; }; }; // ── Monitor is "up" ── it("up + window below threshold → no transition", () => { expect(reach("up", [true, true, true, true, true])).toEqual({ nextStatus: "up", transitioned: false }); }); it("up + window exactly at threshold → transitions to down (>= boundary)", () => { // 4/5 = 80% and threshold is 80, >= so this must trip expect(reach("up", [false, false, false, false, true])).toEqual({ nextStatus: "down", transitioned: true }); }); it("up + window over threshold → transitions to down", () => { expect(reach("up", [false, false, false, false, false])).toEqual({ nextStatus: "down", transitioned: true }); }); // ── Monitor is "down" ── it("down + window below threshold → transitions to up (recovery)", () => { // 0/5 failures, fully healthy window expect(reach("down", [true, true, true, true, true])).toEqual({ nextStatus: "up", transitioned: true }); }); it("down + window exactly at threshold → no transition (strict < for recovery)", () => { // 4/5 = 80%; recovery requires failureRate < threshold, not <=, so stays down const result = reach("down", [false, false, false, false, true]); expect(result.transitioned).toBe(false); // Orchestrator ignores nextStatus when !transitioned, so the returned value // is irrelevant to observable behavior; documenting here for clarity. }); it("down + window over threshold → no transition", () => { const result = reach("down", [false, false, false, false, false]); expect(result.transitioned).toBe(false); }); // ── Monitor is "breached" (hardware state, passes through reachability) ── it("breached + healthy window → no transition (reachability does not touch breached)", () => { // Critical: reachability must NOT claim to recover a breached monitor. That // transition is owned by the hardware block (all thresholds back to normal). const result = reach("breached", [true, true, true, true, true]); expect(result.transitioned).toBe(false); }); it("breached + window at threshold → transitions to down (down beats breached)", () => { // A breached monitor that also loses reachability must flip to "down". // Current status is "breached" which satisfies `!== "down"`, so the branch fires. expect(reach("breached", [false, false, false, false, true])).toEqual({ nextStatus: "down", transitioned: true }); }); it("breached + window over threshold → transitions to down", () => { expect(reach("breached", [false, false, false, false, false])).toEqual({ nextStatus: "down", transitioned: true }); }); // ── Threshold variations ── it("non-default threshold (50%) trips at 3/5 failures", () => { expect(reach("up", [false, false, false, true, true], 50)).toEqual({ nextStatus: "down", transitioned: true }); }); it("non-default threshold (50%) does not trip at 2/5 failures", () => { expect(reach("up", [false, false, true, true, true], 50).transitioned).toBe(false); }); }); describe("computeHardwareStatus (pure)", () => { type Params = { currentStatus: MonitorStatus; reachabilityDown: boolean; metrics: any; thresholds: { cpu: number; memory: number; disk: number; temp: number }; counters: { cpu: number; memory: number; disk: number; temp: number }; }; const healthyMetrics = () => ({ cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {}, }); const defaults = (): Params => ({ currentStatus: "up", reachabilityDown: false, metrics: healthyMetrics(), thresholds: { cpu: 80, memory: 80, disk: 80, temp: 80 }, counters: { cpu: 5, memory: 5, disk: 5, temp: 5 }, }); const compute = (overrides: Partial = {}) => { const { service } = createService(); return (service as any).computeHardwareStatus({ ...defaults(), ...overrides }); }; // ── Breach detection ── it("detects CPU breach when usage > threshold", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } } }); expect(r.breaches.cpu).toBe(true); }); it("does not flag CPU at exactly the threshold (strict >, not >=)", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.8, temperature: [30] } } }); expect(r.breaches.cpu).toBe(false); }); it("does not flag CPU when usage_percent is missing", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { temperature: [30] } } }); expect(r.breaches.cpu).toBe(false); }); it("detects memory breach", () => { const r = compute({ metrics: { ...healthyMetrics(), memory: { usage_percent: 0.9 } } }); expect(r.breaches.memory).toBe(true); }); it("detects disk breach when any disk exceeds threshold", () => { const r = compute({ metrics: { ...healthyMetrics(), disk: [{ usage_percent: 0.1 }, { usage_percent: 0.95 }] } }); expect(r.breaches.disk).toBe(true); }); it("skips null disk entries", () => { const r = compute({ metrics: { ...healthyMetrics(), disk: [null, { usage_percent: 0.1 }] } }); expect(r.breaches.disk).toBe(false); }); it("skips disk entries missing usage_percent", () => { const r = compute({ metrics: { ...healthyMetrics(), disk: [{ device: "/dev/sda" }] } }); expect(r.breaches.disk).toBe(false); }); it("detects temperature breach when any core exceeds threshold", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [30, 50, 90] } } }); expect(r.breaches.temp).toBe(true); }); it("empty temperature array → no temp breach", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [] } } }); expect(r.breaches.temp).toBe(false); }); it("missing temperature → no temp breach", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1 } } }); expect(r.breaches.temp).toBe(false); }); it("detects multiple simultaneous breaches", () => { const r = compute({ metrics: { cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {}, }, }); expect(r.breaches).toEqual({ cpu: true, memory: true, disk: true, temp: true }); }); // ── Counter mechanics ── it("decrements counter on breach", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 3, memory: 5, disk: 5, temp: 5 }, }); expect(r.nextCounters.cpu).toBe(2); }); it("resets counter to start value when metric returns to normal", () => { const r = compute({ counters: { cpu: 2, memory: 2, disk: 2, temp: 2 } }); expect(r.nextCounters).toEqual({ cpu: 5, memory: 5, disk: 5, temp: 5 }); }); it("counter floors at zero (does not go negative)", () => { const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 0, memory: 5, disk: 5, temp: 5 }, }); expect(r.nextCounters.cpu).toBe(0); }); it("counters are independent per metric", () => { // CPU breaching, memory normal: CPU decrements, memory resets. const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 3, memory: 2, disk: 2, temp: 2 }, }); expect(r.nextCounters).toEqual({ cpu: 2, memory: 5, disk: 5, temp: 5 }); }); // ── Status transitions ── it("up → breached when counter hits zero on breach", () => { const r = compute({ currentStatus: "up", metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 1, memory: 5, disk: 5, temp: 5 }, }); expect(r.nextStatus).toBe("breached"); expect(r.transitioned).toBe(true); }); it("up → up (no transition) while counter is still decrementing", () => { const r = compute({ currentStatus: "up", metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 3, memory: 5, disk: 5, temp: 5 }, }); expect(r.nextStatus).toBe("up"); expect(r.transitioned).toBe(false); }); it("breached → breached (no transition) when already breached and still breaching", () => { const r = compute({ currentStatus: "breached", metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 0, memory: 5, disk: 5, temp: 5 }, }); expect(r.transitioned).toBe(false); // nextStatus stays at currentStatus ("breached") since no transition fires expect(r.nextStatus).toBe("breached"); }); it("breached → up when all metrics return to normal", () => { const r = compute({ currentStatus: "breached" }); expect(r.nextStatus).toBe("up"); expect(r.transitioned).toBe(true); }); it("any one counter at zero is enough to trigger initial breach", () => { // memory counter at 1, memory currently breaching, cpu fine const r = compute({ currentStatus: "up", metrics: { ...healthyMetrics(), memory: { usage_percent: 0.9 } }, counters: { cpu: 5, memory: 1, disk: 5, temp: 5 }, }); expect(r.nextStatus).toBe("breached"); expect(r.transitioned).toBe(true); }); // ── Reachability-down gate ── it("reachabilityDown blocks status transition even when counter hits zero", () => { const r = compute({ currentStatus: "up", reachabilityDown: true, metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 1, memory: 5, disk: 5, temp: 5 }, }); expect(r.transitioned).toBe(false); expect(r.nextStatus).toBe("up"); // stays at currentStatus }); it("reachabilityDown still updates counters (the gate is only around status, not metrics math)", () => { const r = compute({ reachabilityDown: true, metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, counters: { cpu: 3, memory: 5, disk: 5, temp: 5 }, }); expect(r.nextCounters.cpu).toBe(2); expect(r.breaches.cpu).toBe(true); }); it("reachabilityDown blocks recovery from breached", () => { // Even with all metrics normal, if reachability says down, we can't recover. const r = compute({ currentStatus: "breached", reachabilityDown: true }); expect(r.transitioned).toBe(false); expect(r.nextStatus).toBe("breached"); }); // ── Percentage vs raw-degree thresholds ── it("CPU/memory/disk thresholds are percentages (compared against usage * 100)", () => { // threshold 50 means 50%, usage 0.6 = 60% → breach const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.6, temperature: [30] } }, thresholds: { cpu: 50, memory: 80, disk: 80, temp: 80 }, }); expect(r.breaches.cpu).toBe(true); }); it("temperature threshold is raw degrees (no /100 scaling)", () => { // threshold 75, temp 80 → breach const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [80] } }, thresholds: { cpu: 80, memory: 80, disk: 80, temp: 75 }, }); expect(r.breaches.temp).toBe(true); }); }); });