diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index 4b591694d..00fd3b3b4 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -19,7 +19,14 @@ import type { import { AppError } from "@/utils/AppError.js"; import { ILogger } from "@/utils/logger.js"; import { IBufferService } from "./bufferService.js"; +import type { HardwareStatusMetrics } from "@/types/network.js"; const SERVICE_NAME = "StatusService"; +const MAX_RECENT_CHECKS = 25; +const HARDWARE_ALERT_COUNTER_START = 5; +const HARDWARE_METRIC_KEYS = ["cpu", "memory", "disk", "temp"] as const; +type HardwareMetricKey = (typeof HARDWARE_METRIC_KEYS)[number]; +type HardwareBreaches = Record; +type HardwareCounters = Record; export interface IStatusService { updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise; @@ -84,6 +91,132 @@ export class StatusService implements IStatusService { } } + private tryUpdateRunningStats = async (monitor: Monitor, statusResponse: MonitorStatusResponse) => { + const statsOk = await this.updateRunningStats(monitor, statusResponse); + if (!statsOk) { + this.logger.warn({ + service: SERVICE_NAME, + method: "updateMonitorStatus", + message: `Stats update failed for monitor ${monitor.id}`, + }); + } + }; + + private appendToWindow = (monitor: Monitor, status: boolean) => { + monitor.statusWindow = monitor.statusWindow || []; + monitor.statusWindow.push(status); + while (monitor.statusWindow.length > monitor.statusWindowSize) { + monitor.statusWindow.shift(); + } + }; + + private toCheckSnapshot = (check: Check): CheckSnapshot => { + return { + id: check.id, + status: check.status, + responseTime: check.responseTime, + timings: check.timings, + statusCode: check.statusCode, + message: check.message, + cpu: check.cpu, + memory: check.memory, + disk: check.disk, + host: check.host, + errors: check.errors, + capture: check.capture, + net: check.net, + accessibility: check.accessibility, + bestPractices: check.bestPractices, + seo: check.seo, + performance: check.performance, + audits: check.audits, + createdAt: check.createdAt, + }; + }; + + private appendToRecentChecks = (monitor: Monitor, check: Check) => { + monitor.recentChecks = monitor.recentChecks || []; + const checkSnapshot = this.toCheckSnapshot(check); + monitor.recentChecks.push(checkSnapshot); + while (monitor.recentChecks.length > MAX_RECENT_CHECKS) { + monitor.recentChecks.shift(); + } + }; + + private computeReachability = ( + currentStatus: MonitorStatus, + window: Array, + threshold: number + ): { nextStatus: "up" | "down"; transitioned: boolean } => { + const failures = window.filter((status) => status === false).length; + const failureRate = (failures / window.length) * 100; + + if (failureRate >= threshold && currentStatus !== "down") { + return { nextStatus: "down", transitioned: true }; + } + + if (failureRate < threshold && currentStatus === "down") { + return { nextStatus: "up", transitioned: true }; + } + return { nextStatus: "up", transitioned: false }; + }; + + private computeHardwareStatus = (params: { + currentStatus: MonitorStatus; + reachabilityDown: boolean; + metrics: HardwareStatusMetrics; + thresholds: { cpu: number; memory: number; disk: number; temp: number }; + counters: HardwareCounters; + }): { + nextStatus: MonitorStatus; + transitioned: boolean; + breaches: HardwareBreaches; + nextCounters: HardwareCounters; + } => { + const { metrics, thresholds, counters, currentStatus, reachabilityDown } = params; + + const cpuUsage = metrics.cpu?.usage_percent ?? -1; + const memoryUsage = metrics.memory?.usage_percent ?? -1; + const temps = metrics.cpu?.temperature ?? []; + + const breaches: HardwareBreaches = { + cpu: cpuUsage !== -1 && cpuUsage > thresholds.cpu / 100, + memory: memoryUsage !== -1 && memoryUsage > thresholds.memory / 100, + disk: metrics.disk + ? metrics.disk.some((d: CheckDiskInfo) => d != null && typeof d.usage_percent === "number" && d.usage_percent > thresholds.disk / 100) + : false, + temp: temps.some((temp: number) => temp > thresholds.temp), + }; + + // Update counters: decrement (floored at 0) if breached, reset to start otherwise. + const nextCounters = { ...counters }; + for (const key of HARDWARE_METRIC_KEYS) { + nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START; + } + + // Status transition: reachability "down" takes precedence; hardware can only drive + // transitions into "breached" and back out to "up". Any other case leaves status untouched. + let nextStatus: MonitorStatus = currentStatus; + let transitioned = false; + + if (!reachabilityDown) { + // A counter can only reach zero via the decrement path, which only runs when that + // metric is currently breaching — so anyCounterZero already implies anyBreached. + const anyCounterZero = HARDWARE_METRIC_KEYS.some((k) => nextCounters[k] === 0); + const allNormal = HARDWARE_METRIC_KEYS.every((k) => !breaches[k]); + + if (anyCounterZero && currentStatus !== "breached") { + nextStatus = "breached"; + transitioned = true; + } else if (allNormal && currentStatus === "breached") { + nextStatus = "up"; + transitioned = true; + } + } + + return { nextStatus, transitioned, breaches, nextCounters }; + }; + updateMonitorStatus = async ( statusResponse: MonitorStatusResponse< | PingStatusPayload @@ -103,56 +236,17 @@ export class StatusService implements IStatusService { const monitor = await this.monitorsRepository.findById(monitorId, teamId); // Update running stats - const statsOk = await this.updateRunningStats(monitor, statusResponse); - if (!statsOk) { - this.logger.warn({ - service: SERVICE_NAME, - method: "updateMonitorStatus", - message: `Stats update failed for monitor ${monitor.id}`, - }); - } + await this.tryUpdateRunningStats(monitor, statusResponse); - monitor.statusWindow = monitor.statusWindow || []; - monitor.statusWindow.push(status); - while (monitor.statusWindow.length > monitor.statusWindowSize) { - monitor.statusWindow.shift(); - } - - const checkSnapshot: CheckSnapshot = { - id: check.id, - status: check.status, - responseTime: check.responseTime, - timings: check.timings, - statusCode: check.statusCode, - message: check.message, - cpu: check.cpu, - memory: check.memory, - disk: check.disk, - host: check.host, - errors: check.errors, - capture: check.capture, - net: check.net, - accessibility: check.accessibility, - bestPractices: check.bestPractices, - seo: check.seo, - performance: check.performance, - audits: check.audits, - createdAt: check.createdAt, - }; - monitor.recentChecks = monitor.recentChecks || []; - monitor.recentChecks.push(checkSnapshot); - const maxRecentChecks = 25; - while (monitor.recentChecks.length > maxRecentChecks) { - monitor.recentChecks.shift(); - } + // Update the sliding window and recent checks + this.appendToWindow(monitor, status); + this.appendToRecentChecks(monitor, check); const prevStatus = monitor.status; - let newStatus: MonitorStatus = status === true ? "up" : "down"; - let statusChanged = false; // Return early if not enough data points if (monitor.statusWindow.length < monitor.statusWindowSize) { - monitor.status = newStatus; + monitor.status = status === true ? "up" : "down"; const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor); return { monitor: updated, @@ -163,101 +257,47 @@ export class StatusService implements IStatusService { }; } - // Check if threshold has been met - const failures = monitor.statusWindow.filter((s) => s === false).length; - const failureRate = (failures / monitor.statusWindow.length) * 100; + // With a full window, a single raw check must not change monitor.status; only the sliding-window threshold can trigger a transition. + let newStatus = monitor.status; + let statusChanged = false; - // If threshold has been met and the monitor is not already down, mark down: - if (failureRate >= monitor.statusWindowThreshold && monitor.status !== "down") { - newStatus = "down"; - statusChanged = true; - } - // If the failure rate is below the threshold and the monitor is down, recover: - else if (failureRate < monitor.statusWindowThreshold && monitor.status === "down") { - newStatus = "up"; + // First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches. + const reachabilityResult = this.computeReachability(monitor.status, monitor.statusWindow, monitor.statusWindowThreshold); + if (reachabilityResult.transitioned) { + newStatus = reachabilityResult.nextStatus; statusChanged = true; } - // Evaluate hardware threshold breaches (only for hardware monitors) - let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined; - if (monitor.type === "hardware" && statusResponse.payload) { - const payload = statusResponse.payload as HardwareStatusPayload; - const metrics = payload.data; + // Evaluate hardware threshold breaches (only for hardware monitors with metrics payload) + let thresholdBreaches: HardwareBreaches | undefined; + const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined; + if (monitor.type === "hardware" && hardwarePayload?.data) { + const hardware = this.computeHardwareStatus({ + currentStatus: monitor.status, + reachabilityDown: newStatus === "down", + metrics: hardwarePayload.data, + thresholds: { + cpu: monitor.cpuAlertThreshold, + memory: monitor.memoryAlertThreshold, + disk: monitor.diskAlertThreshold, + temp: monitor.tempAlertThreshold, + }, + counters: { + cpu: monitor.cpuAlertCounter, + memory: monitor.memoryAlertCounter, + disk: monitor.diskAlertCounter, + temp: monitor.tempAlertCounter, + }, + }); - if (metrics) { - // Evaluate threshold breaches - const cpuUsage = metrics.cpu?.usage_percent ?? -1; - const cpuBreach = cpuUsage !== -1 && cpuUsage > monitor.cpuAlertThreshold / 100; - - const memoryUsage = metrics.memory?.usage_percent ?? -1; - const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100; - - const diskBreach = metrics.disk - ? metrics.disk.some( - (d: CheckDiskInfo) => d != null && typeof d.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100 - ) - : false; - - const temps = metrics.cpu?.temperature ?? []; - const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold); - - thresholdBreaches = { - cpu: cpuBreach, - memory: memoryBreach, - disk: diskBreach, - temp: tempBreach, - }; - - // Update counters: decrement if breached, reset to 5 if not breached - if (cpuBreach) { - monitor.cpuAlertCounter = Math.max(0, monitor.cpuAlertCounter - 1); - } else { - monitor.cpuAlertCounter = 5; - } - - if (memoryBreach) { - monitor.memoryAlertCounter = Math.max(0, monitor.memoryAlertCounter - 1); - } else { - monitor.memoryAlertCounter = 5; - } - - if (diskBreach) { - monitor.diskAlertCounter = Math.max(0, monitor.diskAlertCounter - 1); - } else { - monitor.diskAlertCounter = 5; - } - - if (tempBreach) { - monitor.tempAlertCounter = Math.max(0, monitor.tempAlertCounter - 1); - } else { - monitor.tempAlertCounter = 5; - } - - // Check if any counter has reached zero (initial breach) - const anyCounterZero = - monitor.cpuAlertCounter === 0 || monitor.memoryAlertCounter === 0 || monitor.diskAlertCounter === 0 || monitor.tempAlertCounter === 0; - - const anyThresholdBreached = cpuBreach || memoryBreach || diskBreach || tempBreach; - const allThresholdsNormal = !cpuBreach && !memoryBreach && !diskBreach && !tempBreach; - - // Update monitor status based on threshold breach state - if (newStatus !== "down") { - // Don't override "down" status - service unreachable takes precedence - // Check current monitor status, not newStatus for comparison - if (anyCounterZero && anyThresholdBreached && monitor.status !== "breached") { - // Initial breach: counter hit zero, change status to breached - newStatus = "breached"; - statusChanged = true; - } else if (anyCounterZero && anyThresholdBreached && monitor.status === "breached") { - // Already breached, keep status but don't mark as changed - newStatus = "breached"; - // statusChanged remains false - } else if (allThresholdsNormal && monitor.status === "breached") { - // All thresholds returned to normal, recover from breached state - newStatus = "up"; - statusChanged = true; - } - } + monitor.cpuAlertCounter = hardware.nextCounters.cpu; + monitor.memoryAlertCounter = hardware.nextCounters.memory; + monitor.diskAlertCounter = hardware.nextCounters.disk; + monitor.tempAlertCounter = hardware.nextCounters.temp; + thresholdBreaches = hardware.breaches; + if (hardware.transitioned) { + newStatus = hardware.nextStatus; + statusChanged = true; } } diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index fb2e4ad87..ae8db0924 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, jest, beforeEach } from "@jest/globals"; import { StatusService } from "../../../src/service/infrastructure/statusService.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; -import type { Monitor, MonitorStatusResponse, Check, HardwareStatusPayload } from "../../../src/types/index.ts"; +import type { Monitor, MonitorStatus, MonitorStatusResponse, Check, HardwareStatusPayload } from "../../../src/types/index.ts"; import type { IMonitorsRepository, IMonitorStatsRepository, IChecksRepository } from "../../../src/repositories/index.ts"; import type { IBufferService } from "../../../src/service/infrastructure/bufferService.ts"; @@ -379,6 +379,79 @@ describe("StatusService", () => { await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Unknown error"); }); + it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => { + // Monitor is down, window is [false,false,false,false]. A single successful check makes it + // [false,false,false,false,true] → 4/5 = 80%, still at threshold, must NOT silently recover. + // Pre-fix, newStatus was seeded "up" from the raw check and fell through to be persisted, + // causing the next failing check to fire a spurious second 'Down' notification. + const monitor = makeMonitor({ + statusWindow: [false, false, false, false], + statusWindowSize: 5, + statusWindowThreshold: 80, + status: "down", + }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("down"); + }); + + it("triggers 'down' transition exactly at the failure threshold boundary", async () => { + // 4/5 = 80%, threshold is 80, comparison is >= so this must trip. + const monitor = makeMonitor({ + statusWindow: [false, false, false, true], + statusWindowSize: 5, + statusWindowThreshold: 80, + status: "up", + }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("down"); + }); + + it("during warmup (window not full) the stored status tracks the raw check", async () => { + // Early-return branch: monitor.status is written to reflect the current check and + // no status-changed notification is emitted. This is the service's documented warmup behavior. + const monitor = makeMonitor({ statusWindow: [true, true], statusWindowSize: 5, status: "up" }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("down"); + expect(monitor.statusWindow).toEqual([true, true, false]); + }); + + it("logs a warning but still succeeds when running-stats update fails mid-flight", async () => { + const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" }); + const { service, logger, monitorsRepository, monitorStatsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down")); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck()); + + expect(result.monitor.status).toBe("up"); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + service: "StatusService", + method: "updateMonitorStatus", + message: expect.stringContaining("Stats update failed"), + }) + ); + }); + it("returns code and timestamp in result", async () => { const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5 }); const { service, monitorsRepository } = createService(); @@ -718,6 +791,46 @@ describe("StatusService", () => { expect(result.thresholdBreaches?.disk).toBe(false); }); + it("decrements breach counter without changing status while counter > 0", async () => { + // Counter at 3 → decrements to 2, status must stay 'up' and statusChanged must be false. + // Only counter→0 may flip to 'breached'. + const monitor = makeHardwareMonitor({ cpuAlertCounter: 3 }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + + const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); + const result = await service.updateMonitorStatus(response, makeCheck()); + + expect(monitor.cpuAlertCounter).toBe(2); + expect(result.thresholdBreaches?.cpu).toBe(true); + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("transitions from 'breached' to 'down' when the reachability threshold trips", async () => { + // A hardware monitor currently in 'breached' state starts failing its reachability + // checks enough to cross the statusWindow threshold — 'down' must take precedence. + const monitor = makeHardwareMonitor({ + status: "breached", + statusWindow: [false, false, false, false], + statusWindowSize: 5, + statusWindowThreshold: 80, + }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + + const response = makeHardwareResponse({ + status: false, + data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, + } as any); + const result = await service.updateMonitorStatus(response, makeCheck({ status: false })); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("down"); + }); + it("counters do not go below 0", async () => { const monitor = makeHardwareMonitor({ cpuAlertCounter: 0, @@ -741,4 +854,301 @@ describe("StatusService", () => { }); }); }); + + // ── Pure helpers ───────────────────────────────────────────────────────── + // These test the private state-machine helpers directly, without repository + // or logger mocks. The helpers are accessed via `service as any` because + // they're private implementation details; exposing them publicly would + // widen the API surface just for testability. + + describe("computeReachability (pure)", () => { + const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => { + const { service } = createService(); + return (service as any).computeReachability(currentStatus, window, threshold) as { + nextStatus: "up" | "down"; + transitioned: boolean; + }; + }; + + // ── Monitor is "up" ── + it("up + window below threshold → no transition", () => { + expect(reach("up", [true, true, true, true, true])).toEqual({ nextStatus: "up", transitioned: false }); + }); + + it("up + window exactly at threshold → transitions to down (>= boundary)", () => { + // 4/5 = 80% and threshold is 80, >= so this must trip + expect(reach("up", [false, false, false, false, true])).toEqual({ nextStatus: "down", transitioned: true }); + }); + + it("up + window over threshold → transitions to down", () => { + expect(reach("up", [false, false, false, false, false])).toEqual({ nextStatus: "down", transitioned: true }); + }); + + // ── Monitor is "down" ── + it("down + window below threshold → transitions to up (recovery)", () => { + // 0/5 failures, fully healthy window + expect(reach("down", [true, true, true, true, true])).toEqual({ nextStatus: "up", transitioned: true }); + }); + + it("down + window exactly at threshold → no transition (strict < for recovery)", () => { + // 4/5 = 80%; recovery requires failureRate < threshold, not <=, so stays down + const result = reach("down", [false, false, false, false, true]); + expect(result.transitioned).toBe(false); + // Orchestrator ignores nextStatus when !transitioned, so the returned value + // is irrelevant to observable behavior; documenting here for clarity. + }); + + it("down + window over threshold → no transition", () => { + const result = reach("down", [false, false, false, false, false]); + expect(result.transitioned).toBe(false); + }); + + // ── Monitor is "breached" (hardware state, passes through reachability) ── + it("breached + healthy window → no transition (reachability does not touch breached)", () => { + // Critical: reachability must NOT claim to recover a breached monitor. That + // transition is owned by the hardware block (all thresholds back to normal). + const result = reach("breached", [true, true, true, true, true]); + expect(result.transitioned).toBe(false); + }); + + it("breached + window at threshold → transitions to down (down beats breached)", () => { + // A breached monitor that also loses reachability must flip to "down". + // Current status is "breached" which satisfies `!== "down"`, so the branch fires. + expect(reach("breached", [false, false, false, false, true])).toEqual({ nextStatus: "down", transitioned: true }); + }); + + it("breached + window over threshold → transitions to down", () => { + expect(reach("breached", [false, false, false, false, false])).toEqual({ nextStatus: "down", transitioned: true }); + }); + + // ── Threshold variations ── + it("non-default threshold (50%) trips at 3/5 failures", () => { + expect(reach("up", [false, false, false, true, true], 50)).toEqual({ nextStatus: "down", transitioned: true }); + }); + + it("non-default threshold (50%) does not trip at 2/5 failures", () => { + expect(reach("up", [false, false, true, true, true], 50).transitioned).toBe(false); + }); + }); + + describe("computeHardwareStatus (pure)", () => { + type Params = { + currentStatus: MonitorStatus; + reachabilityDown: boolean; + metrics: any; + thresholds: { cpu: number; memory: number; disk: number; temp: number }; + counters: { cpu: number; memory: number; disk: number; temp: number }; + }; + + const healthyMetrics = () => ({ + cpu: { usage_percent: 0.1, temperature: [30] }, + memory: { usage_percent: 0.1 }, + disk: [{ usage_percent: 0.1 }], + host: {}, + }); + + const defaults = (): Params => ({ + currentStatus: "up", + reachabilityDown: false, + metrics: healthyMetrics(), + thresholds: { cpu: 80, memory: 80, disk: 80, temp: 80 }, + counters: { cpu: 5, memory: 5, disk: 5, temp: 5 }, + }); + + const compute = (overrides: Partial = {}) => { + const { service } = createService(); + return (service as any).computeHardwareStatus({ ...defaults(), ...overrides }); + }; + + // ── Breach detection ── + it("detects CPU breach when usage > threshold", () => { + const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } } }); + expect(r.breaches.cpu).toBe(true); + }); + + it("does not flag CPU at exactly the threshold (strict >, not >=)", () => { + const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.8, temperature: [30] } } }); + expect(r.breaches.cpu).toBe(false); + }); + + it("does not flag CPU when usage_percent is missing", () => { + const r = compute({ metrics: { ...healthyMetrics(), cpu: { temperature: [30] } } }); + expect(r.breaches.cpu).toBe(false); + }); + + it("detects memory breach", () => { + const r = compute({ metrics: { ...healthyMetrics(), memory: { usage_percent: 0.9 } } }); + expect(r.breaches.memory).toBe(true); + }); + + it("detects disk breach when any disk exceeds threshold", () => { + const r = compute({ metrics: { ...healthyMetrics(), disk: [{ usage_percent: 0.1 }, { usage_percent: 0.95 }] } }); + expect(r.breaches.disk).toBe(true); + }); + + it("skips null disk entries", () => { + const r = compute({ metrics: { ...healthyMetrics(), disk: [null, { usage_percent: 0.1 }] } }); + expect(r.breaches.disk).toBe(false); + }); + + it("skips disk entries missing usage_percent", () => { + const r = compute({ metrics: { ...healthyMetrics(), disk: [{ device: "/dev/sda" }] } }); + expect(r.breaches.disk).toBe(false); + }); + + it("detects temperature breach when any core exceeds threshold", () => { + const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [30, 50, 90] } } }); + expect(r.breaches.temp).toBe(true); + }); + + it("empty temperature array → no temp breach", () => { + const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [] } } }); + expect(r.breaches.temp).toBe(false); + }); + + it("missing temperature → no temp breach", () => { + const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1 } } }); + expect(r.breaches.temp).toBe(false); + }); + + it("detects multiple simultaneous breaches", () => { + const r = compute({ + metrics: { + cpu: { usage_percent: 0.9, temperature: [90] }, + memory: { usage_percent: 0.9 }, + disk: [{ usage_percent: 0.95 }], + host: {}, + }, + }); + expect(r.breaches).toEqual({ cpu: true, memory: true, disk: true, temp: true }); + }); + + // ── Counter mechanics ── + it("decrements counter on breach", () => { + const r = compute({ + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 3, memory: 5, disk: 5, temp: 5 }, + }); + expect(r.nextCounters.cpu).toBe(2); + }); + + it("resets counter to start value when metric returns to normal", () => { + const r = compute({ counters: { cpu: 2, memory: 2, disk: 2, temp: 2 } }); + expect(r.nextCounters).toEqual({ cpu: 5, memory: 5, disk: 5, temp: 5 }); + }); + + it("counter floors at zero (does not go negative)", () => { + const r = compute({ + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 0, memory: 5, disk: 5, temp: 5 }, + }); + expect(r.nextCounters.cpu).toBe(0); + }); + + it("counters are independent per metric", () => { + // CPU breaching, memory normal: CPU decrements, memory resets. + const r = compute({ + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 3, memory: 2, disk: 2, temp: 2 }, + }); + expect(r.nextCounters).toEqual({ cpu: 2, memory: 5, disk: 5, temp: 5 }); + }); + + // ── Status transitions ── + it("up → breached when counter hits zero on breach", () => { + const r = compute({ + currentStatus: "up", + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 1, memory: 5, disk: 5, temp: 5 }, + }); + expect(r.nextStatus).toBe("breached"); + expect(r.transitioned).toBe(true); + }); + + it("up → up (no transition) while counter is still decrementing", () => { + const r = compute({ + currentStatus: "up", + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 3, memory: 5, disk: 5, temp: 5 }, + }); + expect(r.nextStatus).toBe("up"); + expect(r.transitioned).toBe(false); + }); + + it("breached → breached (no transition) when already breached and still breaching", () => { + const r = compute({ + currentStatus: "breached", + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 0, memory: 5, disk: 5, temp: 5 }, + }); + expect(r.transitioned).toBe(false); + // nextStatus stays at currentStatus ("breached") since no transition fires + expect(r.nextStatus).toBe("breached"); + }); + + it("breached → up when all metrics return to normal", () => { + const r = compute({ currentStatus: "breached" }); + expect(r.nextStatus).toBe("up"); + expect(r.transitioned).toBe(true); + }); + + it("any one counter at zero is enough to trigger initial breach", () => { + // memory counter at 1, memory currently breaching, cpu fine + const r = compute({ + currentStatus: "up", + metrics: { ...healthyMetrics(), memory: { usage_percent: 0.9 } }, + counters: { cpu: 5, memory: 1, disk: 5, temp: 5 }, + }); + expect(r.nextStatus).toBe("breached"); + expect(r.transitioned).toBe(true); + }); + + // ── Reachability-down gate ── + it("reachabilityDown blocks status transition even when counter hits zero", () => { + const r = compute({ + currentStatus: "up", + reachabilityDown: true, + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 1, memory: 5, disk: 5, temp: 5 }, + }); + expect(r.transitioned).toBe(false); + expect(r.nextStatus).toBe("up"); // stays at currentStatus + }); + + it("reachabilityDown still updates counters (the gate is only around status, not metrics math)", () => { + const r = compute({ + reachabilityDown: true, + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } }, + counters: { cpu: 3, memory: 5, disk: 5, temp: 5 }, + }); + expect(r.nextCounters.cpu).toBe(2); + expect(r.breaches.cpu).toBe(true); + }); + + it("reachabilityDown blocks recovery from breached", () => { + // Even with all metrics normal, if reachability says down, we can't recover. + const r = compute({ currentStatus: "breached", reachabilityDown: true }); + expect(r.transitioned).toBe(false); + expect(r.nextStatus).toBe("breached"); + }); + + // ── Percentage vs raw-degree thresholds ── + it("CPU/memory/disk thresholds are percentages (compared against usage * 100)", () => { + // threshold 50 means 50%, usage 0.6 = 60% → breach + const r = compute({ + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.6, temperature: [30] } }, + thresholds: { cpu: 50, memory: 80, disk: 80, temp: 80 }, + }); + expect(r.breaches.cpu).toBe(true); + }); + + it("temperature threshold is raw degrees (no /100 scaling)", () => { + // threshold 75, temp 80 → breach + const r = compute({ + metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [80] } }, + thresholds: { cpu: 80, memory: 80, disk: 80, temp: 75 }, + }); + expect(r.breaches.temp).toBe(true); + }); + }); });