diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index f2e7ff5ac..6cf41e207 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -19,7 +19,14 @@ import type { import { AppError } from "@/utils/AppError.js"; import { ILogger } from "@/utils/logger.js"; import { IBufferService } from "./bufferService.js"; +import type { HardwareStatusMetrics } from "@/types/network.js"; const SERVICE_NAME = "StatusService"; +const MAX_RECENT_CHECKS = 25; +const HARDWARE_ALERT_COUNTER_START = 5; +const HARDWARE_METRIC_KEYS = ["cpu", "memory", "disk", "temp"] as const; +type HardwareMetricKey = (typeof HARDWARE_METRIC_KEYS)[number]; +type HardwareBreaches = Record; +type HardwareCounters = Record; export interface IStatusService { updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise; @@ -84,6 +91,136 @@ export class StatusService implements IStatusService { } } + private tryUpdateRunningStats = async (monitor: Monitor, statusResponse: MonitorStatusResponse) => { + const statsOk = await this.updateRunningStats(monitor, statusResponse); + if (!statsOk) { + this.logger.warn({ + service: SERVICE_NAME, + method: "updateMonitorStatus", + message: `Stats update failed for monitor ${monitor.id}`, + }); + } + }; + + private appendToWindow = (monitor: Monitor, status: boolean) => { + monitor.statusWindow = monitor.statusWindow || []; + monitor.statusWindow.push(status); + while (monitor.statusWindow.length > monitor.statusWindowSize) { + monitor.statusWindow.shift(); + } + }; + + private toCheckSnapshot = (check: Check): CheckSnapshot => { + return { + id: check.id, + status: check.status, + responseTime: check.responseTime, + timings: check.timings, + statusCode: check.statusCode, + message: check.message, + cpu: check.cpu, + memory: check.memory, + disk: check.disk, + host: check.host, + errors: check.errors, + capture: check.capture, + net: check.net, + accessibility: check.accessibility, + bestPractices: check.bestPractices, + seo: check.seo, + performance: check.performance, + audits: check.audits, + createdAt: check.createdAt, + }; + }; + + private appendToRecentChecks = (monitor: Monitor, check: Check) => { + monitor.recentChecks = monitor.recentChecks || []; + const checkSnapshot = this.toCheckSnapshot(check); + monitor.recentChecks.push(checkSnapshot); + while (monitor.recentChecks.length > MAX_RECENT_CHECKS) { + monitor.recentChecks.shift(); + } + }; + + private computeReachability = ( + currentStatus: MonitorStatus, + window: Array, + threshold: number + ): { nextStatus: "up" | "down"; transitioned: boolean } => { + const failures = window.filter((status) => status === false).length; + const failureRate = (failures / window.length) * 100; + + if (failureRate >= threshold && currentStatus !== "down") { + return { nextStatus: "down", transitioned: true }; + } + + if (failureRate < threshold && currentStatus === "down") { + return { nextStatus: "up", transitioned: true }; + } + return { nextStatus: "up", transitioned: false }; + }; + + private computeHardwareStatus = (params: { + currentStatus: MonitorStatus; + reachabilityDown: boolean; + metrics: HardwareStatusMetrics | undefined; + thresholds: { cpu: number; memory: number; disk: number; temp: number }; + counters: HardwareCounters; + }): { + nextStatus: MonitorStatus; + transitioned: boolean; + breaches: HardwareBreaches; + nextCounters: HardwareCounters; + } | null => { + const { metrics, thresholds, counters, currentStatus, reachabilityDown } = params; + if (!metrics) return null; + + const cpuUsage = metrics.cpu?.usage_percent ?? -1; + const memoryUsage = metrics.memory?.usage_percent ?? -1; + const temps = metrics.cpu?.temperature ?? []; + + const breaches: HardwareBreaches = { + cpu: cpuUsage !== -1 && cpuUsage > thresholds.cpu / 100, + memory: memoryUsage !== -1 && memoryUsage > thresholds.memory / 100, + disk: metrics.disk + ? metrics.disk.some((d: CheckDiskInfo) => d != null && typeof d.usage_percent === "number" && d.usage_percent > thresholds.disk / 100) + : false, + temp: temps.some((temp: number) => temp > thresholds.temp), + }; + + // Update counters: decrement (floored at 0) if breached, reset to start otherwise. + const nextCounters = { ...counters }; + for (const key of HARDWARE_METRIC_KEYS) { + nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START; + } + + // Status transition: reachability "down" takes precedence; hardware can only act when reachable. + let nextStatus: MonitorStatus = currentStatus; + let transitioned = false; + + if (!reachabilityDown) { + const anyCounterZero = HARDWARE_METRIC_KEYS.some((k) => nextCounters[k] === 0); + const anyBreached = HARDWARE_METRIC_KEYS.some((k) => breaches[k]); + const allNormal = !anyBreached; + + if (anyCounterZero && anyBreached && currentStatus !== "breached") { + // Initial breach: sustained N consecutive breaches, flip to 'breached'. + nextStatus = "breached"; + transitioned = true; + } else if (anyCounterZero && anyBreached && currentStatus === "breached") { + // Already breached; stay breached without marking as a transition. + nextStatus = "breached"; + } else if (allNormal && currentStatus === "breached") { + // All thresholds back to normal: recover from breached state. + nextStatus = "up"; + transitioned = true; + } + } + + return { nextStatus, transitioned, breaches, nextCounters }; + }; + updateMonitorStatus = async ( statusResponse: MonitorStatusResponse< | PingStatusPayload @@ -103,56 +240,17 @@ export class StatusService implements IStatusService { const monitor = await this.monitorsRepository.findById(monitorId, teamId); // Update running stats - const statsOk = await this.updateRunningStats(monitor, statusResponse); - if (!statsOk) { - this.logger.warn({ - service: SERVICE_NAME, - method: "updateMonitorStatus", - message: `Stats update failed for monitor ${monitor.id}`, - }); - } + await this.tryUpdateRunningStats(monitor, statusResponse); - monitor.statusWindow = monitor.statusWindow || []; - monitor.statusWindow.push(status); - while (monitor.statusWindow.length > monitor.statusWindowSize) { - monitor.statusWindow.shift(); - } - - const checkSnapshot: CheckSnapshot = { - id: check.id, - status: check.status, - responseTime: check.responseTime, - timings: check.timings, - statusCode: check.statusCode, - message: check.message, - cpu: check.cpu, - memory: check.memory, - disk: check.disk, - host: check.host, - errors: check.errors, - capture: check.capture, - net: check.net, - accessibility: check.accessibility, - bestPractices: check.bestPractices, - seo: check.seo, - performance: check.performance, - audits: check.audits, - createdAt: check.createdAt, - }; - monitor.recentChecks = monitor.recentChecks || []; - monitor.recentChecks.push(checkSnapshot); - const maxRecentChecks = 25; - while (monitor.recentChecks.length > maxRecentChecks) { - monitor.recentChecks.shift(); - } + // Update the sliding window and recent checks + this.appendToWindow(monitor, status); + this.appendToRecentChecks(monitor, check); const prevStatus = monitor.status; - let newStatus: MonitorStatus = status === true ? "up" : "down"; - let statusChanged = false; // Return early if not enough data points if (monitor.statusWindow.length < monitor.statusWindowSize) { - monitor.status = newStatus; + monitor.status = status === true ? "up" : "down"; const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor); return { monitor: updated, @@ -164,102 +262,47 @@ export class StatusService implements IStatusService { } // With a full window, a single raw check must not change monitor.status; only the sliding-window threshold can trigger a transition. - newStatus = monitor.status; + let newStatus = monitor.status; + let statusChanged = false; - // Check if threshold has been met - const failures = monitor.statusWindow.filter((s) => s === false).length; - const failureRate = (failures / monitor.statusWindow.length) * 100; - - // If threshold has been met and the monitor is not already down, mark down: - if (failureRate >= monitor.statusWindowThreshold && monitor.status !== "down") { - newStatus = "down"; - statusChanged = true; - } - // If the failure rate is below the threshold and the monitor is down, recover: - else if (failureRate < monitor.statusWindowThreshold && monitor.status === "down") { - newStatus = "up"; + // First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches. + const reachabilityResult = this.computeReachability(monitor.status, monitor.statusWindow, monitor.statusWindowThreshold); + if (reachabilityResult.transitioned) { + newStatus = reachabilityResult.nextStatus; statusChanged = true; } - // Evaluate hardware threshold breaches (only for hardware monitors) - let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined; + // Evaluate hardware threshold breaches + let thresholdBreaches: HardwareBreaches | undefined; if (monitor.type === "hardware" && statusResponse.payload) { const payload = statusResponse.payload as HardwareStatusPayload; - const metrics = payload.data; + const hardware = this.computeHardwareStatus({ + currentStatus: monitor.status, + reachabilityDown: newStatus === "down", + metrics: payload.data, + thresholds: { + cpu: monitor.cpuAlertThreshold, + memory: monitor.memoryAlertThreshold, + disk: monitor.diskAlertThreshold, + temp: monitor.tempAlertThreshold, + }, + counters: { + cpu: monitor.cpuAlertCounter, + memory: monitor.memoryAlertCounter, + disk: monitor.diskAlertCounter, + temp: monitor.tempAlertCounter, + }, + }); - if (metrics) { - // Evaluate threshold breaches - const cpuUsage = metrics.cpu?.usage_percent ?? -1; - const cpuBreach = cpuUsage !== -1 && cpuUsage > monitor.cpuAlertThreshold / 100; - - const memoryUsage = metrics.memory?.usage_percent ?? -1; - const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100; - - const diskBreach = metrics.disk - ? metrics.disk.some( - (d: CheckDiskInfo) => d != null && typeof d.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100 - ) - : false; - - const temps = metrics.cpu?.temperature ?? []; - const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold); - - thresholdBreaches = { - cpu: cpuBreach, - memory: memoryBreach, - disk: diskBreach, - temp: tempBreach, - }; - - // Update counters: decrement if breached, reset to 5 if not breached - if (cpuBreach) { - monitor.cpuAlertCounter = Math.max(0, monitor.cpuAlertCounter - 1); - } else { - monitor.cpuAlertCounter = 5; - } - - if (memoryBreach) { - monitor.memoryAlertCounter = Math.max(0, monitor.memoryAlertCounter - 1); - } else { - monitor.memoryAlertCounter = 5; - } - - if (diskBreach) { - monitor.diskAlertCounter = Math.max(0, monitor.diskAlertCounter - 1); - } else { - monitor.diskAlertCounter = 5; - } - - if (tempBreach) { - monitor.tempAlertCounter = Math.max(0, monitor.tempAlertCounter - 1); - } else { - monitor.tempAlertCounter = 5; - } - - // Check if any counter has reached zero (initial breach) - const anyCounterZero = - monitor.cpuAlertCounter === 0 || monitor.memoryAlertCounter === 0 || monitor.diskAlertCounter === 0 || monitor.tempAlertCounter === 0; - - const anyThresholdBreached = cpuBreach || memoryBreach || diskBreach || tempBreach; - const allThresholdsNormal = !cpuBreach && !memoryBreach && !diskBreach && !tempBreach; - - // Update monitor status based on threshold breach state - if (newStatus !== "down") { - // Don't override "down" status - service unreachable takes precedence - // Check current monitor status, not newStatus for comparison - if (anyCounterZero && anyThresholdBreached && monitor.status !== "breached") { - // Initial breach: counter hit zero, change status to breached - newStatus = "breached"; - statusChanged = true; - } else if (anyCounterZero && anyThresholdBreached && monitor.status === "breached") { - // Already breached, keep status but don't mark as changed - newStatus = "breached"; - // statusChanged remains false - } else if (allThresholdsNormal && monitor.status === "breached") { - // All thresholds returned to normal, recover from breached state - newStatus = "up"; - statusChanged = true; - } + if (hardware) { + monitor.cpuAlertCounter = hardware.nextCounters.cpu; + monitor.memoryAlertCounter = hardware.nextCounters.memory; + monitor.diskAlertCounter = hardware.nextCounters.disk; + monitor.tempAlertCounter = hardware.nextCounters.temp; + thresholdBreaches = hardware.breaches; + if (hardware.transitioned) { + newStatus = hardware.nextStatus; + statusChanged = true; } } }