diff --git a/client/src/Components/v2/design-elements/StatusLabel.tsx b/client/src/Components/v2/design-elements/StatusLabel.tsx index cf404421e..19bb8d157 100644 --- a/client/src/Components/v2/design-elements/StatusLabel.tsx +++ b/client/src/Components/v2/design-elements/StatusLabel.tsx @@ -21,6 +21,8 @@ export const StatusLabel = ({ status, sx }: { status: MonitorStatus; sx?: SxProp return t("pages.common.monitors.status.up"); } else if (status === "down") { return t("pages.common.monitors.status.down"); + } else if (status === "breached") { + return t("pages.common.monitors.status.breached"); } else if (status === "maintenance") { return t("pages.common.monitors.status.maintenance"); } else if (status === "paused") { diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index 0c7cea5bd..cf637c184 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -18,6 +18,7 @@ export const MonitorStatuses = [ "paused", "initializing", "maintenance", + "breached", ] as const; export type MonitorStatus = (typeof MonitorStatuses)[number]; diff --git a/client/src/Utils/MonitorUtils.ts b/client/src/Utils/MonitorUtils.ts index 02c291efc..b73fc5bc4 100644 --- a/client/src/Utils/MonitorUtils.ts +++ b/client/src/Utils/MonitorUtils.ts @@ -23,6 +23,9 @@ export const getStatusPalette = (status: MonitorStatus): PaletteKey => { if (status === "down") { return "error"; } + if (status === "breached") { + return "error"; + } return "warning"; }; diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 8e18d8e2a..d8a46fb7c 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -387,6 +387,7 @@ }, "status": { "down": "down", + "breached": "breached", "initializing": "initializing", "maintenance": "maintenance", "paused": "paused", diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index 7e4c94586..4082368a7 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -187,7 +187,7 @@ class SuperSimpleQueueHelper { } private evaluateMonitorAction(statusChangeResult: StatusChangeResult): MonitorActionDecision { - const { monitor, statusChanged, prevStatus, thresholdBreaches } = statusChangeResult; + const { monitor, statusChanged, prevStatus } = statusChangeResult; // Initialize result const decision: MonitorActionDecision = { @@ -198,75 +198,28 @@ class SuperSimpleQueueHelper { notificationReason: null, }; - // CASE 1: Non-hardware - if (monitor.type !== "hardware") { - if (statusChanged) { - // This is the current state of the monitor AFTER the most recent check - if (monitor.status === "down") { - decision.shouldCreateIncident = true; - decision.shouldSendNotification = true; - decision.incidentReason = "status_down"; - decision.notificationReason = "status_change"; - } else if (monitor.status === "up" && prevStatus === "down") { - decision.shouldResolveIncident = true; - decision.shouldSendNotification = true; - decision.notificationReason = "status_change"; - } - } + // Simplified logic: Just check status changes + if (!statusChanged) { return decision; } - // CASE 2: Hardware - if (!thresholdBreaches) { - // No breach data available - still handle status changes - if (statusChanged && monitor.status === "down") { - decision.shouldCreateIncident = true; - decision.shouldSendNotification = true; - decision.incidentReason = "status_down"; - decision.notificationReason = "status_change"; - } else if (statusChanged && monitor.status === "up" && prevStatus === "down") { - decision.shouldResolveIncident = true; - decision.shouldSendNotification = true; - decision.notificationReason = "status_change"; - } - return decision; - } - - const anyThresholdBreached = Object.values(thresholdBreaches).some((b) => b); - const noThresholdsBreached = Object.values(thresholdBreaches).every((b) => !b); - - // Check if countdown counter has reached zero for any breached threshold - const shouldNotifyThreshold = - (thresholdBreaches.cpu && monitor.cpuAlertCounter <= 0) || - (thresholdBreaches.memory && monitor.memoryAlertCounter <= 0) || - (thresholdBreaches.disk && monitor.diskAlertCounter <= 0) || - (thresholdBreaches.temp && monitor.tempAlertCounter <= 0); - - // Decision logic for hardware - if (statusChanged && monitor.status === "down") { - // Hardware service is DOWN (unreachable) - // Status down takes precedence over threshold breaches + if (monitor.status === "down") { + // Monitor went down (unreachable) decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "status_down"; decision.notificationReason = "status_change"; - } else if (statusChanged && monitor.status === "up" && prevStatus === "down") { - // Hardware service recovered - decision.shouldResolveIncident = true; - decision.shouldSendNotification = true; - decision.notificationReason = "status_change"; - } else if (anyThresholdBreached && shouldNotifyThreshold) { - // Threshold breach: Counters have reached zero - // Create incident if service is UP but metrics exceed thresholds + } else if (monitor.status === "breached") { + // Hardware monitor exceeded thresholds decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; decision.notificationReason = "threshold_breach"; - decision.thresholdBreaches = thresholdBreaches; - } else if (noThresholdsBreached && !anyThresholdBreached) { - // All thresholds returned to normal - resolve any active threshold incidents + } else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) { + // Monitor recovered from down or breached state decision.shouldResolveIncident = true; - // Don't send notification for resolution (can be added later if needed) + decision.shouldSendNotification = true; + decision.notificationReason = "status_change"; } return decision; diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index cf9e1945a..b01131bbe 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -252,8 +252,6 @@ export class StatusService implements IStatusService { statusChanged = true; } - monitor.status = newStatus; - // Evaluate hardware threshold breaches (only for hardware monitors) let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined; if (monitor.type === "hardware" && statusResponse.payload) { @@ -305,9 +303,38 @@ export class StatusService implements IStatusService { } else { monitor.tempAlertCounter = 5; } + + // Check if any counter has reached zero (initial breach) + const anyCounterZero = + monitor.cpuAlertCounter === 0 || monitor.memoryAlertCounter === 0 || monitor.diskAlertCounter === 0 || monitor.tempAlertCounter === 0; + + const anyThresholdBreached = cpuBreach || memoryBreach || diskBreach || tempBreach; + const allThresholdsNormal = !cpuBreach && !memoryBreach && !diskBreach && !tempBreach; + + // Update monitor status based on threshold breach state + if (newStatus !== "down") { + // Don't override "down" status - service unreachable takes precedence + // Check current monitor status, not newStatus for comparison + if (anyCounterZero && anyThresholdBreached && monitor.status !== "breached") { + // Initial breach: counter hit zero, change status to breached + newStatus = "breached"; + statusChanged = true; + } else if (anyCounterZero && anyThresholdBreached && monitor.status === "breached") { + // Already breached, keep status but don't mark as changed + newStatus = "breached"; + // statusChanged remains false + } else if (allThresholdsNormal && monitor.status === "breached") { + // All thresholds returned to normal, recover from breached state + newStatus = "up"; + statusChanged = true; + } + } } } + // Apply the final status + monitor.status = newStatus; + const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor); return { diff --git a/server/src/types/monitor.ts b/server/src/types/monitor.ts index da95f4292..021172c3f 100644 --- a/server/src/types/monitor.ts +++ b/server/src/types/monitor.ts @@ -4,7 +4,7 @@ export type { CheckSnapshot } from "@/types/check.js"; export const MonitorTypes = ["http", "ping", "pagespeed", "hardware", "docker", "port", "game", "unknown"] as const; export type MonitorType = (typeof MonitorTypes)[number]; -export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance"] as const; +export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const; export type MonitorStatus = (typeof MonitorStatuses)[number]; export type MonitorMatchMethod = "equal" | "include" | "regex" | "";