breached status

client/src/Components/v2/design-elements/StatusLabel.tsx

@@ -21,6 +21,8 @@ export const StatusLabel = ({ status, sx }: { status: MonitorStatus; sx?: SxProp
 			return t("pages.common.monitors.status.up");
 		} else if (status === "down") {
 			return t("pages.common.monitors.status.down");
+		} else if (status === "breached") {
+			return t("pages.common.monitors.status.breached");
 		} else if (status === "maintenance") {
 			return t("pages.common.monitors.status.maintenance");
 		} else if (status === "paused") {

client/src/Types/Monitor.ts

@@ -18,6 +18,7 @@ export const MonitorStatuses = [
 	"paused",
 	"initializing",
 	"maintenance",
+	"breached",
 ] as const;
 export type MonitorStatus = (typeof MonitorStatuses)[number];
 

client/src/Utils/MonitorUtils.ts

@@ -23,6 +23,9 @@ export const getStatusPalette = (status: MonitorStatus): PaletteKey => {
 	if (status === "down") {
 		return "error";
 	}
+	if (status === "breached") {
+		return "error";
+	}
 	return "warning";
 };
 

client/src/locales/en.json

@@ -387,6 +387,7 @@
 				},
 				"status": {
 					"down": "down",
+					"breached": "breached",
 					"initializing": "initializing",
 					"maintenance": "maintenance",
 					"paused": "paused",

server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts

@@ -187,7 +187,7 @@ class SuperSimpleQueueHelper {
 	}
 
 	private evaluateMonitorAction(statusChangeResult: StatusChangeResult): MonitorActionDecision {
-		const { monitor, statusChanged, prevStatus, thresholdBreaches } = statusChangeResult;
+		const { monitor, statusChanged, prevStatus } = statusChangeResult;
 
 		// Initialize result
 		const decision: MonitorActionDecision = {
@@ -198,75 +198,28 @@ class SuperSimpleQueueHelper {
 			notificationReason: null,
 		};
 
-		// CASE 1: Non-hardware
-		if (monitor.type !== "hardware") {
-			if (statusChanged) {
-				// This is the current state of the monitor AFTER the most recent check
-				if (monitor.status === "down") {
-					decision.shouldCreateIncident = true;
-					decision.shouldSendNotification = true;
-					decision.incidentReason = "status_down";
-					decision.notificationReason = "status_change";
-				} else if (monitor.status === "up" && prevStatus === "down") {
-					decision.shouldResolveIncident = true;
-					decision.shouldSendNotification = true;
-					decision.notificationReason = "status_change";
-				}
-			}
+		// Simplified logic: Just check status changes
+		if (!statusChanged) {
 			return decision;
 		}
 
-		// CASE 2: Hardware
-		if (!thresholdBreaches) {
-			// No breach data available - still handle status changes
-			if (statusChanged && monitor.status === "down") {
-				decision.shouldCreateIncident = true;
-				decision.shouldSendNotification = true;
-				decision.incidentReason = "status_down";
-				decision.notificationReason = "status_change";
-			} else if (statusChanged && monitor.status === "up" && prevStatus === "down") {
-				decision.shouldResolveIncident = true;
-				decision.shouldSendNotification = true;
-				decision.notificationReason = "status_change";
-			}
-			return decision;
-		}
-
-		const anyThresholdBreached = Object.values(thresholdBreaches).some((b) => b);
-		const noThresholdsBreached = Object.values(thresholdBreaches).every((b) => !b);
-
-		// Check if countdown counter has reached zero for any breached threshold
-		const shouldNotifyThreshold =
-			(thresholdBreaches.cpu && monitor.cpuAlertCounter <= 0) ||
-			(thresholdBreaches.memory && monitor.memoryAlertCounter <= 0) ||
-			(thresholdBreaches.disk && monitor.diskAlertCounter <= 0) ||
-			(thresholdBreaches.temp && monitor.tempAlertCounter <= 0);
-
-		// Decision logic for hardware
-		if (statusChanged && monitor.status === "down") {
-			// Hardware service is DOWN (unreachable)
-			// Status down takes precedence over threshold breaches
+		if (monitor.status === "down") {
+			// Monitor went down (unreachable)
 			decision.shouldCreateIncident = true;
 			decision.shouldSendNotification = true;
 			decision.incidentReason = "status_down";
 			decision.notificationReason = "status_change";
-		} else if (statusChanged && monitor.status === "up" && prevStatus === "down") {
-			// Hardware service recovered
-			decision.shouldResolveIncident = true;
-			decision.shouldSendNotification = true;
-			decision.notificationReason = "status_change";
-		} else if (anyThresholdBreached && shouldNotifyThreshold) {
-			// Threshold breach: Counters have reached zero
-			// Create incident if service is UP but metrics exceed thresholds
+		} else if (monitor.status === "breached") {
+			// Hardware monitor exceeded thresholds
 			decision.shouldCreateIncident = true;
 			decision.shouldSendNotification = true;
 			decision.incidentReason = "threshold_breach";
 			decision.notificationReason = "threshold_breach";
-			decision.thresholdBreaches = thresholdBreaches;
-		} else if (noThresholdsBreached && !anyThresholdBreached) {
-			// All thresholds returned to normal - resolve any active threshold incidents
+		} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
+			// Monitor recovered from down or breached state
 			decision.shouldResolveIncident = true;
-			// Don't send notification for resolution (can be added later if needed)
+			decision.shouldSendNotification = true;
+			decision.notificationReason = "status_change";
 		}
 
 		return decision;

server/src/service/infrastructure/statusService.ts

@@ -252,8 +252,6 @@ export class StatusService implements IStatusService {
 				statusChanged = true;
 			}
 
-			monitor.status = newStatus;
-
 			// Evaluate hardware threshold breaches (only for hardware monitors)
 			let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined;
 			if (monitor.type === "hardware" && statusResponse.payload) {
@@ -305,9 +303,38 @@ export class StatusService implements IStatusService {
 					} else {
 						monitor.tempAlertCounter = 5;
 					}
+
+					// Check if any counter has reached zero (initial breach)
+					const anyCounterZero =
+						monitor.cpuAlertCounter === 0 || monitor.memoryAlertCounter === 0 || monitor.diskAlertCounter === 0 || monitor.tempAlertCounter === 0;
+
+					const anyThresholdBreached = cpuBreach || memoryBreach || diskBreach || tempBreach;
+					const allThresholdsNormal = !cpuBreach && !memoryBreach && !diskBreach && !tempBreach;
+
+					// Update monitor status based on threshold breach state
+					if (newStatus !== "down") {
+						// Don't override "down" status - service unreachable takes precedence
+						// Check current monitor status, not newStatus for comparison
+						if (anyCounterZero && anyThresholdBreached && monitor.status !== "breached") {
+							// Initial breach: counter hit zero, change status to breached
+							newStatus = "breached";
+							statusChanged = true;
+						} else if (anyCounterZero && anyThresholdBreached && monitor.status === "breached") {
+							// Already breached, keep status but don't mark as changed
+							newStatus = "breached";
+							// statusChanged remains false
+						} else if (allThresholdsNormal && monitor.status === "breached") {
+							// All thresholds returned to normal, recover from breached state
+							newStatus = "up";
+							statusChanged = true;
+						}
+					}
 				}
 			}
 
+			// Apply the final status
+			monitor.status = newStatus;
+
 			const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
 
 			return {

server/src/types/monitor.ts

@@ -4,7 +4,7 @@ export type { CheckSnapshot } from "@/types/check.js";
 export const MonitorTypes = ["http", "ping", "pagespeed", "hardware", "docker", "port", "game", "unknown"] as const;
 export type MonitorType = (typeof MonitorTypes)[number];
 
-export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance"] as const;
+export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
 export type MonitorStatus = (typeof MonitorStatuses)[number];
 
 export type MonitorMatchMethod = "equal" | "include" | "regex" | "";