handle infra thresholds

server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts

@@ -183,7 +183,7 @@ class SuperSimpleQueueHelper {
 	}
 
 	private evaluateMonitorAction(statusChangeResult: StatusChangeResult, networkResponse: MonitorStatusResponse): MonitorActionDecision {
-		const { monitor, statusChanged, prevStatus } = statusChangeResult;
+		const { monitor, statusChanged, prevStatus, thresholdBreaches } = statusChangeResult;
 
 		// Initialize result
 		const decision: MonitorActionDecision = {
@@ -213,45 +213,18 @@ class SuperSimpleQueueHelper {
 		}
 
 		// CASE 2: Hardware
-		const payload = networkResponse.payload as HardwareStatusPayload;
-		const metrics = payload?.data;
-
-		if (!metrics) {
-			return decision; // No metrics, can't evaluate
+		if (!thresholdBreaches) {
+			return decision; // No breach data available
 		}
 
-		// Evaluate threshold breaches
-		// CPU check
-		const cpuUsage = metrics.cpu?.usage_percent ?? -1;
-		const cpuBreach = cpuUsage !== -1 && cpuUsage > monitor.cpuAlertThreshold / 100;
-
-		// Memory check
-		const memoryUsage = metrics.memory?.usage_percent ?? -1;
-		const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100;
-
-		// Disk check
-		const diskBreach =
-			metrics.disk?.some((d: any) => typeof d?.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100) ?? false;
-
-		// Temperature check (alert if ANY sensor exceeds threshold)
-		const temps = metrics.cpu?.temperature ?? [];
-		const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold);
-
-		const breaches = {
-			cpu: cpuBreach,
-			memory: memoryBreach,
-			disk: diskBreach,
-			temp: tempBreach,
-		};
-
-		const anyThresholdBreached = Object.values(breaches).some((b) => b);
+		const anyThresholdBreached = Object.values(thresholdBreaches).some((b) => b);
 
 		// Check if countdown counter has reached zero for any breached threshold
 		const shouldNotifyThreshold =
-			(breaches.cpu && monitor.cpuAlertCounter <= 0) ||
-			(breaches.memory && monitor.memoryAlertCounter <= 0) ||
-			(breaches.disk && monitor.diskAlertCounter <= 0) ||
-			(breaches.temp && monitor.tempAlertCounter <= 0);
+			(thresholdBreaches.cpu && monitor.cpuAlertCounter <= 0) ||
+			(thresholdBreaches.memory && monitor.memoryAlertCounter <= 0) ||
+			(thresholdBreaches.disk && monitor.diskAlertCounter <= 0) ||
+			(thresholdBreaches.temp && monitor.tempAlertCounter <= 0);
 
 		// Decision logic for hardware
 		if (statusChanged && monitor.status === "down") {
@@ -271,7 +244,7 @@ class SuperSimpleQueueHelper {
 			decision.shouldSendNotification = true;
 			decision.incidentReason = "threshold_breach";
 			decision.notificationReason = "threshold_breach";
-			decision.thresholdBreaches = breaches;
+			decision.thresholdBreaches = thresholdBreaches;
 		}
 
 		return decision;

server/src/service/infrastructure/statusService.ts

@@ -253,6 +253,61 @@ export class StatusService implements IStatusService {
 			}
 
 			monitor.status = newStatus;
+
+			// Evaluate hardware threshold breaches (only for hardware monitors)
+			let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined;
+			if (monitor.type === "hardware" && statusResponse.payload) {
+				const payload = statusResponse.payload as HardwareStatusPayload;
+				const metrics = payload?.data;
+
+				if (metrics) {
+					// Evaluate threshold breaches
+					const cpuUsage = metrics.cpu?.usage_percent ?? -1;
+					const cpuBreach = cpuUsage !== -1 && cpuUsage > monitor.cpuAlertThreshold / 100;
+
+					const memoryUsage = metrics.memory?.usage_percent ?? -1;
+					const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100;
+
+					const diskBreach =
+						metrics.disk?.some((d: any) => typeof d?.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100) ?? false;
+
+					const temps = metrics.cpu?.temperature ?? [];
+					const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold);
+
+					thresholdBreaches = {
+						cpu: cpuBreach,
+						memory: memoryBreach,
+						disk: diskBreach,
+						temp: tempBreach,
+					};
+
+					// Update counters: decrement if breached, reset to 5 if not breached
+					if (cpuBreach) {
+						monitor.cpuAlertCounter = Math.max(0, monitor.cpuAlertCounter - 1);
+					} else {
+						monitor.cpuAlertCounter = 5;
+					}
+
+					if (memoryBreach) {
+						monitor.memoryAlertCounter = Math.max(0, monitor.memoryAlertCounter - 1);
+					} else {
+						monitor.memoryAlertCounter = 5;
+					}
+
+					if (diskBreach) {
+						monitor.diskAlertCounter = Math.max(0, monitor.diskAlertCounter - 1);
+					} else {
+						monitor.diskAlertCounter = 5;
+					}
+
+					if (tempBreach) {
+						monitor.tempAlertCounter = Math.max(0, monitor.tempAlertCounter - 1);
+					} else {
+						monitor.tempAlertCounter = 5;
+					}
+				}
+			}
+
 			const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
 
 			return {
@@ -261,6 +316,7 @@ export class StatusService implements IStatusService {
 				prevStatus,
 				code,
 				timestamp: new Date().getTime(),
+				thresholdBreaches,
 			};
 		} catch (error: any) {
 			error.service = SERVICE_NAME;

server/src/types/network.ts

@@ -110,4 +110,10 @@ export type StatusChangeResult = {
 	prevStatus: MonitorStatus;
 	code: number;
 	timestamp: number;
+	thresholdBreaches?: {
+		cpu: boolean;
+		memory: boolean;
+		disk: boolean;
+		temp: boolean;
+	};
 };