Merge pull request #3290 from bluewave-labs/feat/incident-on-threshold-exception

Feat/incident on threshold exception

client/src/Components/v2/design-elements/StatusBox.tsx

@@ -139,6 +139,17 @@ export const InitializingStatusBox = ({ n }: { n: number }) => {
 		/>
 	);
 };
+export const BreachedStatusBox = ({ n }: { n: number }) => {
+	const theme = useTheme();
+	const { t } = useTranslation();
+	return (
+		<StatusBox
+			label={t("pages.common.monitors.status.breached")}
+			n={n}
+			color={theme.palette.warning.main}
+		/>
+	);
+};
 
 export const TotalChecksBox = ({ n }: { n: number }) => {
 	const theme = useTheme();

client/src/Components/v2/design-elements/StatusLabel.tsx

@@ -21,6 +21,8 @@ export const StatusLabel = ({ status, sx }: { status: MonitorStatus; sx?: SxProp
 			return t("pages.common.monitors.status.up");
 		} else if (status === "down") {
 			return t("pages.common.monitors.status.down");
+		} else if (status === "breached") {
+			return t("pages.common.monitors.status.breached");
 		} else if (status === "maintenance") {
 			return t("pages.common.monitors.status.maintenance");
 		} else if (status === "paused") {

client/src/Components/v2/monitors/HeaderMonitorsSummary.tsx

@@ -3,6 +3,7 @@ import {
 	DownStatusBox,
 	PausedStatusBox,
 	InitializingStatusBox,
+	BreachedStatusBox,
 } from "@/Components/v2/design-elements";
 import Stack from "@mui/material/Stack";
 
@@ -11,9 +12,13 @@ import { useTheme } from "@mui/material";
 
 interface MonitorsSummaryProps {
 	summary: MonitorsSummary | null;
+	showBreached?: boolean;
 }
 
-export const HeaderMonitorsSummary = ({ summary }: MonitorsSummaryProps) => {
+export const HeaderMonitorsSummary = ({
+	summary,
+	showBreached = false,
+}: MonitorsSummaryProps) => {
 	const theme = useTheme();
 	return (
 		<Stack
@@ -22,6 +27,7 @@ export const HeaderMonitorsSummary = ({ summary }: MonitorsSummaryProps) => {
 		>
 			<UpStatusBox n={summary?.upMonitors || 0} />
 			<DownStatusBox n={summary?.downMonitors || 0} />
+			{showBreached && <BreachedStatusBox n={summary?.breachedMonitors || 0} />}
 			<PausedStatusBox n={summary?.pausedMonitors || 0} />
 			<InitializingStatusBox n={summary?.initializingMonitors || 0} />
 		</Stack>

client/src/Hooks/useMonitorForm.ts

@@ -83,10 +83,10 @@ export const useMonitorForm = ({
 					type: "hardware",
 					url: data?.url || "",
 					secret: data?.secret || "",
-					cpuAlertThreshold: data?.cpuAlertThreshold ?? 80,
-					memoryAlertThreshold: data?.memoryAlertThreshold ?? 80,
-					diskAlertThreshold: data?.diskAlertThreshold ?? 80,
-					tempAlertThreshold: data?.tempAlertThreshold ?? 80,
+					cpuAlertThreshold: data?.cpuAlertThreshold ?? 100,
+					memoryAlertThreshold: data?.memoryAlertThreshold ?? 100,
+					diskAlertThreshold: data?.diskAlertThreshold ?? 100,
+					tempAlertThreshold: data?.tempAlertThreshold ?? 100,
 					selectedDisks: data?.selectedDisks || [],
 				};
 				break;

client/src/Pages/Infrastructure/Monitors/index.tsx

@@ -133,7 +133,10 @@ const InfrastructureMonitors = () => {
 				isLoading={isLoading}
 				isAdmin={isAdmin}
 			/>
-			<HeaderMonitorsSummary summary={summary} />
+			<HeaderMonitorsSummary
+				summary={summary}
+				showBreached={true}
+			/>
 			<Stack
 				direction={isSmall ? "column" : "row"}
 				justifyContent={isSmall ? "flex-start" : "space-between"}

client/src/Types/Monitor.ts

@@ -18,16 +18,10 @@ export const MonitorStatuses = [
 	"paused",
 	"initializing",
 	"maintenance",
+	"breached",
 ] as const;
 export type MonitorStatus = (typeof MonitorStatuses)[number];
 
-export interface MonitorThresholds {
-	usage_cpu?: number;
-	usage_memory?: number;
-	usage_disk?: number;
-	usage_temperature?: number;
-}
-
 export type MonitorMatchMethod = "equal" | "include" | "regex" | "";
 
 export interface Monitor {
@@ -53,12 +47,14 @@ export interface Monitor {
 	uptimePercentage?: number;
 	notifications: string[];
 	secret?: string;
-	thresholds?: MonitorThresholds;
-	alertThreshold: number;
 	cpuAlertThreshold: number;
+	cpuAlertCounter: number;
 	memoryAlertThreshold: number;
+	memoryAlertCounter: number;
 	diskAlertThreshold: number;
+	diskAlertCounter: number;
 	tempAlertThreshold: number;
+	tempAlertCounter: number;
 	selectedDisks: string[];
 	gameId?: string;
 	group: string | null;
@@ -76,6 +72,7 @@ export interface MonitorsSummary {
 	pausedMonitors: number;
 	initializingMonitors: number;
 	maintenanceMonitors: number;
+	breachedMonitors: number;
 }
 
 export interface MonitorsWithChecksResponse {

client/src/Utils/MonitorUtils.ts

@@ -23,6 +23,9 @@ export const getStatusPalette = (status: MonitorStatus): PaletteKey => {
 	if (status === "down") {
 		return "error";
 	}
+	if (status === "breached") {
+		return "error";
+	}
 	return "warning";
 };
 

client/src/locales/en.json

@@ -387,6 +387,7 @@
 				},
 				"status": {
 					"down": "down",
+					"breached": "breached",
 					"initializing": "initializing",
 					"maintenance": "maintenance",
 					"paused": "paused",

server/src/config/services.ts

@@ -182,6 +182,7 @@ export const initializeServices = async ({
 		discordProvider,
 		pagerDutyProvider,
 		matrixProvider,
+		settingsService,
 		logger
 	);
 

server/src/db/models/Monitor.ts

@@ -1,5 +1,5 @@
 import { Schema, model, Types, type UpdateQuery } from "mongoose";
-import type { Monitor, MonitorMatchMethod, MonitorThresholds, CheckSnapshot } from "@/types/monitor.js";
+import type { Monitor, MonitorMatchMethod, CheckSnapshot } from "@/types/monitor.js";
 import { MonitorTypes, MonitorStatuses } from "@/types/monitor.js";
 
 type CheckSnapshotDocument = Omit<CheckSnapshot, "createdAt"> & { createdAt: Date };
@@ -13,7 +13,6 @@ type MonitorDocumentBase = Omit<
 	notifications: Types.ObjectId[];
 	selectedDisks: string[];
 	matchMethod?: MonitorMatchMethod;
-	thresholds?: MonitorThresholds;
 };
 
 interface MonitorDocument extends MonitorDocumentBase {
@@ -24,16 +23,6 @@ interface MonitorDocument extends MonitorDocumentBase {
 	updatedAt: Date;
 }
 
-const thresholdsSchema = new Schema<MonitorThresholds>(
-	{
-		usage_cpu: { type: Number },
-		usage_memory: { type: Number },
-		usage_disk: { type: Number },
-		usage_temperature: { type: Number },
-	},
-	{ _id: false }
-);
-
 const checkSnapshotSchema = new Schema<CheckSnapshotDocument>(
 	{
 		id: { type: String, required: true },
@@ -132,36 +121,37 @@ const MonitorSchema = new Schema<MonitorDocument>(
 		secret: {
 			type: String,
 		},
-		thresholds: {
-			type: thresholdsSchema,
+		cpuAlertThreshold: {
+			type: Number,
+			default: 100,
 		},
-		alertThreshold: {
+		cpuAlertCounter: {
 			type: Number,
 			default: 5,
 		},
-		cpuAlertThreshold: {
-			type: Number,
-			default: function () {
-				return this.alertThreshold;
-			},
-		},
 		memoryAlertThreshold: {
 			type: Number,
-			default: function () {
-				return this.alertThreshold;
-			},
+			default: 100,
+		},
+		memoryAlertCounter: {
+			type: Number,
+			default: 5,
 		},
 		diskAlertThreshold: {
 			type: Number,
-			default: function () {
-				return this.alertThreshold;
-			},
+			default: 100,
+		},
+		diskAlertCounter: {
+			type: Number,
+			default: 5,
 		},
 		tempAlertThreshold: {
 			type: Number,
-			default: function () {
-				return this.alertThreshold;
-			},
+			default: 100,
+		},
+		tempAlertCounter: {
+			type: Number,
+			default: 5,
 		},
 		selectedDisks: {
 			type: [String],
@@ -189,33 +179,6 @@ const MonitorSchema = new Schema<MonitorDocument>(
 	}
 );
 
-MonitorSchema.pre("save", function (next) {
-	if (!this.cpuAlertThreshold || this.isModified("alertThreshold")) {
-		this.cpuAlertThreshold = this.alertThreshold;
-	}
-	if (!this.memoryAlertThreshold || this.isModified("alertThreshold")) {
-		this.memoryAlertThreshold = this.alertThreshold;
-	}
-	if (!this.diskAlertThreshold || this.isModified("alertThreshold")) {
-		this.diskAlertThreshold = this.alertThreshold;
-	}
-	if (!this.tempAlertThreshold || this.isModified("alertThreshold")) {
-		this.tempAlertThreshold = this.alertThreshold;
-	}
-	next();
-});
-
-MonitorSchema.pre("findOneAndUpdate", function (next) {
-	const update = this.getUpdate() as UpdateQuery<MonitorDocument> | null;
-	if (update && !Array.isArray(update) && update.alertThreshold !== undefined) {
-		update.cpuAlertThreshold = update.alertThreshold;
-		update.memoryAlertThreshold = update.alertThreshold;
-		update.diskAlertThreshold = update.alertThreshold;
-		update.tempAlertThreshold = update.alertThreshold;
-	}
-	next();
-});
-
 MonitorSchema.index({ teamId: 1, type: 1 });
 
 const MonitorModel = model<MonitorDocument>("Monitor", MonitorSchema);

server/src/repositories/monitors/MongoMonitorsRepository.ts

@@ -252,13 +252,28 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 							$cond: [{ $eq: ["$status", "maintenance"] }, 1, 0],
 						},
 					},
+					breachedMonitors: {
+						$sum: {
+							$cond: [{ $eq: ["$status", "breached"] }, 1, 0],
+						},
+					},
 				},
 			},
 			{ $project: { _id: 0 } },
 		];
 
 		const [summary] = await MonitorModel.aggregate(pipeline);
-		return summary ?? { totalMonitors: 0, upMonitors: 0, downMonitors: 0, pausedMonitors: 0, initializingMonitors: 0, maintenanceMonitors: 0 };
+		return (
+			summary ?? {
+				totalMonitors: 0,
+				upMonitors: 0,
+				downMonitors: 0,
+				pausedMonitors: 0,
+				initializingMonitors: 0,
+				maintenanceMonitors: 0,
+				breachedMonitors: 0,
+			}
+		);
 	};
 
 	findGroupsByTeamId = async (teamId: string): Promise<string[]> => {
@@ -317,12 +332,14 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
 			secret: doc.secret ?? undefined,
-			thresholds: doc.thresholds ?? undefined,
-			alertThreshold: doc.alertThreshold,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
+			cpuAlertCounter: doc.cpuAlertCounter,
 			memoryAlertThreshold: doc.memoryAlertThreshold,
+			memoryAlertCounter: doc.memoryAlertCounter,
 			diskAlertThreshold: doc.diskAlertThreshold,
+			diskAlertCounter: doc.diskAlertCounter,
 			tempAlertThreshold: doc.tempAlertThreshold,
+			tempAlertCounter: doc.tempAlertCounter,
 			selectedDisks: doc.selectedDisks ?? [],
 			gameId: doc.gameId ?? undefined,
 			group: doc.group ?? null,
@@ -370,12 +387,14 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			uptimePercentage: doc.uptimePercentage ?? undefined,
 			notifications: notificationIds,
 			secret: doc.secret ?? undefined,
-			thresholds: doc.thresholds ?? undefined,
-			alertThreshold: doc.alertThreshold,
 			cpuAlertThreshold: doc.cpuAlertThreshold,
+			cpuAlertCounter: doc.cpuAlertCounter,
 			memoryAlertThreshold: doc.memoryAlertThreshold,
+			memoryAlertCounter: doc.memoryAlertCounter,
 			diskAlertThreshold: doc.diskAlertThreshold,
+			diskAlertCounter: doc.diskAlertCounter,
 			tempAlertThreshold: doc.tempAlertThreshold,
+			tempAlertCounter: doc.tempAlertCounter,
 			selectedDisks: doc.selectedDisks ?? [],
 			gameId: doc.gameId ?? undefined,
 			group: doc.group ?? null,

server/src/service/business/incidentService.ts

@@ -4,6 +4,7 @@ import { AppError } from "@/utils/AppError.js";
 import { ParseBoolean } from "@/utils/utils.js";
 import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from "@/repositories/index.js";
 import type { Incident } from "@/types/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 
 const dateRangeLookup: Record<string, Date | undefined> = {
 	recent: new Date(new Date().setHours(new Date().getHours() - 2)),
@@ -43,10 +44,14 @@ class IncidentService {
 		return IncidentService.SERVICE_NAME;
 	}
 
-	handleIncident = async (monitor: Monitor, code: number): Promise<Incident | null> => {
+	handleIncident = async (monitor: Monitor, code: number, decision: MonitorActionDecision): Promise<Incident | null> => {
+		if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) {
+			return null;
+		}
+
 		const activeIncident = await this.incidentsRepository.findActiveByMonitorId(monitor.id, monitor.teamId);
-		// Monitor is down, create an incident
-		if (monitor.status === "down") {
+
+		if (decision.shouldCreateIncident) {
 			if (activeIncident) {
 				return activeIncident;
 			} else {
@@ -61,14 +66,17 @@ class IncidentService {
 			}
 		}
 
-		// Monitor is up, resolve active incidents
-		if (!activeIncident) {
-			return null;
+		if (decision.shouldResolveIncident) {
+			if (!activeIncident) {
+				return null;
+			}
+			activeIncident.status = false;
+			activeIncident.endTime = Date.now().toString();
+			activeIncident.resolutionType = "automatic";
+			return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
 		}
-		activeIncident.status = false;
-		activeIncident.endTime = Date.now().toString();
-		activeIncident.resolutionType = "automatic";
-		return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
+
+		return null;
 	};
 
 	resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => {

server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts

@@ -2,9 +2,24 @@ const SERVICE_NAME = "JobQueueHelper";
 import type { Monitor } from "@/types/monitor.js";
 import { AppError } from "@/utils/AppError.js";
 import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js";
+import type { StatusChangeResult, MonitorStatusResponse, HardwareStatusPayload, MonitorStatus } from "@/types/index.js";
 import IncidentService from "@/service/business/incidentService.js";
 import { IMaintenanceWindowsRepository, IMonitorsRepository } from "@/repositories/index.js";
 
+export interface MonitorActionDecision {
+	shouldCreateIncident: boolean;
+	shouldResolveIncident: boolean;
+	shouldSendNotification: boolean;
+	incidentReason: "status_down" | "threshold_breach" | null;
+	notificationReason: "status_change" | "threshold_breach" | null;
+	thresholdBreaches?: {
+		cpu?: boolean;
+		memory?: boolean;
+		disk?: boolean;
+		temp?: boolean;
+	};
+}
+
 class SuperSimpleQueueHelper {
 	static SERVICE_NAME = SERVICE_NAME;
 
@@ -100,10 +115,12 @@ class SuperSimpleQueueHelper {
 				// Step 4.  Update monitor status
 				const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
 
-				// Step 5 handle notifications (best effort, continue even in event of failure, don't wait)
-				this.notificationsService
-					.handleNotifications(statusChangeResult.monitor, status, statusChangeResult.prevStatus, statusChangeResult.statusChanged)
-					.catch((error: any) => {
+				// Step 5.  Get decisions
+				const decision = this.evaluateMonitorAction(statusChangeResult);
+
+				// Step 6. Handle notifications (best effort, continue even in event of failure, don't wait)
+				if (decision.shouldSendNotification) {
+					this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: any) => {
 						this.logger.error({
 							message: error.message,
 							service: SERVICE_NAME,
@@ -112,19 +129,18 @@ class SuperSimpleQueueHelper {
 							stack: error.stack,
 						});
 					});
-
-				// Step 6.  Handle incidents (best effort, don't wait)
-				if (statusChangeResult.statusChanged) {
-					this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code).catch((error: any) => {
-						this.logger.warn({
-							message: error.message,
-							service: SERVICE_NAME,
-							method: "getMonitorJob",
-							details: `Error handling incident for job ${monitor.id}: ${error.message}`,
-							stack: error.stack,
-						});
-					});
 				}
+
+				// Step 7. Handle incidents (best effort, don't wait)
+				this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision).catch((error: any) => {
+					this.logger.warn({
+						message: error.message,
+						service: SERVICE_NAME,
+						method: "getMonitorJob",
+						details: `Error handling incident for job ${monitor.id}: ${error.message}`,
+						stack: error.stack,
+					});
+				});
 			} catch (error: any) {
 				this.logger.warn({
 					message: error.message,
@@ -166,6 +182,45 @@ class SuperSimpleQueueHelper {
 		}, false);
 		return maintenanceWindowIsActive;
 	}
+
+	private evaluateMonitorAction(statusChangeResult: StatusChangeResult): MonitorActionDecision {
+		const { monitor, statusChanged, prevStatus } = statusChangeResult;
+
+		// Initialize result
+		const decision: MonitorActionDecision = {
+			shouldCreateIncident: false,
+			shouldResolveIncident: false,
+			shouldSendNotification: false,
+			incidentReason: null,
+			notificationReason: null,
+		};
+
+		// Simplified logic: Just check status changes
+		if (!statusChanged) {
+			return decision;
+		}
+
+		if (monitor.status === "down") {
+			// Monitor went down (unreachable)
+			decision.shouldCreateIncident = true;
+			decision.shouldSendNotification = true;
+			decision.incidentReason = "status_down";
+			decision.notificationReason = "status_change";
+		} else if (monitor.status === "breached") {
+			// Hardware monitor exceeded thresholds
+			decision.shouldCreateIncident = true;
+			decision.shouldSendNotification = true;
+			decision.incidentReason = "threshold_breach";
+			decision.notificationReason = "threshold_breach";
+		} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
+			// Monitor recovered from down or breached state
+			decision.shouldResolveIncident = true;
+			decision.shouldSendNotification = true;
+			decision.notificationReason = "status_change";
+		}
+
+		return decision;
+	}
 }
 
 export default SuperSimpleQueueHelper;

server/src/service/infrastructure/notificationProviders/INotificationProvider.ts

@@ -1,6 +1,13 @@
 import type { Monitor, Notification, Alert, MonitorStatusResponse } from "@/types/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 
 export interface INotificationProvider {
-	sendAlert: (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => Promise<boolean>;
+	sendAlert: (
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	) => Promise<boolean>;
 	sendTestAlert(notification: Notification): Promise<boolean>;
 }

server/src/service/infrastructure/notificationProviders/discord.ts

@@ -1,6 +1,7 @@
 const SERVICE_NAME = "DiscordProvider";
 import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
 import { INotificationProvider } from "@/service/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 import { buildHardwareAlerts, buildDiscordBody, getTestMessage } from "@/service/infrastructure/notificationProviders/utils.js";
 import got from "got";
 
@@ -10,15 +11,31 @@ export class DiscordProvider implements INotificationProvider {
 	constructor(logger: any) {
 		this.logger = logger;
 	}
-	private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
-		const { discordPayload } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
+	private getHardwareContent = (
+		clientHost: string,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision
+	) => {
+		// For status changes (recovery), use standard format
+		if (decision.notificationReason === "status_change") {
+			return buildDiscordBody(monitor, monitorStatusResponse);
+		}
+		// For threshold breaches, use hardware alert format
+		const { discordPayload } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse);
 		return discordPayload;
 	};
 
-	sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
+	sendAlert = async (
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	) => {
 		let body;
 		if (monitor.type === "hardware") {
-			body = this.getHardwareContent(monitor, monitorStatusResponse);
+			body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision);
 		} else {
 			body = buildDiscordBody(monitor, monitorStatusResponse);
 		}

server/src/service/infrastructure/notificationProviders/email.ts

@@ -1,6 +1,7 @@
 const SERVICE_NAME = "EmailProvider";
 import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
 import { INotificationProvider } from "@/service/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 import { buildHardwareAlerts, buildHardwareEmail, buildEmail, buildTestEmail } from "@/service/infrastructure/notificationProviders/utils.js";
 
 export class EmailProvider implements INotificationProvider {
@@ -12,21 +13,48 @@ export class EmailProvider implements INotificationProvider {
 		this.logger = logger;
 	}
 
-	private buildHardwareEmail = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
-		const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
+	private buildHardwareEmail = async (
+		clientHost: string,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision
+	) => {
+		// For status changes (recovery), use standard email format
+		if (decision.notificationReason === "status_change") {
+			return await buildEmail(this.emailService, monitor);
+		}
+		// For threshold breaches, use hardware alert format
+		const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse);
 		const html = buildHardwareEmail(this.emailService, monitor, alertsToSend);
 		return html;
 	};
 
-	async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
+	async sendAlert(
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	): Promise<boolean> {
 		// For grouped notifications (identified by ":" in name), customize subject to indicate multiple services.
 		// Example: "2 services: Service A, Service B" becomes "Alert: 2 services are down"
 		const isGroupedNotification = monitor.name.includes(":");
-		const subject = isGroupedNotification ? `Alert: ${monitor.name} are down` : `Monitor ${monitor.name} is down`;
+
+		// Build subject based on notification reason and monitor status
+		let subject: string;
+		if (isGroupedNotification) {
+			subject = `Alert: ${monitor.name} are down`;
+		} else if (decision.notificationReason === "threshold_breach") {
+			subject = `Monitor ${monitor.name} threshold breached`;
+		} else if (monitor.status === "up") {
+			subject = `Monitor ${monitor.name} is back up`;
+		} else {
+			subject = `Monitor ${monitor.name} is down`;
+		}
 
 		let html;
 		if (monitor.type === "hardware") {
-			html = this.buildHardwareEmail(monitor, monitorStatusResponse);
+			html = await this.buildHardwareEmail(clientHost, monitor, monitorStatusResponse, decision);
 		} else {
 			html = await buildEmail(this.emailService, monitor);
 		}

server/src/service/infrastructure/notificationProviders/matrix.ts

@@ -1,10 +1,11 @@
 const SERVICE_NAME = "MatrixProvider";
 import got from "got";
 import type { INotificationProvider } from "@/service/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 import type { Notification, Monitor, MonitorStatusResponse } from "@/types/index.js";
 import {
 	buildHardwareAlerts,
-	buildHardwareWebhookBody,
+	buildHardwareNotificationMessage,
 	buildWebhookBody,
 	getTestMessage,
 } from "@/service/infrastructure/notificationProviders/utils.js";
@@ -15,9 +16,19 @@ export class MatrixProvider implements INotificationProvider {
 	constructor(logger: any) {
 		this.logger = logger;
 	}
-	private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
-		const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
-		const body = buildHardwareWebhookBody(alertsToSend, monitor);
+	private getHardwareContent = (
+		clientHost: string,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision
+	) => {
+		// For status changes (recovery), use standard format
+		if (decision.notificationReason === "status_change") {
+			return buildWebhookBody(monitor, monitorStatusResponse);
+		}
+		// For threshold breaches, use hardware alert format
+		const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse);
+		const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor);
 		return body;
 	};
 
@@ -26,12 +37,18 @@ export class MatrixProvider implements INotificationProvider {
 		return body;
 	};
 
-	sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
+	sendAlert = async (
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	) => {
 		const { homeserverUrl, accessToken, roomId } = notification;
 
 		let content;
 		if (monitor.type === "hardware") {
-			content = this.getHardwareContent(monitor, monitorStatusResponse);
+			content = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision);
 		} else {
 			content = this.getContent(monitor, monitorStatusResponse);
 		}

server/src/service/infrastructure/notificationProviders/pagerduty.ts

@@ -1,10 +1,11 @@
 const SERVICE_NAME = "PagerDutyProvider";
 import got from "got";
 import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 import { INotificationProvider } from "@/service/index.js";
 import {
 	buildHardwareAlerts,
-	buildHardwareWebhookBody,
+	buildHardwareNotificationMessage,
 	buildWebhookBody,
 	getTestMessage,
 } from "@/service/infrastructure/notificationProviders/utils.js";
@@ -15,9 +16,19 @@ export class PagerDutyProvider implements INotificationProvider {
 	constructor(logger: any) {
 		this.logger = logger;
 	}
-	private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
-		const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
-		const body = buildHardwareWebhookBody(alertsToSend, monitor);
+	private getHardwareContent = (
+		clientHost: string,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision
+	) => {
+		// For status changes (recovery), use standard format
+		if (decision.notificationReason === "status_change") {
+			return buildWebhookBody(monitor, monitorStatusResponse);
+		}
+		// For threshold breaches, use hardware alert format
+		const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse);
+		const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor);
 		return body;
 	};
 
@@ -26,10 +37,16 @@ export class PagerDutyProvider implements INotificationProvider {
 		return body;
 	};
 
-	async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
+	async sendAlert(
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	): Promise<boolean> {
 		let body;
 		if (monitor.type === "hardware") {
-			body = this.getHardwareContent(monitor, monitorStatusResponse);
+			body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision);
 		} else {
 			body = this.getContent(monitor, monitorStatusResponse);
 		}

server/src/service/infrastructure/notificationProviders/slack.ts

@@ -1,9 +1,10 @@
 const SERVICE_NAME = "SlackProvider";
 import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
 import { INotificationProvider } from "@/service/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 import {
 	buildHardwareAlerts,
-	buildHardwareWebhookBody,
+	buildHardwareNotificationMessage,
 	buildWebhookBody,
 	getTestMessage,
 } from "@/service/infrastructure/notificationProviders/utils.js";
@@ -15,9 +16,19 @@ export class SlackProvider implements INotificationProvider {
 	constructor(logger: any) {
 		this.logger = logger;
 	}
-	private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
-		const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
-		const body = buildHardwareWebhookBody(alertsToSend, monitor);
+	private getHardwareContent = (
+		clientHost: string,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision
+	) => {
+		// For status changes (recovery), use standard format
+		if (decision.notificationReason === "status_change") {
+			return buildWebhookBody(monitor, monitorStatusResponse);
+		}
+		// For threshold breaches, use hardware alert format
+		const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse);
+		const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor);
 		return body;
 	};
 
@@ -26,10 +37,16 @@ export class SlackProvider implements INotificationProvider {
 		return body;
 	};
 
-	async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
+	async sendAlert(
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	): Promise<boolean> {
 		let body;
 		if (monitor.type === "hardware") {
-			body = this.getHardwareContent(monitor, monitorStatusResponse);
+			body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision);
 		} else {
 			body = this.getContent(monitor, monitorStatusResponse);
 		}

server/src/service/infrastructure/notificationProviders/utils.ts

@@ -23,17 +23,26 @@ export const buildHardwareAlerts = (
 	monitor: Monitor,
 	networkResponse: MonitorStatusResponse
 ): { alertsToSend: string[]; discordPayload: any } => {
-	const thresholds = monitor.thresholds || {};
-	const { usage_cpu: cpuThreshold = -1, usage_memory: memoryThreshold = -1, usage_disk: diskThreshold = -1 } = thresholds;
+	// Thresholds are stored as percentages (0-100), convert to decimal (0-1) for comparison
+	const cpuThreshold = monitor.cpuAlertThreshold !== undefined ? monitor.cpuAlertThreshold / 100 : -1;
+	const memoryThreshold = monitor.memoryAlertThreshold !== undefined ? monitor.memoryAlertThreshold / 100 : -1;
+	const diskThreshold = monitor.diskAlertThreshold !== undefined ? monitor.diskAlertThreshold / 100 : -1;
+	const tempThreshold = monitor.tempAlertThreshold !== undefined ? monitor.tempAlertThreshold : -1;
 
 	const payload = networkResponse?.payload as HardwareStatusPayload;
 	const metrics = payload.data || {};
-	const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics;
+	const { cpu = {}, memory = {}, disk = [] } = metrics;
+	const cpuUsage = cpu.usage_percent ?? -1;
+	const memoryUsage = memory.usage_percent ?? -1;
+	// Get max temperature from CPU temperature sensors array
+	const temps = cpu.temperature ?? [];
+	const maxTemp = temps.length > 0 ? Math.max(...temps) : -1;
 
 	const alerts: Record<string, boolean> = {
 		cpu: cpuThreshold !== -1 && cpuUsage > cpuThreshold ? true : false,
 		memory: memoryThreshold !== -1 && memoryUsage > memoryThreshold ? true : false,
 		disk: disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold) ?? false,
+		temp: tempThreshold !== -1 && maxTemp > tempThreshold ? true : false,
 	};
 
 	const alertsToSend = [];
@@ -43,6 +52,7 @@ export const buildHardwareAlerts = (
 		{ name: "URL", value: monitor.url, inline: false },
 	];
 	const goToIncidentField = { name: `Go to incident`, value: `${clientHost}/infrastructure/${monitor.id}` };
+
 	const formatDiscordAlert = {
 		cpu: () => ({
 			title: "CPU alert",
@@ -75,78 +85,42 @@ export const buildHardwareAlerts = (
 				goToIncidentField,
 			],
 		}),
+
+		temp: () => ({
+			title: "Temperature alert",
+			description: `Your current temperature (${maxTemp.toFixed(0)}°C) is above your threshold (${tempThreshold.toFixed(0)}°C)`,
+			color: 15548997,
+			fields: [...monitorInfoFields, goToIncidentField],
+			footer: { text: "Checkmate" },
+		}),
 	};
-	const alertTypes = ["cpu", "memory", "disk"] as const;
-	const alertThresholdKeyMap: Record<(typeof alertTypes)[number], "cpuAlertThreshold" | "memoryAlertThreshold" | "diskAlertThreshold"> = {
-		cpu: "cpuAlertThreshold",
-		memory: "memoryAlertThreshold",
-		disk: "diskAlertThreshold",
-	};
+
+	const alertTypes = ["cpu", "memory", "disk", "temp"] as const;
+
 	for (const type of alertTypes) {
-		const thresholdKey = alertThresholdKeyMap[type];
-		// Iterate over each alert type to see if any need to be decremented
 		if (alerts[type] === true) {
-			const nextValue = ((monitor[thresholdKey] ?? monitor.alertThreshold) as number) - 1;
-			monitor[thresholdKey] = nextValue; // Decrement threshold if an alert is triggered
-
-			if (monitor[thresholdKey] <= 0) {
-				// If threshold drops below 0, reset and send notification
-				monitor[thresholdKey] = monitor.alertThreshold;
-
-				const formatAlert = {
-					cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
-					memory: () =>
-						`Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
-					disk: () =>
-						`Your current disk usage: ${disk
-							.map((d, idx) => `(Disk${idx}: ${(d?.usage_percent ?? 0 * 100).toFixed(0)}%)`)
-							.join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
-				};
-				alertsToSend.push(formatAlert[type]());
-				discordEmbeds.push(formatDiscordAlert[type]());
-			}
+			const formatAlert = {
+				cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
+				memory: () =>
+					`Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
+				disk: () =>
+					`Your current disk usage: ${disk.map((d, idx) => `(Disk${idx}: ${(d?.usage_percent ?? 0 * 100).toFixed(0)}%)`).join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
+				temp: () => `Your current temperature (${maxTemp.toFixed(0)}°C) is above your threshold (${tempThreshold.toFixed(0)}°C)`,
+			};
+			alertsToSend.push(formatAlert[type]());
+			discordEmbeds.push(formatDiscordAlert[type]());
 		}
 	}
+
 	const discordPayload = discordEmbeds.length ? { embeds: discordEmbeds } : null;
 	return { alertsToSend, discordPayload };
 };
 
-export const buildHardwareNotificationMessage = (clientHost: string, alerts: any, monitor: Monitor) => {
+export const buildHardwareNotificationMessage = (clientHost: string, alerts: string[], monitor: Monitor): string => {
 	const alertsHeader = [`Monitor: ${monitor.name}`, `URL: ${monitor.url}`];
 	const alertFooter = [`Go to incident: ${clientHost}/infrastructure/${monitor.id}`];
 	const alertText = alerts.length > 0 ? [...alertsHeader, ...alerts, ...alertFooter] : [];
-	return alertText.map((alert) => alert).join("\n");
-};
-
-export const buildHardwareWebhookBody = (alerts: string[], monitor: Monitor): string => {
-	const content = alerts.map((alert) => alert).join("\n");
-	return content;
-};
-
-export const shouldSendHardwareAlert = (monitor: Monitor, networkResponse: MonitorStatusResponse): boolean => {
-	const thresholds = monitor.thresholds || {};
-	const { usage_cpu: cpuThreshold = -1, usage_memory: memoryThreshold = -1, usage_disk: diskThreshold = -1 } = thresholds;
-
-	const payload = networkResponse?.payload as HardwareStatusPayload;
-	const metrics = payload.data || {};
-	const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics;
-
-	const cpuBreach = cpuThreshold !== -1 && cpuUsage > cpuThreshold;
-	if (cpuBreach && ((monitor.cpuAlertThreshold ?? monitor.alertThreshold) as number) - 1 <= 0) {
-		return true;
-	}
-
-	const memoryBreach = memoryThreshold !== -1 && memoryUsage > memoryThreshold;
-	if (memoryBreach && ((monitor.memoryAlertThreshold ?? monitor.alertThreshold) as number) - 1 <= 0) {
-		return true;
-	}
-
-	const diskBreach = disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold);
-	if (diskBreach && ((monitor.diskAlertThreshold ?? monitor.alertThreshold) as number) - 1 <= 0) {
-		return true;
-	}
-
-	return false;
+	return alertText.join("\n");
 };
 
 export const buildWebhookBody = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {

server/src/service/infrastructure/notificationProviders/webhook.ts

@@ -1,9 +1,10 @@
 const SERVICE_NAME = "WebhookProvider";
 import type { Monitor, Alert, Notification, MonitorStatusResponse } from "@/types/index.js";
 import { INotificationProvider } from "@/service/index.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
 import {
 	buildHardwareAlerts,
-	buildHardwareWebhookBody,
+	buildHardwareNotificationMessage,
 	buildWebhookBody,
 	getTestMessage,
 } from "@/service/infrastructure/notificationProviders/utils.js";
@@ -15,9 +16,19 @@ export class WebhookProvider implements INotificationProvider {
 	constructor(logger: any) {
 		this.logger = logger;
 	}
-	private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
-		const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
-		const body = buildHardwareWebhookBody(alertsToSend, monitor);
+	private getHardwareContent = (
+		clientHost: string,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision
+	) => {
+		// For status changes (recovery), use standard format
+		if (decision.notificationReason === "status_change") {
+			return buildWebhookBody(monitor, monitorStatusResponse);
+		}
+		// For threshold breaches, use hardware alert format
+		const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse);
+		const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor);
 		return body;
 	};
 
@@ -26,10 +37,16 @@ export class WebhookProvider implements INotificationProvider {
 		return body;
 	};
 
-	sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
+	sendAlert = async (
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	) => {
 		let body;
 		if (monitor.type === "hardware") {
-			body = this.getHardwareContent(monitor, monitorStatusResponse);
+			body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision);
 		} else {
 			body = this.getContent(monitor, monitorStatusResponse);
 		}

server/src/service/infrastructure/notificationsService.ts

@@ -1,7 +1,9 @@
 import type { HardwareStatusPayload, Monitor, MonitorStatusResponse, Notification, MonitorStatus } from "@/types/index.js";
-import { shouldSendHardwareAlert } from "@/service/infrastructure/notificationProviders/utils.js";
 import { IMonitorsRepository, INotificationsRepository } from "@/repositories/index.js";
 import { INotificationProvider } from "./notificationProviders/INotificationProvider.js";
+import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
+import type { ISettingsService } from "@/service/system/settingsService.js";
+import { ILogger } from "@/utils/logger.js";
 
 export interface INotificationsService {
 	createNotification: (notificationData: Partial<Notification>) => Promise<Notification>;
@@ -9,12 +11,7 @@ export interface INotificationsService {
 	findNotificationsByTeamId: (teamId: string) => Promise<Notification[]>;
 	updateById(id: string, teamId: string, updateData: Partial<Notification>): Promise<Notification>;
 	deleteById: (id: string, teamId: string) => Promise<Notification>;
-	handleNotifications: (
-		monitor: Monitor,
-		monitorStatusResponse: MonitorStatusResponse,
-		prevStatus: MonitorStatus,
-		statusChanged: boolean
-	) => Promise<boolean>;
+	handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise<boolean>;
 
 	sendTestNotification: (notification: Notification) => Promise<boolean>;
 	testAllNotifications: (notificationIds: string[]) => Promise<boolean>;
@@ -33,19 +30,8 @@ export class NotificationsService implements INotificationsService {
 	private discordProvider: INotificationProvider;
 	private pagerDutyProvider: INotificationProvider;
 	private matrixProvider: INotificationProvider;
-	private logger: any;
-
-	// Email grouping (batching) configuration
-	private emailGroupingWindowMs: number;
-	private pendingEmailGroups: Map<
-		string,
-		{
-			monitors: Monitor[];
-			statusResponses: MonitorStatusResponse[];
-			timer: ReturnType<typeof setTimeout>;
-			createdAt: number;
-		}
-	>;
+	private logger: ILogger;
+	private settingsService: ISettingsService;
 
 	constructor(
 		notificationsRepository: INotificationsRepository,
@@ -56,6 +42,7 @@ export class NotificationsService implements INotificationsService {
 		discordProvider: INotificationProvider,
 		pagerDutyProvider: INotificationProvider,
 		matrixProvider: INotificationProvider,
+		settingsService: ISettingsService,
 		logger: any
 	) {
 		this.notificationsRepository = notificationsRepository;
@@ -66,57 +53,42 @@ export class NotificationsService implements INotificationsService {
 		this.discordProvider = discordProvider;
 		this.pagerDutyProvider = pagerDutyProvider;
 		this.matrixProvider = matrixProvider;
+		this.settingsService = settingsService;
 		this.logger = logger;
-
-		// Configure email grouping window (in milliseconds).
-		// When > 0, multiple DOWN events for monitors that share the same
-		// email notification within this window will be batched into a single email.
-		const rawGroupingWindow = process.env.NOTIFICATION_GROUP_WINDOW_MS ?? process.env.NOTIFICATION_GROUP_WINDOW_SECONDS;
-		let groupingWindowMs = 0;
-		if (rawGroupingWindow) {
-			const parsed = Number(rawGroupingWindow);
-			if (!Number.isNaN(parsed) && parsed > 0) {
-				// If value looks like seconds (small number), convert to ms.
-				// This allows either milliseconds (e.g. 60000) or seconds (e.g. 60).
-				groupingWindowMs = parsed <= 300 ? parsed * 1000 : parsed;
-			}
-		}
-		this.emailGroupingWindowMs = groupingWindowMs;
-		this.pendingEmailGroups = new Map();
 	}
 
-	private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
+	private send = async (
+		notification: Notification,
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision
+	): Promise<boolean> => {
+		const settings = this.settingsService.getSettings();
+		const clientHost = settings.clientHost || "Host not defined";
+
 		switch (notification.type) {
 			case "email":
-				return await this.emailProvider.sendAlert(notification, monitor, monitorStatusResponse);
+				return await this.emailProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost);
 			case "slack":
-				return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse);
+				return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost);
 			case "discord":
-				return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse);
+				return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost);
 			case "pager_duty":
-				return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse);
+				return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost);
 			case "matrix":
-				return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse);
+				return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost);
 			case "webhook":
-				return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse);
+				return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost);
 			default:
 				return false;
 		}
 	};
 
-	private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
+	private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
 		const notificationIds = monitor.notifications ?? [];
 		const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
 
-		const tasks = notifications.map((notification) => {
-			// Only group emails, only for DOWN transitions, and only if a window is configured.
-			if (notification.type === "email" && this.emailGroupingWindowMs > 0 && monitorStatusResponse.status === false) {
-				return this.queueGroupedEmailNotification(notification, monitor, monitorStatusResponse);
-			}
-
-			// For all other cases (UP notifications or non-email channels), send immediately.
-			return this.send(notification, monitor, monitorStatusResponse);
-		});
+		const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision));
 
 		const outcomes = await Promise.all(tasks);
 		const succeeded = outcomes.filter(Boolean).length;
@@ -125,136 +97,21 @@ export class NotificationsService implements INotificationsService {
 			this.logger.warn({
 				message: `Notification send completed with ${succeeded} success, ${failed} failure(s)`,
 				service: SERVICE_NAME,
-				method: "getMonitorJob",
+				method: "sendNotifications",
 			});
 		}
-		// Return true if all notificaitons succeeded
+		// Return true if all notifications succeeded
 		return succeeded === notifications.length;
 	};
 
-	/**
-	 * Queue a DOWN email notification to be potentially grouped with other
-	 * DOWN events for the same email notification within the configured window.
-	 *
-	 * This method returns immediately; the actual email is sent asynchronously
-	 * when the grouping window expires.
-	 */
-	private queueGroupedEmailNotification = async (
-		notification: Notification,
-		monitor: Monitor,
-		monitorStatusResponse: MonitorStatusResponse
-	): Promise<boolean> => {
-		// If grouping is disabled, fallback to immediate send.
-		if (this.emailGroupingWindowMs <= 0) {
-			return await this.send(notification, monitor, monitorStatusResponse);
-		}
-
-		const key = notification.id;
-		const now = Date.now();
-		const existingGroup = this.pendingEmailGroups.get(key);
-
-		if (!existingGroup) {
-			// Create a new group and schedule a flush after the window expires.
-			const timer = setTimeout(async () => {
-				const group = this.pendingEmailGroups.get(key);
-				if (!group) return;
-
-				this.pendingEmailGroups.delete(key);
-
-				try {
-					await this.flushEmailGroup(notification, group.monitors, group.statusResponses);
-				} catch (error: any) {
-					this.logger.error({
-						message: error?.message,
-						service: SERVICE_NAME,
-						method: "flushEmailGroup",
-						stack: error?.stack,
-					});
-				}
-			}, this.emailGroupingWindowMs);
-
-			this.pendingEmailGroups.set(key, {
-				monitors: [monitor],
-				statusResponses: [monitorStatusResponse],
-				timer,
-				createdAt: now,
-			});
-		} else {
-			// Append to existing group.
-			existingGroup.monitors.push(monitor);
-			existingGroup.statusResponses.push(monitorStatusResponse);
-		}
-
-		// Consider queueing as "succeeded" from the caller's perspective.
-		return true;
-	};
-
-	/**
-	 * Flush a grouped set of DOWN events into a single email.
-	 *
-	 * To avoid changing email templates, we construct a synthetic Monitor
-	 * whose name concisely lists all affected services. The existing
-	 * `serverIsDownTemplate` is then reused.
-	 *
-	 * @param notification The email notification to send to
-	 * @param monitors Array of monitors that went down
-	 * @param statusResponses Array of status responses (parallel to monitors)
-	 * @returns true if email was sent successfully, false otherwise
-	 */
-	private flushEmailGroup = async (notification: Notification, monitors: Monitor[], statusResponses: MonitorStatusResponse[]): Promise<boolean> => {
-		if (!monitors.length || !statusResponses.length) {
+	handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
+		// Early return if no notification should be sent
+		if (!decision.shouldSendNotification) {
 			return false;
 		}
 
-		// Build a combined monitor name listing all affected services.
-		// Example: "Service A, Service B" (2 services) or "Service A" (1 service)
-		const uniqueNames = Array.from(new Set(monitors.map((m) => m.name)));
-		const servicesCount = uniqueNames.length;
-		const servicesList = uniqueNames.join(", ");
-
-		const combinedName = servicesCount === 1 ? servicesList : `${servicesCount} services: ${servicesList}`;
-
-		// Use the first monitor as a base for URL and other fields.
-		const baseMonitor = monitors[0]!;
-		const baseStatus = statusResponses[0]!;
-
-		// Create a shallow clone so we don't mutate the original entity.
-		// This preserves monitor properties while overriding the name for grouped display.
-		const syntheticMonitor: Monitor = {
-			...baseMonitor,
-			name: combinedName,
-		};
-
-		// Reuse existing email provider to send grouped notification.
-		return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus);
-	};
-
-	handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, prevStatus: MonitorStatus, statusChanged: boolean) => {
-		const { type } = monitor;
-		const payload = monitorStatusResponse.payload as HardwareStatusPayload;
-		// If this is a non-hardeware type monitor and status did not change, we're done
-		if (type !== "hardware" && statusChanged === false) return false;
-		// if prevStatus is undefined, monitor is resuming, we're done
-		if (type !== "hardware" && prevStatus === undefined) return false;
-
-		// Deal with hardware thresholds
-		if (type === "hardware") {
-			const thresholds = monitor.thresholds;
-
-			if (thresholds === undefined) return false; // No thresholds set, we're done
-			const metrics = payload?.data ?? null;
-			if (metrics === null) return false; // No metrics, we're done
-
-			// We should send a notificaiton
-
-			const shouldSend = shouldSendHardwareAlert(monitor, monitorStatusResponse);
-			if (shouldSend === false) return false;
-
-			return await this.sendNotifications(monitor, monitorStatusResponse);
-		}
-
-		// We should send a notification for non-hardware monitor status change
-		return await this.sendNotifications(monitor, monitorStatusResponse);
+		// Send notifications based on decision
+		return await this.sendNotifications(monitor, monitorStatusResponse, decision);
 	};
 
 	sendTestNotification = async (notification: Notification) => {

server/src/service/infrastructure/statusService.ts

@@ -252,7 +252,89 @@ export class StatusService implements IStatusService {
 				statusChanged = true;
 			}
 
+			// Evaluate hardware threshold breaches (only for hardware monitors)
+			let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined;
+			if (monitor.type === "hardware" && statusResponse.payload) {
+				const payload = statusResponse.payload as HardwareStatusPayload;
+				const metrics = payload?.data;
+
+				if (metrics) {
+					// Evaluate threshold breaches
+					const cpuUsage = metrics.cpu?.usage_percent ?? -1;
+					const cpuBreach = cpuUsage !== -1 && cpuUsage > monitor.cpuAlertThreshold / 100;
+
+					const memoryUsage = metrics.memory?.usage_percent ?? -1;
+					const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100;
+
+					const diskBreach =
+						metrics.disk?.some((d: any) => typeof d?.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100) ?? false;
+
+					const temps = metrics.cpu?.temperature ?? [];
+					const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold);
+
+					thresholdBreaches = {
+						cpu: cpuBreach,
+						memory: memoryBreach,
+						disk: diskBreach,
+						temp: tempBreach,
+					};
+
+					// Update counters: decrement if breached, reset to 5 if not breached
+					if (cpuBreach) {
+						monitor.cpuAlertCounter = Math.max(0, monitor.cpuAlertCounter - 1);
+					} else {
+						monitor.cpuAlertCounter = 5;
+					}
+
+					if (memoryBreach) {
+						monitor.memoryAlertCounter = Math.max(0, monitor.memoryAlertCounter - 1);
+					} else {
+						monitor.memoryAlertCounter = 5;
+					}
+
+					if (diskBreach) {
+						monitor.diskAlertCounter = Math.max(0, monitor.diskAlertCounter - 1);
+					} else {
+						monitor.diskAlertCounter = 5;
+					}
+
+					if (tempBreach) {
+						monitor.tempAlertCounter = Math.max(0, monitor.tempAlertCounter - 1);
+					} else {
+						monitor.tempAlertCounter = 5;
+					}
+
+					// Check if any counter has reached zero (initial breach)
+					const anyCounterZero =
+						monitor.cpuAlertCounter === 0 || monitor.memoryAlertCounter === 0 || monitor.diskAlertCounter === 0 || monitor.tempAlertCounter === 0;
+
+					const anyThresholdBreached = cpuBreach || memoryBreach || diskBreach || tempBreach;
+					const allThresholdsNormal = !cpuBreach && !memoryBreach && !diskBreach && !tempBreach;
+
+					// Update monitor status based on threshold breach state
+					if (newStatus !== "down") {
+						// Don't override "down" status - service unreachable takes precedence
+						// Check current monitor status, not newStatus for comparison
+						if (anyCounterZero && anyThresholdBreached && monitor.status !== "breached") {
+							// Initial breach: counter hit zero, change status to breached
+							newStatus = "breached";
+							statusChanged = true;
+						} else if (anyCounterZero && anyThresholdBreached && monitor.status === "breached") {
+							// Already breached, keep status but don't mark as changed
+							newStatus = "breached";
+							// statusChanged remains false
+						} else if (allThresholdsNormal && monitor.status === "breached") {
+							// All thresholds returned to normal, recover from breached state
+							newStatus = "up";
+							statusChanged = true;
+						}
+					}
+				}
+			}
+
+			// Apply the final status
 			monitor.status = newStatus;
+
 			const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
 
 			return {
@@ -261,6 +343,7 @@ export class StatusService implements IStatusService {
 				prevStatus,
 				code,
 				timestamp: new Date().getTime(),
+				thresholdBreaches,
 			};
 		} catch (error: any) {
 			error.service = SERVICE_NAME;

server/src/types/monitor.ts

@@ -4,16 +4,9 @@ export type { CheckSnapshot } from "@/types/check.js";
 export const MonitorTypes = ["http", "ping", "pagespeed", "hardware", "docker", "port", "game", "unknown"] as const;
 export type MonitorType = (typeof MonitorTypes)[number];
 
-export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance"] as const;
+export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
 export type MonitorStatus = (typeof MonitorStatuses)[number];
 
-export interface MonitorThresholds {
-	usage_cpu?: number;
-	usage_memory?: number;
-	usage_disk?: number;
-	usage_temperature?: number;
-}
-
 export type MonitorMatchMethod = "equal" | "include" | "regex" | "";
 
 export interface Monitor {
@@ -39,12 +32,14 @@ export interface Monitor {
 	uptimePercentage?: number;
 	notifications: string[];
 	secret?: string;
-	thresholds?: MonitorThresholds;
-	alertThreshold: number;
 	cpuAlertThreshold: number;
+	cpuAlertCounter: number;
 	memoryAlertThreshold: number;
+	memoryAlertCounter: number;
 	diskAlertThreshold: number;
+	diskAlertCounter: number;
 	tempAlertThreshold: number;
+	tempAlertCounter: number;
 	selectedDisks: string[];
 	gameId?: string;
 	group: string | null;
@@ -60,6 +55,7 @@ export interface MonitorsSummary {
 	pausedMonitors: number;
 	initializingMonitors: number;
 	maintenanceMonitors: number;
+	breachedMonitors: number;
 }
 
 export interface MonitorsWithChecksByTeamIdResult {

server/src/types/network.ts

@@ -110,4 +110,10 @@ export type StatusChangeResult = {
 	prevStatus: MonitorStatus;
 	code: number;
 	timestamp: number;
+	thresholdBreaches?: {
+		cpu: boolean;
+		memory: boolean;
+		disk: boolean;
+		temp: boolean;
+	};
 };