diff --git a/client/src/Components/v2/design-elements/StatusBox.tsx b/client/src/Components/v2/design-elements/StatusBox.tsx index 1be76988b..73a881736 100644 --- a/client/src/Components/v2/design-elements/StatusBox.tsx +++ b/client/src/Components/v2/design-elements/StatusBox.tsx @@ -139,6 +139,17 @@ export const InitializingStatusBox = ({ n }: { n: number }) => { /> ); }; +export const BreachedStatusBox = ({ n }: { n: number }) => { + const theme = useTheme(); + const { t } = useTranslation(); + return ( + + ); +}; export const TotalChecksBox = ({ n }: { n: number }) => { const theme = useTheme(); diff --git a/client/src/Components/v2/design-elements/StatusLabel.tsx b/client/src/Components/v2/design-elements/StatusLabel.tsx index cf404421e..19bb8d157 100644 --- a/client/src/Components/v2/design-elements/StatusLabel.tsx +++ b/client/src/Components/v2/design-elements/StatusLabel.tsx @@ -21,6 +21,8 @@ export const StatusLabel = ({ status, sx }: { status: MonitorStatus; sx?: SxProp return t("pages.common.monitors.status.up"); } else if (status === "down") { return t("pages.common.monitors.status.down"); + } else if (status === "breached") { + return t("pages.common.monitors.status.breached"); } else if (status === "maintenance") { return t("pages.common.monitors.status.maintenance"); } else if (status === "paused") { diff --git a/client/src/Components/v2/monitors/HeaderMonitorsSummary.tsx b/client/src/Components/v2/monitors/HeaderMonitorsSummary.tsx index b2a93a807..38a6adfde 100644 --- a/client/src/Components/v2/monitors/HeaderMonitorsSummary.tsx +++ b/client/src/Components/v2/monitors/HeaderMonitorsSummary.tsx @@ -3,6 +3,7 @@ import { DownStatusBox, PausedStatusBox, InitializingStatusBox, + BreachedStatusBox, } from "@/Components/v2/design-elements"; import Stack from "@mui/material/Stack"; @@ -11,9 +12,13 @@ import { useTheme } from "@mui/material"; interface MonitorsSummaryProps { summary: MonitorsSummary | null; + showBreached?: boolean; } -export const HeaderMonitorsSummary = ({ summary }: MonitorsSummaryProps) => { +export const HeaderMonitorsSummary = ({ + summary, + showBreached = false, +}: MonitorsSummaryProps) => { const theme = useTheme(); return ( { > + {showBreached && } diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 153cf9f19..633fcb5c9 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -83,10 +83,10 @@ export const useMonitorForm = ({ type: "hardware", url: data?.url || "", secret: data?.secret || "", - cpuAlertThreshold: data?.cpuAlertThreshold ?? 80, - memoryAlertThreshold: data?.memoryAlertThreshold ?? 80, - diskAlertThreshold: data?.diskAlertThreshold ?? 80, - tempAlertThreshold: data?.tempAlertThreshold ?? 80, + cpuAlertThreshold: data?.cpuAlertThreshold ?? 100, + memoryAlertThreshold: data?.memoryAlertThreshold ?? 100, + diskAlertThreshold: data?.diskAlertThreshold ?? 100, + tempAlertThreshold: data?.tempAlertThreshold ?? 100, selectedDisks: data?.selectedDisks || [], }; break; diff --git a/client/src/Pages/Infrastructure/Monitors/index.tsx b/client/src/Pages/Infrastructure/Monitors/index.tsx index 01de1c09c..8d1ef1a3a 100644 --- a/client/src/Pages/Infrastructure/Monitors/index.tsx +++ b/client/src/Pages/Infrastructure/Monitors/index.tsx @@ -133,7 +133,10 @@ const InfrastructureMonitors = () => { isLoading={isLoading} isAdmin={isAdmin} /> - + { if (status === "down") { return "error"; } + if (status === "breached") { + return "error"; + } return "warning"; }; diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 8e18d8e2a..d8a46fb7c 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -387,6 +387,7 @@ }, "status": { "down": "down", + "breached": "breached", "initializing": "initializing", "maintenance": "maintenance", "paused": "paused", diff --git a/server/src/config/services.ts b/server/src/config/services.ts index f88bc2584..67c16ff6a 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -182,6 +182,7 @@ export const initializeServices = async ({ discordProvider, pagerDutyProvider, matrixProvider, + settingsService, logger ); diff --git a/server/src/db/models/Monitor.ts b/server/src/db/models/Monitor.ts index 9111ce47f..a459cdf3e 100644 --- a/server/src/db/models/Monitor.ts +++ b/server/src/db/models/Monitor.ts @@ -1,5 +1,5 @@ import { Schema, model, Types, type UpdateQuery } from "mongoose"; -import type { Monitor, MonitorMatchMethod, MonitorThresholds, CheckSnapshot } from "@/types/monitor.js"; +import type { Monitor, MonitorMatchMethod, CheckSnapshot } from "@/types/monitor.js"; import { MonitorTypes, MonitorStatuses } from "@/types/monitor.js"; type CheckSnapshotDocument = Omit & { createdAt: Date }; @@ -13,7 +13,6 @@ type MonitorDocumentBase = Omit< notifications: Types.ObjectId[]; selectedDisks: string[]; matchMethod?: MonitorMatchMethod; - thresholds?: MonitorThresholds; }; interface MonitorDocument extends MonitorDocumentBase { @@ -24,16 +23,6 @@ interface MonitorDocument extends MonitorDocumentBase { updatedAt: Date; } -const thresholdsSchema = new Schema( - { - usage_cpu: { type: Number }, - usage_memory: { type: Number }, - usage_disk: { type: Number }, - usage_temperature: { type: Number }, - }, - { _id: false } -); - const checkSnapshotSchema = new Schema( { id: { type: String, required: true }, @@ -132,36 +121,37 @@ const MonitorSchema = new Schema( secret: { type: String, }, - thresholds: { - type: thresholdsSchema, + cpuAlertThreshold: { + type: Number, + default: 100, }, - alertThreshold: { + cpuAlertCounter: { type: Number, default: 5, }, - cpuAlertThreshold: { - type: Number, - default: function () { - return this.alertThreshold; - }, - }, memoryAlertThreshold: { type: Number, - default: function () { - return this.alertThreshold; - }, + default: 100, + }, + memoryAlertCounter: { + type: Number, + default: 5, }, diskAlertThreshold: { type: Number, - default: function () { - return this.alertThreshold; - }, + default: 100, + }, + diskAlertCounter: { + type: Number, + default: 5, }, tempAlertThreshold: { type: Number, - default: function () { - return this.alertThreshold; - }, + default: 100, + }, + tempAlertCounter: { + type: Number, + default: 5, }, selectedDisks: { type: [String], @@ -189,33 +179,6 @@ const MonitorSchema = new Schema( } ); -MonitorSchema.pre("save", function (next) { - if (!this.cpuAlertThreshold || this.isModified("alertThreshold")) { - this.cpuAlertThreshold = this.alertThreshold; - } - if (!this.memoryAlertThreshold || this.isModified("alertThreshold")) { - this.memoryAlertThreshold = this.alertThreshold; - } - if (!this.diskAlertThreshold || this.isModified("alertThreshold")) { - this.diskAlertThreshold = this.alertThreshold; - } - if (!this.tempAlertThreshold || this.isModified("alertThreshold")) { - this.tempAlertThreshold = this.alertThreshold; - } - next(); -}); - -MonitorSchema.pre("findOneAndUpdate", function (next) { - const update = this.getUpdate() as UpdateQuery | null; - if (update && !Array.isArray(update) && update.alertThreshold !== undefined) { - update.cpuAlertThreshold = update.alertThreshold; - update.memoryAlertThreshold = update.alertThreshold; - update.diskAlertThreshold = update.alertThreshold; - update.tempAlertThreshold = update.alertThreshold; - } - next(); -}); - MonitorSchema.index({ teamId: 1, type: 1 }); const MonitorModel = model("Monitor", MonitorSchema); diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index 65d93421e..c6f3266a6 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -252,13 +252,28 @@ class MongoMonitorsRepository implements IMonitorsRepository { $cond: [{ $eq: ["$status", "maintenance"] }, 1, 0], }, }, + breachedMonitors: { + $sum: { + $cond: [{ $eq: ["$status", "breached"] }, 1, 0], + }, + }, }, }, { $project: { _id: 0 } }, ]; const [summary] = await MonitorModel.aggregate(pipeline); - return summary ?? { totalMonitors: 0, upMonitors: 0, downMonitors: 0, pausedMonitors: 0, initializingMonitors: 0, maintenanceMonitors: 0 }; + return ( + summary ?? { + totalMonitors: 0, + upMonitors: 0, + downMonitors: 0, + pausedMonitors: 0, + initializingMonitors: 0, + maintenanceMonitors: 0, + breachedMonitors: 0, + } + ); }; findGroupsByTeamId = async (teamId: string): Promise => { @@ -317,12 +332,14 @@ class MongoMonitorsRepository implements IMonitorsRepository { uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, secret: doc.secret ?? undefined, - thresholds: doc.thresholds ?? undefined, - alertThreshold: doc.alertThreshold, cpuAlertThreshold: doc.cpuAlertThreshold, + cpuAlertCounter: doc.cpuAlertCounter, memoryAlertThreshold: doc.memoryAlertThreshold, + memoryAlertCounter: doc.memoryAlertCounter, diskAlertThreshold: doc.diskAlertThreshold, + diskAlertCounter: doc.diskAlertCounter, tempAlertThreshold: doc.tempAlertThreshold, + tempAlertCounter: doc.tempAlertCounter, selectedDisks: doc.selectedDisks ?? [], gameId: doc.gameId ?? undefined, group: doc.group ?? null, @@ -370,12 +387,14 @@ class MongoMonitorsRepository implements IMonitorsRepository { uptimePercentage: doc.uptimePercentage ?? undefined, notifications: notificationIds, secret: doc.secret ?? undefined, - thresholds: doc.thresholds ?? undefined, - alertThreshold: doc.alertThreshold, cpuAlertThreshold: doc.cpuAlertThreshold, + cpuAlertCounter: doc.cpuAlertCounter, memoryAlertThreshold: doc.memoryAlertThreshold, + memoryAlertCounter: doc.memoryAlertCounter, diskAlertThreshold: doc.diskAlertThreshold, + diskAlertCounter: doc.diskAlertCounter, tempAlertThreshold: doc.tempAlertThreshold, + tempAlertCounter: doc.tempAlertCounter, selectedDisks: doc.selectedDisks ?? [], gameId: doc.gameId ?? undefined, group: doc.group ?? null, diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index 881f98f56..bcb17d3a5 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -4,6 +4,7 @@ import { AppError } from "@/utils/AppError.js"; import { ParseBoolean } from "@/utils/utils.js"; import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from "@/repositories/index.js"; import type { Incident } from "@/types/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; const dateRangeLookup: Record = { recent: new Date(new Date().setHours(new Date().getHours() - 2)), @@ -43,10 +44,14 @@ class IncidentService { return IncidentService.SERVICE_NAME; } - handleIncident = async (monitor: Monitor, code: number): Promise => { + handleIncident = async (monitor: Monitor, code: number, decision: MonitorActionDecision): Promise => { + if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) { + return null; + } + const activeIncident = await this.incidentsRepository.findActiveByMonitorId(monitor.id, monitor.teamId); - // Monitor is down, create an incident - if (monitor.status === "down") { + + if (decision.shouldCreateIncident) { if (activeIncident) { return activeIncident; } else { @@ -61,14 +66,17 @@ class IncidentService { } } - // Monitor is up, resolve active incidents - if (!activeIncident) { - return null; + if (decision.shouldResolveIncident) { + if (!activeIncident) { + return null; + } + activeIncident.status = false; + activeIncident.endTime = Date.now().toString(); + activeIncident.resolutionType = "automatic"; + return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); } - activeIncident.status = false; - activeIncident.endTime = Date.now().toString(); - activeIncident.resolutionType = "automatic"; - return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident); + + return null; }; resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => { diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index 6c9e9264a..274318ad7 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -2,9 +2,24 @@ const SERVICE_NAME = "JobQueueHelper"; import type { Monitor } from "@/types/monitor.js"; import { AppError } from "@/utils/AppError.js"; import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js"; +import type { StatusChangeResult, MonitorStatusResponse, HardwareStatusPayload, MonitorStatus } from "@/types/index.js"; import IncidentService from "@/service/business/incidentService.js"; import { IMaintenanceWindowsRepository, IMonitorsRepository } from "@/repositories/index.js"; +export interface MonitorActionDecision { + shouldCreateIncident: boolean; + shouldResolveIncident: boolean; + shouldSendNotification: boolean; + incidentReason: "status_down" | "threshold_breach" | null; + notificationReason: "status_change" | "threshold_breach" | null; + thresholdBreaches?: { + cpu?: boolean; + memory?: boolean; + disk?: boolean; + temp?: boolean; + }; +} + class SuperSimpleQueueHelper { static SERVICE_NAME = SERVICE_NAME; @@ -100,10 +115,12 @@ class SuperSimpleQueueHelper { // Step 4. Update monitor status const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); - // Step 5 handle notifications (best effort, continue even in event of failure, don't wait) - this.notificationsService - .handleNotifications(statusChangeResult.monitor, status, statusChangeResult.prevStatus, statusChangeResult.statusChanged) - .catch((error: any) => { + // Step 5. Get decisions + const decision = this.evaluateMonitorAction(statusChangeResult); + + // Step 6. Handle notifications (best effort, continue even in event of failure, don't wait) + if (decision.shouldSendNotification) { + this.notificationsService.handleNotifications(statusChangeResult.monitor, status, decision).catch((error: any) => { this.logger.error({ message: error.message, service: SERVICE_NAME, @@ -112,19 +129,18 @@ class SuperSimpleQueueHelper { stack: error.stack, }); }); - - // Step 6. Handle incidents (best effort, don't wait) - if (statusChangeResult.statusChanged) { - this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code).catch((error: any) => { - this.logger.warn({ - message: error.message, - service: SERVICE_NAME, - method: "getMonitorJob", - details: `Error handling incident for job ${monitor.id}: ${error.message}`, - stack: error.stack, - }); - }); } + + // Step 7. Handle incidents (best effort, don't wait) + this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision).catch((error: any) => { + this.logger.warn({ + message: error.message, + service: SERVICE_NAME, + method: "getMonitorJob", + details: `Error handling incident for job ${monitor.id}: ${error.message}`, + stack: error.stack, + }); + }); } catch (error: any) { this.logger.warn({ message: error.message, @@ -166,6 +182,45 @@ class SuperSimpleQueueHelper { }, false); return maintenanceWindowIsActive; } + + private evaluateMonitorAction(statusChangeResult: StatusChangeResult): MonitorActionDecision { + const { monitor, statusChanged, prevStatus } = statusChangeResult; + + // Initialize result + const decision: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }; + + // Simplified logic: Just check status changes + if (!statusChanged) { + return decision; + } + + if (monitor.status === "down") { + // Monitor went down (unreachable) + decision.shouldCreateIncident = true; + decision.shouldSendNotification = true; + decision.incidentReason = "status_down"; + decision.notificationReason = "status_change"; + } else if (monitor.status === "breached") { + // Hardware monitor exceeded thresholds + decision.shouldCreateIncident = true; + decision.shouldSendNotification = true; + decision.incidentReason = "threshold_breach"; + decision.notificationReason = "threshold_breach"; + } else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) { + // Monitor recovered from down or breached state + decision.shouldResolveIncident = true; + decision.shouldSendNotification = true; + decision.notificationReason = "status_change"; + } + + return decision; + } } export default SuperSimpleQueueHelper; diff --git a/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts b/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts index 2d363d6b3..21a221c9c 100644 --- a/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts +++ b/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts @@ -1,6 +1,13 @@ import type { Monitor, Notification, Alert, MonitorStatusResponse } from "@/types/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; export interface INotificationProvider { - sendAlert: (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => Promise; + sendAlert: ( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ) => Promise; sendTestAlert(notification: Notification): Promise; } diff --git a/server/src/service/infrastructure/notificationProviders/discord.ts b/server/src/service/infrastructure/notificationProviders/discord.ts index ad1f2f28a..560c18aae 100644 --- a/server/src/service/infrastructure/notificationProviders/discord.ts +++ b/server/src/service/infrastructure/notificationProviders/discord.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "DiscordProvider"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, buildDiscordBody, getTestMessage } from "@/service/infrastructure/notificationProviders/utils.js"; import got from "got"; @@ -10,15 +11,31 @@ export class DiscordProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { - const { discordPayload } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildDiscordBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format + const { discordPayload } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse); return discordPayload; }; - sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + sendAlert = async ( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ) => { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision); } else { body = buildDiscordBody(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/email.ts b/server/src/service/infrastructure/notificationProviders/email.ts index 32ccf1859..fd15034d1 100644 --- a/server/src/service/infrastructure/notificationProviders/email.ts +++ b/server/src/service/infrastructure/notificationProviders/email.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "EmailProvider"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, buildHardwareEmail, buildEmail, buildTestEmail } from "@/service/infrastructure/notificationProviders/utils.js"; export class EmailProvider implements INotificationProvider { @@ -12,21 +13,48 @@ export class EmailProvider implements INotificationProvider { this.logger = logger; } - private buildHardwareEmail = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { - const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); + private buildHardwareEmail = async ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard email format + if (decision.notificationReason === "status_change") { + return await buildEmail(this.emailService, monitor); + } + // For threshold breaches, use hardware alert format + const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse); const html = buildHardwareEmail(this.emailService, monitor, alertsToSend); return html; }; - async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise { + async sendAlert( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ): Promise { // For grouped notifications (identified by ":" in name), customize subject to indicate multiple services. // Example: "2 services: Service A, Service B" becomes "Alert: 2 services are down" const isGroupedNotification = monitor.name.includes(":"); - const subject = isGroupedNotification ? `Alert: ${monitor.name} are down` : `Monitor ${monitor.name} is down`; + + // Build subject based on notification reason and monitor status + let subject: string; + if (isGroupedNotification) { + subject = `Alert: ${monitor.name} are down`; + } else if (decision.notificationReason === "threshold_breach") { + subject = `Monitor ${monitor.name} threshold breached`; + } else if (monitor.status === "up") { + subject = `Monitor ${monitor.name} is back up`; + } else { + subject = `Monitor ${monitor.name} is down`; + } let html; if (monitor.type === "hardware") { - html = this.buildHardwareEmail(monitor, monitorStatusResponse); + html = await this.buildHardwareEmail(clientHost, monitor, monitorStatusResponse, decision); } else { html = await buildEmail(this.emailService, monitor); } diff --git a/server/src/service/infrastructure/notificationProviders/matrix.ts b/server/src/service/infrastructure/notificationProviders/matrix.ts index 8914dbf80..f2543886d 100644 --- a/server/src/service/infrastructure/notificationProviders/matrix.ts +++ b/server/src/service/infrastructure/notificationProviders/matrix.ts @@ -1,10 +1,11 @@ const SERVICE_NAME = "MatrixProvider"; import got from "got"; import type { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { Notification, Monitor, MonitorStatusResponse } from "@/types/index.js"; import { buildHardwareAlerts, - buildHardwareWebhookBody, + buildHardwareNotificationMessage, buildWebhookBody, getTestMessage, } from "@/service/infrastructure/notificationProviders/utils.js"; @@ -15,9 +16,19 @@ export class MatrixProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { - const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format + const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse); + const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor); return body; }; @@ -26,12 +37,18 @@ export class MatrixProvider implements INotificationProvider { return body; }; - sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + sendAlert = async ( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ) => { const { homeserverUrl, accessToken, roomId } = notification; let content; if (monitor.type === "hardware") { - content = this.getHardwareContent(monitor, monitorStatusResponse); + content = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision); } else { content = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/pagerduty.ts b/server/src/service/infrastructure/notificationProviders/pagerduty.ts index f65e50c68..291cd7de5 100644 --- a/server/src/service/infrastructure/notificationProviders/pagerduty.ts +++ b/server/src/service/infrastructure/notificationProviders/pagerduty.ts @@ -1,10 +1,11 @@ const SERVICE_NAME = "PagerDutyProvider"; import got from "got"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { INotificationProvider } from "@/service/index.js"; import { buildHardwareAlerts, - buildHardwareWebhookBody, + buildHardwareNotificationMessage, buildWebhookBody, getTestMessage, } from "@/service/infrastructure/notificationProviders/utils.js"; @@ -15,9 +16,19 @@ export class PagerDutyProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { - const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format + const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse); + const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor); return body; }; @@ -26,10 +37,16 @@ export class PagerDutyProvider implements INotificationProvider { return body; }; - async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise { + async sendAlert( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ): Promise { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision); } else { body = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/slack.ts b/server/src/service/infrastructure/notificationProviders/slack.ts index 2aeb442eb..f99af48d4 100644 --- a/server/src/service/infrastructure/notificationProviders/slack.ts +++ b/server/src/service/infrastructure/notificationProviders/slack.ts @@ -1,9 +1,10 @@ const SERVICE_NAME = "SlackProvider"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, - buildHardwareWebhookBody, + buildHardwareNotificationMessage, buildWebhookBody, getTestMessage, } from "@/service/infrastructure/notificationProviders/utils.js"; @@ -15,9 +16,19 @@ export class SlackProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { - const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format + const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse); + const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor); return body; }; @@ -26,10 +37,16 @@ export class SlackProvider implements INotificationProvider { return body; }; - async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise { + async sendAlert( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ): Promise { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision); } else { body = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/utils.ts b/server/src/service/infrastructure/notificationProviders/utils.ts index 2fe22e834..72797c1d0 100644 --- a/server/src/service/infrastructure/notificationProviders/utils.ts +++ b/server/src/service/infrastructure/notificationProviders/utils.ts @@ -23,17 +23,26 @@ export const buildHardwareAlerts = ( monitor: Monitor, networkResponse: MonitorStatusResponse ): { alertsToSend: string[]; discordPayload: any } => { - const thresholds = monitor.thresholds || {}; - const { usage_cpu: cpuThreshold = -1, usage_memory: memoryThreshold = -1, usage_disk: diskThreshold = -1 } = thresholds; + // Thresholds are stored as percentages (0-100), convert to decimal (0-1) for comparison + const cpuThreshold = monitor.cpuAlertThreshold !== undefined ? monitor.cpuAlertThreshold / 100 : -1; + const memoryThreshold = monitor.memoryAlertThreshold !== undefined ? monitor.memoryAlertThreshold / 100 : -1; + const diskThreshold = monitor.diskAlertThreshold !== undefined ? monitor.diskAlertThreshold / 100 : -1; + const tempThreshold = monitor.tempAlertThreshold !== undefined ? monitor.tempAlertThreshold : -1; const payload = networkResponse?.payload as HardwareStatusPayload; const metrics = payload.data || {}; - const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics; + const { cpu = {}, memory = {}, disk = [] } = metrics; + const cpuUsage = cpu.usage_percent ?? -1; + const memoryUsage = memory.usage_percent ?? -1; + // Get max temperature from CPU temperature sensors array + const temps = cpu.temperature ?? []; + const maxTemp = temps.length > 0 ? Math.max(...temps) : -1; const alerts: Record = { cpu: cpuThreshold !== -1 && cpuUsage > cpuThreshold ? true : false, memory: memoryThreshold !== -1 && memoryUsage > memoryThreshold ? true : false, disk: disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold) ?? false, + temp: tempThreshold !== -1 && maxTemp > tempThreshold ? true : false, }; const alertsToSend = []; @@ -43,6 +52,7 @@ export const buildHardwareAlerts = ( { name: "URL", value: monitor.url, inline: false }, ]; const goToIncidentField = { name: `Go to incident`, value: `${clientHost}/infrastructure/${monitor.id}` }; + const formatDiscordAlert = { cpu: () => ({ title: "CPU alert", @@ -75,78 +85,42 @@ export const buildHardwareAlerts = ( goToIncidentField, ], }), + + temp: () => ({ + title: "Temperature alert", + description: `Your current temperature (${maxTemp.toFixed(0)}°C) is above your threshold (${tempThreshold.toFixed(0)}°C)`, + color: 15548997, + fields: [...monitorInfoFields, goToIncidentField], + footer: { text: "Checkmate" }, + }), }; - const alertTypes = ["cpu", "memory", "disk"] as const; - const alertThresholdKeyMap: Record<(typeof alertTypes)[number], "cpuAlertThreshold" | "memoryAlertThreshold" | "diskAlertThreshold"> = { - cpu: "cpuAlertThreshold", - memory: "memoryAlertThreshold", - disk: "diskAlertThreshold", - }; + + const alertTypes = ["cpu", "memory", "disk", "temp"] as const; + for (const type of alertTypes) { - const thresholdKey = alertThresholdKeyMap[type]; - // Iterate over each alert type to see if any need to be decremented if (alerts[type] === true) { - const nextValue = ((monitor[thresholdKey] ?? monitor.alertThreshold) as number) - 1; - monitor[thresholdKey] = nextValue; // Decrement threshold if an alert is triggered - - if (monitor[thresholdKey] <= 0) { - // If threshold drops below 0, reset and send notification - monitor[thresholdKey] = monitor.alertThreshold; - - const formatAlert = { - cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`, - memory: () => - `Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`, - disk: () => - `Your current disk usage: ${disk - .map((d, idx) => `(Disk${idx}: ${(d?.usage_percent ?? 0 * 100).toFixed(0)}%)`) - .join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`, - }; - alertsToSend.push(formatAlert[type]()); - discordEmbeds.push(formatDiscordAlert[type]()); - } + const formatAlert = { + cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`, + memory: () => + `Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`, + disk: () => + `Your current disk usage: ${disk.map((d, idx) => `(Disk${idx}: ${(d?.usage_percent ?? 0 * 100).toFixed(0)}%)`).join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`, + temp: () => `Your current temperature (${maxTemp.toFixed(0)}°C) is above your threshold (${tempThreshold.toFixed(0)}°C)`, + }; + alertsToSend.push(formatAlert[type]()); + discordEmbeds.push(formatDiscordAlert[type]()); } } + const discordPayload = discordEmbeds.length ? { embeds: discordEmbeds } : null; return { alertsToSend, discordPayload }; }; -export const buildHardwareNotificationMessage = (clientHost: string, alerts: any, monitor: Monitor) => { +export const buildHardwareNotificationMessage = (clientHost: string, alerts: string[], monitor: Monitor): string => { const alertsHeader = [`Monitor: ${monitor.name}`, `URL: ${monitor.url}`]; const alertFooter = [`Go to incident: ${clientHost}/infrastructure/${monitor.id}`]; const alertText = alerts.length > 0 ? [...alertsHeader, ...alerts, ...alertFooter] : []; - return alertText.map((alert) => alert).join("\n"); -}; - -export const buildHardwareWebhookBody = (alerts: string[], monitor: Monitor): string => { - const content = alerts.map((alert) => alert).join("\n"); - return content; -}; - -export const shouldSendHardwareAlert = (monitor: Monitor, networkResponse: MonitorStatusResponse): boolean => { - const thresholds = monitor.thresholds || {}; - const { usage_cpu: cpuThreshold = -1, usage_memory: memoryThreshold = -1, usage_disk: diskThreshold = -1 } = thresholds; - - const payload = networkResponse?.payload as HardwareStatusPayload; - const metrics = payload.data || {}; - const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics; - - const cpuBreach = cpuThreshold !== -1 && cpuUsage > cpuThreshold; - if (cpuBreach && ((monitor.cpuAlertThreshold ?? monitor.alertThreshold) as number) - 1 <= 0) { - return true; - } - - const memoryBreach = memoryThreshold !== -1 && memoryUsage > memoryThreshold; - if (memoryBreach && ((monitor.memoryAlertThreshold ?? monitor.alertThreshold) as number) - 1 <= 0) { - return true; - } - - const diskBreach = disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold); - if (diskBreach && ((monitor.diskAlertThreshold ?? monitor.alertThreshold) as number) - 1 <= 0) { - return true; - } - - return false; + return alertText.join("\n"); }; export const buildWebhookBody = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { diff --git a/server/src/service/infrastructure/notificationProviders/webhook.ts b/server/src/service/infrastructure/notificationProviders/webhook.ts index 9e743ae53..17d871950 100644 --- a/server/src/service/infrastructure/notificationProviders/webhook.ts +++ b/server/src/service/infrastructure/notificationProviders/webhook.ts @@ -1,9 +1,10 @@ const SERVICE_NAME = "WebhookProvider"; import type { Monitor, Alert, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, - buildHardwareWebhookBody, + buildHardwareNotificationMessage, buildWebhookBody, getTestMessage, } from "@/service/infrastructure/notificationProviders/utils.js"; @@ -15,9 +16,19 @@ export class WebhookProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { - const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format + const { alertsToSend } = buildHardwareAlerts(clientHost, monitor, monitorStatusResponse); + const body = buildHardwareNotificationMessage(clientHost, alertsToSend, monitor); return body; }; @@ -26,10 +37,16 @@ export class WebhookProvider implements INotificationProvider { return body; }; - sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + sendAlert = async ( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ) => { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent(clientHost, monitor, monitorStatusResponse, decision); } else { body = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index 531014a7a..5f057f0c5 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -1,7 +1,9 @@ import type { HardwareStatusPayload, Monitor, MonitorStatusResponse, Notification, MonitorStatus } from "@/types/index.js"; -import { shouldSendHardwareAlert } from "@/service/infrastructure/notificationProviders/utils.js"; import { IMonitorsRepository, INotificationsRepository } from "@/repositories/index.js"; import { INotificationProvider } from "./notificationProviders/INotificationProvider.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; +import type { ISettingsService } from "@/service/system/settingsService.js"; +import { ILogger } from "@/utils/logger.js"; export interface INotificationsService { createNotification: (notificationData: Partial) => Promise; @@ -9,12 +11,7 @@ export interface INotificationsService { findNotificationsByTeamId: (teamId: string) => Promise; updateById(id: string, teamId: string, updateData: Partial): Promise; deleteById: (id: string, teamId: string) => Promise; - handleNotifications: ( - monitor: Monitor, - monitorStatusResponse: MonitorStatusResponse, - prevStatus: MonitorStatus, - statusChanged: boolean - ) => Promise; + handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; sendTestNotification: (notification: Notification) => Promise; testAllNotifications: (notificationIds: string[]) => Promise; @@ -33,19 +30,8 @@ export class NotificationsService implements INotificationsService { private discordProvider: INotificationProvider; private pagerDutyProvider: INotificationProvider; private matrixProvider: INotificationProvider; - private logger: any; - - // Email grouping (batching) configuration - private emailGroupingWindowMs: number; - private pendingEmailGroups: Map< - string, - { - monitors: Monitor[]; - statusResponses: MonitorStatusResponse[]; - timer: ReturnType; - createdAt: number; - } - >; + private logger: ILogger; + private settingsService: ISettingsService; constructor( notificationsRepository: INotificationsRepository, @@ -56,6 +42,7 @@ export class NotificationsService implements INotificationsService { discordProvider: INotificationProvider, pagerDutyProvider: INotificationProvider, matrixProvider: INotificationProvider, + settingsService: ISettingsService, logger: any ) { this.notificationsRepository = notificationsRepository; @@ -66,57 +53,42 @@ export class NotificationsService implements INotificationsService { this.discordProvider = discordProvider; this.pagerDutyProvider = pagerDutyProvider; this.matrixProvider = matrixProvider; + this.settingsService = settingsService; this.logger = logger; - - // Configure email grouping window (in milliseconds). - // When > 0, multiple DOWN events for monitors that share the same - // email notification within this window will be batched into a single email. - const rawGroupingWindow = process.env.NOTIFICATION_GROUP_WINDOW_MS ?? process.env.NOTIFICATION_GROUP_WINDOW_SECONDS; - let groupingWindowMs = 0; - if (rawGroupingWindow) { - const parsed = Number(rawGroupingWindow); - if (!Number.isNaN(parsed) && parsed > 0) { - // If value looks like seconds (small number), convert to ms. - // This allows either milliseconds (e.g. 60000) or seconds (e.g. 60). - groupingWindowMs = parsed <= 300 ? parsed * 1000 : parsed; - } - } - this.emailGroupingWindowMs = groupingWindowMs; - this.pendingEmailGroups = new Map(); } - private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise => { + private send = async ( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ): Promise => { + const settings = this.settingsService.getSettings(); + const clientHost = settings.clientHost || "Host not defined"; + switch (notification.type) { case "email": - return await this.emailProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.emailProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost); case "slack": - return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost); case "discord": - return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost); case "pager_duty": - return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost); case "matrix": - return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost); case "webhook": - return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse, decision, clientHost); default: return false; } }; - private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { const notificationIds = monitor.notifications ?? []; const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds); - const tasks = notifications.map((notification) => { - // Only group emails, only for DOWN transitions, and only if a window is configured. - if (notification.type === "email" && this.emailGroupingWindowMs > 0 && monitorStatusResponse.status === false) { - return this.queueGroupedEmailNotification(notification, monitor, monitorStatusResponse); - } - - // For all other cases (UP notifications or non-email channels), send immediately. - return this.send(notification, monitor, monitorStatusResponse); - }); + const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision)); const outcomes = await Promise.all(tasks); const succeeded = outcomes.filter(Boolean).length; @@ -125,136 +97,21 @@ export class NotificationsService implements INotificationsService { this.logger.warn({ message: `Notification send completed with ${succeeded} success, ${failed} failure(s)`, service: SERVICE_NAME, - method: "getMonitorJob", + method: "sendNotifications", }); } - // Return true if all notificaitons succeeded + // Return true if all notifications succeeded return succeeded === notifications.length; }; - /** - * Queue a DOWN email notification to be potentially grouped with other - * DOWN events for the same email notification within the configured window. - * - * This method returns immediately; the actual email is sent asynchronously - * when the grouping window expires. - */ - private queueGroupedEmailNotification = async ( - notification: Notification, - monitor: Monitor, - monitorStatusResponse: MonitorStatusResponse - ): Promise => { - // If grouping is disabled, fallback to immediate send. - if (this.emailGroupingWindowMs <= 0) { - return await this.send(notification, monitor, monitorStatusResponse); - } - - const key = notification.id; - const now = Date.now(); - const existingGroup = this.pendingEmailGroups.get(key); - - if (!existingGroup) { - // Create a new group and schedule a flush after the window expires. - const timer = setTimeout(async () => { - const group = this.pendingEmailGroups.get(key); - if (!group) return; - - this.pendingEmailGroups.delete(key); - - try { - await this.flushEmailGroup(notification, group.monitors, group.statusResponses); - } catch (error: any) { - this.logger.error({ - message: error?.message, - service: SERVICE_NAME, - method: "flushEmailGroup", - stack: error?.stack, - }); - } - }, this.emailGroupingWindowMs); - - this.pendingEmailGroups.set(key, { - monitors: [monitor], - statusResponses: [monitorStatusResponse], - timer, - createdAt: now, - }); - } else { - // Append to existing group. - existingGroup.monitors.push(monitor); - existingGroup.statusResponses.push(monitorStatusResponse); - } - - // Consider queueing as "succeeded" from the caller's perspective. - return true; - }; - - /** - * Flush a grouped set of DOWN events into a single email. - * - * To avoid changing email templates, we construct a synthetic Monitor - * whose name concisely lists all affected services. The existing - * `serverIsDownTemplate` is then reused. - * - * @param notification The email notification to send to - * @param monitors Array of monitors that went down - * @param statusResponses Array of status responses (parallel to monitors) - * @returns true if email was sent successfully, false otherwise - */ - private flushEmailGroup = async (notification: Notification, monitors: Monitor[], statusResponses: MonitorStatusResponse[]): Promise => { - if (!monitors.length || !statusResponses.length) { + handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + // Early return if no notification should be sent + if (!decision.shouldSendNotification) { return false; } - // Build a combined monitor name listing all affected services. - // Example: "Service A, Service B" (2 services) or "Service A" (1 service) - const uniqueNames = Array.from(new Set(monitors.map((m) => m.name))); - const servicesCount = uniqueNames.length; - const servicesList = uniqueNames.join(", "); - - const combinedName = servicesCount === 1 ? servicesList : `${servicesCount} services: ${servicesList}`; - - // Use the first monitor as a base for URL and other fields. - const baseMonitor = monitors[0]!; - const baseStatus = statusResponses[0]!; - - // Create a shallow clone so we don't mutate the original entity. - // This preserves monitor properties while overriding the name for grouped display. - const syntheticMonitor: Monitor = { - ...baseMonitor, - name: combinedName, - }; - - // Reuse existing email provider to send grouped notification. - return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus); - }; - - handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, prevStatus: MonitorStatus, statusChanged: boolean) => { - const { type } = monitor; - const payload = monitorStatusResponse.payload as HardwareStatusPayload; - // If this is a non-hardeware type monitor and status did not change, we're done - if (type !== "hardware" && statusChanged === false) return false; - // if prevStatus is undefined, monitor is resuming, we're done - if (type !== "hardware" && prevStatus === undefined) return false; - - // Deal with hardware thresholds - if (type === "hardware") { - const thresholds = monitor.thresholds; - - if (thresholds === undefined) return false; // No thresholds set, we're done - const metrics = payload?.data ?? null; - if (metrics === null) return false; // No metrics, we're done - - // We should send a notificaiton - - const shouldSend = shouldSendHardwareAlert(monitor, monitorStatusResponse); - if (shouldSend === false) return false; - - return await this.sendNotifications(monitor, monitorStatusResponse); - } - - // We should send a notification for non-hardware monitor status change - return await this.sendNotifications(monitor, monitorStatusResponse); + // Send notifications based on decision + return await this.sendNotifications(monitor, monitorStatusResponse, decision); }; sendTestNotification = async (notification: Notification) => { diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index 0ab47f85b..b01131bbe 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -252,7 +252,89 @@ export class StatusService implements IStatusService { statusChanged = true; } + // Evaluate hardware threshold breaches (only for hardware monitors) + let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined; + if (monitor.type === "hardware" && statusResponse.payload) { + const payload = statusResponse.payload as HardwareStatusPayload; + const metrics = payload?.data; + + if (metrics) { + // Evaluate threshold breaches + const cpuUsage = metrics.cpu?.usage_percent ?? -1; + const cpuBreach = cpuUsage !== -1 && cpuUsage > monitor.cpuAlertThreshold / 100; + + const memoryUsage = metrics.memory?.usage_percent ?? -1; + const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100; + + const diskBreach = + metrics.disk?.some((d: any) => typeof d?.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100) ?? false; + + const temps = metrics.cpu?.temperature ?? []; + const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold); + + thresholdBreaches = { + cpu: cpuBreach, + memory: memoryBreach, + disk: diskBreach, + temp: tempBreach, + }; + + // Update counters: decrement if breached, reset to 5 if not breached + if (cpuBreach) { + monitor.cpuAlertCounter = Math.max(0, monitor.cpuAlertCounter - 1); + } else { + monitor.cpuAlertCounter = 5; + } + + if (memoryBreach) { + monitor.memoryAlertCounter = Math.max(0, monitor.memoryAlertCounter - 1); + } else { + monitor.memoryAlertCounter = 5; + } + + if (diskBreach) { + monitor.diskAlertCounter = Math.max(0, monitor.diskAlertCounter - 1); + } else { + monitor.diskAlertCounter = 5; + } + + if (tempBreach) { + monitor.tempAlertCounter = Math.max(0, monitor.tempAlertCounter - 1); + } else { + monitor.tempAlertCounter = 5; + } + + // Check if any counter has reached zero (initial breach) + const anyCounterZero = + monitor.cpuAlertCounter === 0 || monitor.memoryAlertCounter === 0 || monitor.diskAlertCounter === 0 || monitor.tempAlertCounter === 0; + + const anyThresholdBreached = cpuBreach || memoryBreach || diskBreach || tempBreach; + const allThresholdsNormal = !cpuBreach && !memoryBreach && !diskBreach && !tempBreach; + + // Update monitor status based on threshold breach state + if (newStatus !== "down") { + // Don't override "down" status - service unreachable takes precedence + // Check current monitor status, not newStatus for comparison + if (anyCounterZero && anyThresholdBreached && monitor.status !== "breached") { + // Initial breach: counter hit zero, change status to breached + newStatus = "breached"; + statusChanged = true; + } else if (anyCounterZero && anyThresholdBreached && monitor.status === "breached") { + // Already breached, keep status but don't mark as changed + newStatus = "breached"; + // statusChanged remains false + } else if (allThresholdsNormal && monitor.status === "breached") { + // All thresholds returned to normal, recover from breached state + newStatus = "up"; + statusChanged = true; + } + } + } + } + + // Apply the final status monitor.status = newStatus; + const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor); return { @@ -261,6 +343,7 @@ export class StatusService implements IStatusService { prevStatus, code, timestamp: new Date().getTime(), + thresholdBreaches, }; } catch (error: any) { error.service = SERVICE_NAME; diff --git a/server/src/types/monitor.ts b/server/src/types/monitor.ts index 384e14fe0..a196aaae6 100644 --- a/server/src/types/monitor.ts +++ b/server/src/types/monitor.ts @@ -4,16 +4,9 @@ export type { CheckSnapshot } from "@/types/check.js"; export const MonitorTypes = ["http", "ping", "pagespeed", "hardware", "docker", "port", "game", "unknown"] as const; export type MonitorType = (typeof MonitorTypes)[number]; -export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance"] as const; +export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const; export type MonitorStatus = (typeof MonitorStatuses)[number]; -export interface MonitorThresholds { - usage_cpu?: number; - usage_memory?: number; - usage_disk?: number; - usage_temperature?: number; -} - export type MonitorMatchMethod = "equal" | "include" | "regex" | ""; export interface Monitor { @@ -39,12 +32,14 @@ export interface Monitor { uptimePercentage?: number; notifications: string[]; secret?: string; - thresholds?: MonitorThresholds; - alertThreshold: number; cpuAlertThreshold: number; + cpuAlertCounter: number; memoryAlertThreshold: number; + memoryAlertCounter: number; diskAlertThreshold: number; + diskAlertCounter: number; tempAlertThreshold: number; + tempAlertCounter: number; selectedDisks: string[]; gameId?: string; group: string | null; @@ -60,6 +55,7 @@ export interface MonitorsSummary { pausedMonitors: number; initializingMonitors: number; maintenanceMonitors: number; + breachedMonitors: number; } export interface MonitorsWithChecksByTeamIdResult { diff --git a/server/src/types/network.ts b/server/src/types/network.ts index 7fbfb79fe..d87f1ea4f 100644 --- a/server/src/types/network.ts +++ b/server/src/types/network.ts @@ -110,4 +110,10 @@ export type StatusChangeResult = { prevStatus: MonitorStatus; code: number; timestamp: number; + thresholdBreaches?: { + cpu: boolean; + memory: boolean; + disk: boolean; + temp: boolean; + }; };