From 584e4633d3ba69d4e378013e69b442a4ff0be79f Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 17 Feb 2026 17:42:11 +0000 Subject: [PATCH 1/3] incident details --- .../src/service/business/incidentService.ts | 86 ++++++++++++++++++- .../SuperSimpleQueueHelper.ts | 2 +- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index bcb17d3a5..67431f0ae 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -1,5 +1,6 @@ const SERVICE_NAME = "incidentService"; import type { Monitor } from "@/types/monitor.js"; +import type { MonitorStatusResponse } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { ParseBoolean } from "@/utils/utils.js"; import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from "@/repositories/index.js"; @@ -44,7 +45,12 @@ class IncidentService { return IncidentService.SERVICE_NAME; } - handleIncident = async (monitor: Monitor, code: number, decision: MonitorActionDecision): Promise => { + handleIncident = async ( + monitor: Monitor, + code: number, + decision: MonitorActionDecision, + monitorStatusResponse?: MonitorStatusResponse + ): Promise => { if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) { return null; } @@ -55,12 +61,22 @@ class IncidentService { if (activeIncident) { return activeIncident; } else { + let statusCode = code; + let message: string | undefined; + + // For threshold breaches, use 9999 status code and build descriptive message + if (decision.incidentReason === "threshold_breach") { + statusCode = 9999; + message = this.buildThresholdBreachMessage(monitor, monitorStatusResponse); + } + const incident = { monitorId: monitor.id, teamId: monitor.teamId, startTime: Date.now().toString(), status: true, - statusCode: code, + statusCode, + message, }; return await this.incidentsRepository.create(incident); } @@ -79,6 +95,72 @@ class IncidentService { return null; }; + private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string { + const breaches: string[] = []; + + if (!monitorStatusResponse?.payload || monitor.type !== "hardware") { + return "Threshold breach detected"; + } + + const payload = monitorStatusResponse.payload as { data?: any }; + const hardware = payload.data; + + if (!hardware) { + return "Threshold breach detected"; + } + + // Check CPU breach + if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && hardware.cpu?.usage_percent !== undefined) { + const cpuPercent = hardware.cpu.usage_percent * 100; + if (cpuPercent > monitor.cpuAlertThreshold) { + breaches.push(`CPU: ${cpuPercent.toFixed(1)}% (threshold: ${monitor.cpuAlertThreshold}%)`); + } + } + + // Check Memory breach + if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && hardware.memory?.usage_percent !== undefined) { + const memoryPercent = hardware.memory.usage_percent * 100; + if (memoryPercent > monitor.memoryAlertThreshold) { + breaches.push(`Memory: ${memoryPercent.toFixed(1)}% (threshold: ${monitor.memoryAlertThreshold}%)`); + } + } + + // Check Disk breach + if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(hardware.disk)) { + let maxDiskPercent = 0; + for (const disk of hardware.disk) { + if (disk.usage_percent !== undefined) { + const diskPercent = disk.usage_percent * 100; + if (diskPercent > maxDiskPercent) { + maxDiskPercent = diskPercent; + } + } + } + if (maxDiskPercent > monitor.diskAlertThreshold) { + breaches.push(`Disk: ${maxDiskPercent.toFixed(1)}% (threshold: ${monitor.diskAlertThreshold}%)`); + } + } + + // Check Temperature breach + if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && Array.isArray(hardware.cpu?.temperature)) { + let maxTemp = 0; + for (const temp of hardware.cpu.temperature) { + if (temp.current !== undefined && temp.current > maxTemp) { + maxTemp = temp.current; + } + } + if (maxTemp > monitor.tempAlertThreshold) { + breaches.push(`Temperature: ${maxTemp.toFixed(1)}°C (threshold: ${monitor.tempAlertThreshold}°C)`); + } + } + + if (breaches.length === 0) { + return "Threshold breach detected"; + } + + return breaches.join(", "); + } + resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => { try { if (!incidentId) { diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index 274318ad7..d59f3722b 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -132,7 +132,7 @@ class SuperSimpleQueueHelper { } // Step 7. Handle incidents (best effort, don't wait) - this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision).catch((error: any) => { + this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status).catch((error: any) => { this.logger.warn({ message: error.message, service: SERVICE_NAME, From 73f05ab78ea2bf092101f3257be8e335d267a0e7 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 17 Feb 2026 17:49:56 +0000 Subject: [PATCH 2/3] refactor to remove duplication --- server/src/config/services.ts | 5 +- .../src/service/business/incidentService.ts | 63 +++---------------- .../notificationMessageBuilder.ts | 3 +- 3 files changed, 13 insertions(+), 58 deletions(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 8707bc643..a3fdcdad3 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -151,11 +151,14 @@ export const initializeServices = async ({ }); const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger); + const notificationMessageBuilder = new NotificationMessageBuilder(); + const incidentService = new IncidentService({ logger, incidentsRepository, monitorsRepository, usersRepository, + notificationMessageBuilder, }); const checkService = new CheckService({ @@ -175,8 +178,6 @@ export const initializeServices = async ({ const pagerDutyProvider = new PagerDutyProvider(logger); const matrixProvider = new MatrixProvider(logger); - const notificationMessageBuilder = new NotificationMessageBuilder(); - const notificationsService = new NotificationsService( notificationsRepository, monitorsRepository, diff --git a/server/src/service/business/incidentService.ts b/server/src/service/business/incidentService.ts index 67431f0ae..9b3344f11 100644 --- a/server/src/service/business/incidentService.ts +++ b/server/src/service/business/incidentService.ts @@ -6,6 +6,7 @@ import { ParseBoolean } from "@/utils/utils.js"; import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from "@/repositories/index.js"; import type { Incident } from "@/types/index.js"; import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; +import type { INotificationMessageBuilder } from "@/service/infrastructure/notificationMessageBuilder.js"; const dateRangeLookup: Record = { recent: new Date(new Date().setHours(new Date().getHours() - 2)), @@ -23,22 +24,26 @@ class IncidentService { private incidentsRepository: IIncidentsRepository; private monitorsRepository: IMonitorsRepository; private usersRepository: IUsersRepository; + private notificationMessageBuilder: INotificationMessageBuilder; constructor({ logger, incidentsRepository, monitorsRepository, usersRepository, + notificationMessageBuilder, }: { logger: any; incidentsRepository: IIncidentsRepository; monitorsRepository: IMonitorsRepository; usersRepository: IUsersRepository; + notificationMessageBuilder: INotificationMessageBuilder; }) { this.logger = logger; this.incidentsRepository = incidentsRepository; this.monitorsRepository = monitorsRepository; this.usersRepository = usersRepository; + this.notificationMessageBuilder = notificationMessageBuilder; } get serviceName() { @@ -96,69 +101,17 @@ class IncidentService { }; private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string { - const breaches: string[] = []; - - if (!monitorStatusResponse?.payload || monitor.type !== "hardware") { + if (!monitorStatusResponse) { return "Threshold breach detected"; } - const payload = monitorStatusResponse.payload as { data?: any }; - const hardware = payload.data; - - if (!hardware) { - return "Threshold breach detected"; - } - - // Check CPU breach - if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && hardware.cpu?.usage_percent !== undefined) { - const cpuPercent = hardware.cpu.usage_percent * 100; - if (cpuPercent > monitor.cpuAlertThreshold) { - breaches.push(`CPU: ${cpuPercent.toFixed(1)}% (threshold: ${monitor.cpuAlertThreshold}%)`); - } - } - - // Check Memory breach - if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && hardware.memory?.usage_percent !== undefined) { - const memoryPercent = hardware.memory.usage_percent * 100; - if (memoryPercent > monitor.memoryAlertThreshold) { - breaches.push(`Memory: ${memoryPercent.toFixed(1)}% (threshold: ${monitor.memoryAlertThreshold}%)`); - } - } - - // Check Disk breach - if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(hardware.disk)) { - let maxDiskPercent = 0; - for (const disk of hardware.disk) { - if (disk.usage_percent !== undefined) { - const diskPercent = disk.usage_percent * 100; - if (diskPercent > maxDiskPercent) { - maxDiskPercent = diskPercent; - } - } - } - if (maxDiskPercent > monitor.diskAlertThreshold) { - breaches.push(`Disk: ${maxDiskPercent.toFixed(1)}% (threshold: ${monitor.diskAlertThreshold}%)`); - } - } - - // Check Temperature breach - if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && Array.isArray(hardware.cpu?.temperature)) { - let maxTemp = 0; - for (const temp of hardware.cpu.temperature) { - if (temp.current !== undefined && temp.current > maxTemp) { - maxTemp = temp.current; - } - } - if (maxTemp > monitor.tempAlertThreshold) { - breaches.push(`Temperature: ${maxTemp.toFixed(1)}°C (threshold: ${monitor.tempAlertThreshold}°C)`); - } - } + const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse); if (breaches.length === 0) { return "Threshold breach detected"; } - return breaches.join(", "); + return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", "); } resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => { diff --git a/server/src/service/infrastructure/notificationMessageBuilder.ts b/server/src/service/infrastructure/notificationMessageBuilder.ts index 95e0da2d7..d8235e263 100644 --- a/server/src/service/infrastructure/notificationMessageBuilder.ts +++ b/server/src/service/infrastructure/notificationMessageBuilder.ts @@ -15,6 +15,7 @@ export interface INotificationMessageBuilder { decision: MonitorActionDecision, clientHost: string ): NotificationMessage; + extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[]; } const SERVICE_NAME = "NotificationMessageBuilder"; @@ -192,7 +193,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } - private extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[] { + public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[] { const breaches: ThresholdBreach[] = []; // Check if this is a hardware monitor with threshold data From 7bc03c976b21929c8b82e2d47b6593c8863a9092 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 17 Feb 2026 17:53:00 +0000 Subject: [PATCH 3/3] remove comment --- .../infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index d59f3722b..72a0b03f9 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -195,7 +195,6 @@ class SuperSimpleQueueHelper { notificationReason: null, }; - // Simplified logic: Just check status changes if (!statusChanged) { return decision; }