diff --git a/server/src/services/business/IncidentService.ts b/server/src/services/business/IncidentService.ts index 6fcd11930..8b786f3cb 100644 --- a/server/src/services/business/IncidentService.ts +++ b/server/src/services/business/IncidentService.ts @@ -9,12 +9,18 @@ import type { ResolutionType } from "@/db/models/index.js"; import mongoose from "mongoose"; import { getChildLogger } from "@/logger/Logger.js"; import ApiError from "@/utils/ApiError.js"; +import { ThresholdEvaluationResult } from "@/services/infrastructure/StatusService.js"; export interface IIncidentService { handleStatusChange: ( updatedMonitor: IMonitor, lastCheck: ICheck ) => Promise; + handleThresholdBreach: ( + updatedMonitor: IMonitor, + lastCheck: ICheck, + evalResult: ThresholdEvaluationResult + ) => Promise; create: ( teamId: mongoose.Types.ObjectId, monitorId: mongoose.Types.ObjectId, @@ -102,6 +108,64 @@ class IncidentService implements IIncidentService { return null; }; + handleThresholdBreach = async ( + updatedMonitor: IMonitor, + lastCheck: ICheck, + evalResult: ThresholdEvaluationResult + ) => { + const existing = await Incident.findOne({ + monitorId: updatedMonitor._id, + teamId: updatedMonitor.teamId, + resolved: false, + }); + + if (evalResult.hasBreach) { + if (!existing) { + const incident = await this.create( + updatedMonitor.teamId, + updatedMonitor._id, + lastCheck._id + ); + if (evalResult.notes.length) { + await Incident.updateOne( + { _id: incident._id }, + { + $set: { + resolutionNote: `threshold breach: ${evalResult.notes.join( + ", " + )}`, + }, + } + ); + } + return incident; + } + if (evalResult.notes.length) { + await Incident.updateOne( + { _id: existing._id }, + { + $set: { + resolutionNote: `threshold breach: ${evalResult.notes.join( + ", " + )}`, + }, + } + ); + } + return existing; + } else if (existing) { + const resolved = await this.resolve( + updatedMonitor.teamId.toString(), + existing._id.toString(), + "auto", + lastCheck + ); + return resolved; + } + + return null; + }; + create = async ( teamId: mongoose.Types.ObjectId, monitorId: mongoose.Types.ObjectId, diff --git a/server/src/services/infrastructure/JobGenerator.ts b/server/src/services/infrastructure/JobGenerator.ts index 8742647d3..160722b0d 100644 --- a/server/src/services/infrastructure/JobGenerator.ts +++ b/server/src/services/infrastructure/JobGenerator.ts @@ -1,4 +1,4 @@ -import { IMonitor, ICheck, Incident } from "@/db/models/index.js"; +import { IMonitor, ICheck } from "@/db/models/index.js"; import { INetworkService } from "./NetworkService.js"; import { ICheckService } from "../business/CheckService.js"; import { IMonitorStatsService } from "../business/MonitorStatsService.js"; @@ -45,110 +45,6 @@ class JobGenerator implements IJobGenerator { this.maintenanceService = maintenanceService; } - private evaluateThresholds = async ( - monitor: IMonitor, - check: ICheck - ) => { - try { - // Only for infrastructure monitors and only when service is up - if (monitor.type !== "infrastructure" || monitor.status !== "up") return; - if (!monitor.thresholds) return; - if (!check?.system) return; - - const { thresholds: t } = monitor; - const breaches: string[] = []; - - const cpu = check.system?.cpu?.usage_percent; - if ( - typeof t?.cpu === "number" && - typeof cpu === "number" && - cpu > t.cpu - ) { - breaches.push(`cpu ${cpu}% > ${t.cpu}%`); - } - - const mem = check.system?.memory?.usage_percent; - if ( - typeof t?.memory === "number" && - typeof mem === "number" && - mem > t.memory - ) { - breaches.push(`memory ${mem}% > ${t.memory}%`); - } - - const diskUsages: number[] = Array.isArray(check.system?.disk) - ? check.system.disk - .map((d) => d?.usage_percent) - .filter((n): n is number => typeof n === "number") - : []; - const maxDisk = diskUsages.length ? Math.max(...diskUsages) : undefined; - if ( - typeof t?.disk === "number" && - typeof maxDisk === "number" && - maxDisk > t.disk - ) { - breaches.push(`disk ${maxDisk}% > ${t.disk}%`); - } - - const temps: number[] = Array.isArray(check.system?.cpu?.temperature) - ? check.system.cpu.temperature.filter( - (n): n is number => typeof n === "number" - ) - : []; - const maxTemp = temps.length ? Math.max(...temps) : undefined; - if ( - typeof t?.temperature === "number" && - typeof maxTemp === "number" && - maxTemp > t.temperature - ) { - breaches.push(`temperature ${maxTemp}C > ${t.temperature}C`); - } - - // If any breach → ensure one open incident exists; else resolve if one exists - const hasBreach = breaches.length > 0; - const existingOpen = await Incident.findOne({ - monitorId: monitor._id, - teamId: monitor.teamId, - resolved: false, - }); - - if (hasBreach && !existingOpen) { - const incident = await this.incidentService.create( - monitor.teamId, - monitor._id, - check._id - ); - // annotate with threshold note best-effort - try { - if (breaches.length) { - await Incident.updateOne( - { _id: incident._id }, - { - $set: { - resolutionNote: `threshold breach: ${breaches.join(", ")}`, - }, - } - ); - } - } catch {} - this.notificationService - .handleNotifications(monitor, incident) - .catch(() => {}); - } - - if (!hasBreach && existingOpen) { - await this.incidentService.resolve( - monitor.teamId.toString(), - existingOpen._id.toString(), - "auto", - check - ); - } - } catch { - // best-effort; do not throw from evaluator - } - }; - generateJob = () => { return async (monitor: IMonitor) => { try { @@ -157,7 +53,7 @@ class JobGenerator implements IJobGenerator { throw new ApiError("No monitorID for creating job", 400); } - // Check for active maintenance window, if found, skip the check + // Step 1. Check for active maintenance window, if found, skip the check const isInMaintenance = await this.maintenanceService.isInMaintenance( monitorId ); @@ -165,12 +61,18 @@ class JobGenerator implements IJobGenerator { return; } + // Setp 2. Request monitor status const status = await this.networkService.requestStatus(monitor); + + // Step 3. Build check const check = await this.checkService.buildCheck(status, monitor.type); await check.save(); + + // Step 4. Update monitor's status const [updatedMonitor, statusChanged] = await this.statusService.updateMonitorStatus(monitor, status); + // Step 5. Handle status change if (statusChanged) { const incident = await this.incidentService.handleStatusChange( updatedMonitor, @@ -186,14 +88,35 @@ class JobGenerator implements IJobGenerator { }); } } + + // Step 6. Update aggregate monitor stats await this.statusService.updateMonitorStats( updatedMonitor, status, statusChanged ); - // Evaluate thresholds after saving the check and updating stats - await this.evaluateThresholds(updatedMonitor, check); + // Step 7. Evaluate thresholds for Infrastrucutre monitors + if (updatedMonitor.type === "infrastructure") { + const evalResult = await this.statusService.evaluateThresholds( + updatedMonitor, + check + ); + + const incident = await this.incidentService.handleThresholdBreach( + updatedMonitor, + check, + evalResult + ); + + if (incident) { + this.notificationService + .handleNotifications(updatedMonitor, incident) + .catch((error) => { + logger.warn(error); + }); + } + } } catch (error) { throw error; } diff --git a/server/src/services/infrastructure/StatusService.ts b/server/src/services/infrastructure/StatusService.ts index b9759cf87..bb023cf47 100644 --- a/server/src/services/infrastructure/StatusService.ts +++ b/server/src/services/infrastructure/StatusService.ts @@ -1,6 +1,34 @@ -import { IMonitor, IMonitorStats, MonitorStats } from "@/db/models/index.js"; +import { + IMonitor, + IMonitorStats, + MonitorStats, + ICheck, +} from "@/db/models/index.js"; import { StatusResponse } from "./NetworkService.js"; -import ApiError from "@/utils/ApiError.js"; + +const emptyMetrics = (): ThresholdEvaluationResult["metrics"] => ({ + cpu: { breached: false }, + memory: { breached: false }, + disk: { breached: false }, + temperature: { breached: false }, +}); + +export interface ThresholdMetricEvaluation { + value?: number; + threshold?: number; + breached: boolean; +} + +export interface ThresholdEvaluationResult { + hasBreach: boolean; + notes: string[]; + metrics: { + cpu: ThresholdMetricEvaluation; + memory: ThresholdMetricEvaluation; + disk: ThresholdMetricEvaluation; + temperature: ThresholdMetricEvaluation; + }; +} const SERVICE_NAME = "StatusService"; const MAX_LATEST_CHECKS = 25; @@ -20,6 +48,10 @@ export interface IStatusService { status: StatusResponse, statusChanged: boolean ) => Promise; + evaluateThresholds: ( + monitor: IMonitor, + check: ICheck + ) => Promise; } export type StatusChangeResult = [ @@ -142,6 +174,92 @@ class StatusService implements IStatusService { return await stats.save(); }; + + evaluateThresholds = async ( + monitor: IMonitor, + check: ICheck + ): Promise => { + try { + if (monitor.type !== "infrastructure" || monitor.status !== "up") + return { hasBreach: false, notes: [], metrics: emptyMetrics() }; + if (!monitor.thresholds) + return { hasBreach: false, notes: [], metrics: emptyMetrics() }; + if (!check.system) + return { hasBreach: false, notes: [], metrics: emptyMetrics() }; + + const t = monitor.thresholds; + const breaches: string[] = []; + const metrics: ThresholdEvaluationResult["metrics"] = emptyMetrics(); + + const cpu = check.system.cpu?.usage_percent; + metrics.cpu = { + value: typeof cpu === "number" ? cpu : undefined, + threshold: typeof t.cpu === "number" ? t.cpu : undefined, + breached: + typeof t.cpu === "number" && typeof cpu === "number" && cpu > t.cpu, + }; + if ( + metrics.cpu.breached && + metrics.cpu.value !== undefined && + metrics.cpu.threshold !== undefined + ) { + breaches.push(`cpu ${metrics.cpu.value}% > ${metrics.cpu.threshold}%`); + } + + const mem = check.system.memory?.usage_percent; + metrics.memory = { + value: typeof mem === "number" ? mem : undefined, + threshold: typeof t.memory === "number" ? t.memory : undefined, + breached: + typeof t.memory === "number" && + typeof mem === "number" && + mem > t.memory, + }; + if ( + metrics.memory.breached && + metrics.memory.value !== undefined && + metrics.memory.threshold !== undefined + ) { + breaches.push( + `memory ${metrics.memory.value}% > ${metrics.memory.threshold}%` + ); + } + + const diskUsages: number[] = Array.isArray(check.system.disk) + ? check.system.disk + .map((d) => d?.usage_percent) + .filter((n): n is number => typeof n === "number") + : []; + const maxDisk = diskUsages.length ? Math.max(...diskUsages) : undefined; + if ( + typeof t.disk === "number" && + typeof maxDisk === "number" && + maxDisk > t.disk + ) { + breaches.push(`disk ${maxDisk}% > ${t.disk}%`); + } + + const temps: number[] = Array.isArray(check.system.cpu?.temperature) + ? check.system.cpu.temperature.filter( + (n): n is number => typeof n === "number" + ) + : []; + const maxTemp = temps.length ? Math.max(...temps) : undefined; + if ( + typeof t.temperature === "number" && + typeof maxTemp === "number" && + maxTemp > t.temperature + ) { + breaches.push(`temperature ${maxTemp}C > ${t.temperature}C`); + } + + const hasBreach = breaches.length > 0; + return { hasBreach, notes: breaches, metrics }; + } catch { + // best-effort; do not throw from evaluator + return { hasBreach: false, notes: [], metrics: emptyMetrics() }; + } + }; } export default StatusService;