const SERVICE_NAME = "JobQueueHelper"; import type { Monitor } from "@/types/monitor.js"; import { AppError } from "@/utils/AppError.js"; import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js"; import type { StatusChangeResult, MonitorStatusResponse, HardwareStatusPayload, MonitorStatus } from "@/types/index.js"; import IncidentService from "@/service/business/incidentService.js"; import { IMaintenanceWindowsRepository, IMonitorsRepository } from "@/repositories/index.js"; export interface MonitorActionDecision { shouldCreateIncident: boolean; shouldResolveIncident: boolean; shouldSendNotification: boolean; incidentReason: "status_down" | "threshold_breach" | null; notificationReason: "status_change" | "threshold_breach" | null; thresholdBreaches?: { cpu?: boolean; memory?: boolean; disk?: boolean; temp?: boolean; }; } class SuperSimpleQueueHelper { static SERVICE_NAME = SERVICE_NAME; private logger: any; private networkService: INetworkService; private statusService: IStatusService; private notificationsService: INotificationsService; private checkService: any; private buffer: any; private incidentService: IncidentService; private maintenanceWindowsRepository: IMaintenanceWindowsRepository; private monitorsRepository: IMonitorsRepository; constructor({ logger, networkService, statusService, notificationsService, checkService, buffer, incidentService, maintenanceWindowsRepository, monitorsRepository, }: { logger: any; networkService: INetworkService; statusService: IStatusService; notificationsService: INotificationsService; checkService: any; buffer: any; incidentService: IncidentService; maintenanceWindowsRepository: IMaintenanceWindowsRepository; monitorsRepository: IMonitorsRepository; }) { this.logger = logger; this.networkService = networkService; this.statusService = statusService; this.checkService = checkService; this.buffer = buffer; this.notificationsService = notificationsService; this.incidentService = incidentService; this.maintenanceWindowsRepository = maintenanceWindowsRepository; this.monitorsRepository = monitorsRepository; } get serviceName() { return SuperSimpleQueueHelper.SERVICE_NAME; } getMonitorJob = () => { return async (monitor: Monitor) => { try { const monitorId = monitor.id; const teamId = monitor.teamId; if (!monitorId) { throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getMonitorJob" }); } // Step 1. Check for maintenance window, if found, skip the check const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId, teamId); if (maintenanceWindowActive) { this.logger.debug({ message: `Monitor ${monitorId} is in maintenance window`, service: SERVICE_NAME, method: "getMonitorJob", }); if (monitor.status !== "maintenance") { await this.monitorsRepository.updateById(monitorId, teamId, { status: "maintenance" }); } return; } // Step 2. Request monitor status const status = await this.networkService.requestStatus(monitor); if (!status) { throw new Error("No network response"); } // Step 3. Build check const check = await this.checkService.buildCheck(status); if (!check) { this.logger.warn({ message: `No check could be built for monitor ${monitorId}`, service: SERVICE_NAME, method: "getMonitorJob", details: { code: status.code, message: status.message }, }); return; } // Step 4 Add check to buffer this.buffer.addToBuffer({ check }); // Step 4. Update monitor status const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); // Step 5 handle notifications (best effort, continue even in event of failure, don't wait) this.notificationsService .handleNotifications(statusChangeResult.monitor, status, statusChangeResult.prevStatus, statusChangeResult.statusChanged) .catch((error: any) => { this.logger.error({ message: error.message, service: SERVICE_NAME, method: "getMonitorJob", details: `Error sending notifications for job ${statusChangeResult.monitor.id}: ${error.message}`, stack: error.stack, }); }); // Step 6. Handle incidents (best effort, don't wait) if (statusChangeResult.statusChanged) { this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code).catch((error: any) => { this.logger.warn({ message: error.message, service: SERVICE_NAME, method: "getMonitorJob", details: `Error handling incident for job ${monitor.id}: ${error.message}`, stack: error.stack, }); }); } } catch (error: any) { this.logger.warn({ message: error.message, service: error.service || SERVICE_NAME, method: error.method || "getMonitorJob", stack: error.stack, }); throw error; } }; }; async isInMaintenanceWindow(monitorId: string, teamId: string) { const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId); // Check for active maintenance window: const maintenanceWindowIsActive = maintenanceWindows.reduce((acc: any, window: any) => { if (window.active) { const start = new Date(window.start); const end = new Date(window.end); const now = new Date(); const repeatInterval = window.repeat || 0; // If start is < now and end > now, we're in maintenance if (start <= now && end >= now) return true; // If maintenance window was set in the past with a repeat, // we need to advance start and end to see if we are in range while (start < now && repeatInterval !== 0) { start.setTime(start.getTime() + repeatInterval); end.setTime(end.getTime() + repeatInterval); if (start <= now && end >= now) { return true; } } return false; } return acc; }, false); return maintenanceWindowIsActive; } /** * Evaluates what actions should be taken based on monitor status and hardware thresholds. * This is a pure decision function that returns what should happen without executing anything. * * @param statusChangeResult - Result from statusService.updateMonitorStatus() * @param networkResponse - Network response containing payload data * @returns Decision object indicating what actions to take */ private evaluateMonitorAction(statusChangeResult: StatusChangeResult, networkResponse: MonitorStatusResponse): MonitorActionDecision { const { monitor, statusChanged, prevStatus } = statusChangeResult; // Initialize result const decision: MonitorActionDecision = { shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false, incidentReason: null, notificationReason: null, }; // CASE 1: Non-hardware monitors (http, ping, port, docker, pagespeed, game) if (monitor.type !== "hardware") { // Only act on status changes if (statusChanged) { if (monitor.status === "down") { decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "status_down"; decision.notificationReason = "status_change"; } else if (monitor.status === "up" && prevStatus === "down") { decision.shouldResolveIncident = true; decision.shouldSendNotification = true; decision.notificationReason = "status_change"; } } return decision; } // CASE 2: Hardware monitors const payload = networkResponse.payload as HardwareStatusPayload; const metrics = payload?.data; if (!metrics) { return decision; // No metrics, can't evaluate } // Check if thresholds exist const thresholds = monitor.thresholds || {}; const hasThresholds = thresholds.usage_cpu !== undefined || thresholds.usage_memory !== undefined || thresholds.usage_disk !== undefined || thresholds.usage_temperature !== undefined; // Evaluate threshold breaches const breaches = { cpu: false, memory: false, disk: false, temp: false, }; if (hasThresholds) { // CPU check if (thresholds.usage_cpu !== undefined) { const cpuUsage = metrics.cpu?.usage_percent ?? -1; breaches.cpu = cpuUsage !== -1 && cpuUsage > thresholds.usage_cpu; } // Memory check if (thresholds.usage_memory !== undefined) { const memoryUsage = metrics.memory?.usage_percent ?? -1; breaches.memory = memoryUsage !== -1 && memoryUsage > thresholds.usage_memory; } // Disk check if (thresholds.usage_disk !== undefined) { breaches.disk = metrics.disk?.some((d: any) => typeof d?.usage_percent === "number" && d.usage_percent > thresholds.usage_disk!) ?? false; } // Temperature check (alert if ANY sensor exceeds threshold) if (thresholds.usage_temperature !== undefined) { const temps = metrics.cpu?.temperature ?? []; breaches.temp = temps.some((temp: number) => temp > thresholds.usage_temperature!); } } const anyThresholdBreached = Object.values(breaches).some((b) => b); // Check if countdown has reached zero for any threshold const shouldNotifyThreshold = (breaches.cpu && (monitor.cpuAlertThreshold ?? monitor.alertThreshold) <= 0) || (breaches.memory && (monitor.memoryAlertThreshold ?? monitor.alertThreshold) <= 0) || (breaches.disk && (monitor.diskAlertThreshold ?? monitor.alertThreshold) <= 0) || (breaches.temp && (monitor.tempAlertThreshold ?? monitor.alertThreshold) <= 0); // Decision logic for hardware if (statusChanged && monitor.status === "down") { // Hardware service is DOWN (unreachable) decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "status_down"; decision.notificationReason = "status_change"; } else if (statusChanged && monitor.status === "up" && prevStatus === "down") { // Hardware service recovered decision.shouldResolveIncident = true; decision.shouldSendNotification = true; decision.notificationReason = "status_change"; } else if (anyThresholdBreached && shouldNotifyThreshold) { // Threshold breach alert (service is UP but metrics are high) decision.shouldCreateIncident = true; decision.shouldSendNotification = true; decision.incidentReason = "threshold_breach"; decision.notificationReason = "threshold_breach"; decision.thresholdBreaches = breaches; } return decision; } } export default SuperSimpleQueueHelper;