diff --git a/server/src/repositories/monitors/IMonitorsRepository.ts b/server/src/repositories/monitors/IMonitorsRepository.ts index 8c5f63e7d..130a44ce5 100644 --- a/server/src/repositories/monitors/IMonitorsRepository.ts +++ b/server/src/repositories/monitors/IMonitorsRepository.ts @@ -1,4 +1,4 @@ -import { type MonitorType, type Monitor, type MonitorsSummary } from "@/types/index.js"; +import { type MonitorType, type Monitor, type MonitorsSummary, CheckSnapshot } from "@/types/index.js"; export interface TeamQueryConfig { limit?: number; @@ -29,6 +29,14 @@ export interface IMonitorsRepository { // update updateById(monitorId: string, teamId: string, updates: Partial): Promise; + updateStatusWindowAndChecks( + monitorId: string, + teamId: string, + status: boolean, + checkSnapshot: CheckSnapshot, + windowSize: number, + maxRecentChecks: number + ): Promise; togglePauseById(monitorId: string, teamId: string): Promise; // delete deleteById(monitorId: string, teamId: string): Promise; diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d759448..5b0549677 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -182,6 +182,31 @@ class MongoMonitorsRepository implements IMonitorsRepository { return this.toEntity(updatedMonitor); }; + updateStatusWindowAndChecks = async ( + monitorId: string, + teamId: string, + status: boolean, + checkSnapshot: CheckSnapshot, + windowSize: number, + maxRecentChecks: number + ): Promise => { + const updatedMonitor = await MonitorModel.findOneAndUpdate( + { _id: monitorId, teamId }, + { + $push: { + statusWindow: { $each: [status], $slice: -windowSize }, + recentChecks: { $each: [checkSnapshot], $slice: -maxRecentChecks }, + }, + }, + { returnDocument: "after" } + ); + + if (!updatedMonitor) { + throw new AppError({ message: `Failed to update status and checks for monitor with id ${monitorId}`, status: 500 }); + } + return this.toEntity(updatedMonitor); + }; + togglePauseById = async (monitorId: string, teamId: string) => { const monitor = await MonitorModel.findOneAndUpdate( { _id: monitorId, teamId }, diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index fa57117dc..edf432b17 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -217,6 +217,125 @@ export class StatusService implements IStatusService { return { nextStatus, transitioned, breaches, nextCounters }; }; + private updateStatusWindowAndRecentChecks = async (monitor: Monitor, check: Check) => { + const checkSnapshot = this.toCheckSnapshot(check); + return await this.monitorsRepository.updateStatusWindowAndChecks( + monitor.id, + monitor.teamId, + check.status, + checkSnapshot, + monitor.statusWindowSize, + MAX_RECENT_CHECKS + ); + }; + + newUpdateMonitorStatus = async ( + statusResponse: MonitorStatusResponse< + | PingStatusPayload + | HttpStatusPayload + | PageSpeedStatusPayload + | HardwareStatusPayload + | DockerStatusPayload + | PortStatusPayload + | GameStatusPayload + | GrpcStatusPayload + | undefined + >, + check: Check + ): Promise => { + try { + const { monitorId, teamId, status, code } = statusResponse; + const monitor = await this.monitorsRepository.findById(monitorId, teamId); + + // Update running stats + await this.tryUpdateRunningStats(monitor, statusResponse); + const updatedMonitor = await this.updateStatusWindowAndRecentChecks(monitor, check); + const prevStatus = updatedMonitor.status; + + // Return early if not enough data points + if (updatedMonitor.statusWindow.length < updatedMonitor.statusWindowSize) { + updatedMonitor.status = status === true ? "up" : "down"; + const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, { status: updatedMonitor.status }); + return { + monitor: updated, + statusChanged: false, + prevStatus, + code, + timestamp: Date.now(), + }; + } + + // With a full window, a single raw check must not change UNLESS we are initializing. Otherwise, only the sliding-window threshold can trigger a transition. + let newStatus: MonitorStatus; + if (updatedMonitor.status === "initializing") { + newStatus = status === true ? "up" : "down"; + } else { + newStatus = updatedMonitor.status; + } + + let statusChanged = false; + + // First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches. + const reachabilityResult = this.computeReachability(newStatus, updatedMonitor.statusWindow, updatedMonitor.statusWindowThreshold); + if (reachabilityResult.transitioned) { + newStatus = reachabilityResult.nextStatus; + statusChanged = true; + } + + // Evaluate hardware threshold breaches (only for hardware monitors with metrics payload) + let thresholdBreaches: HardwareBreaches | undefined; + const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined; + if (updatedMonitor.type === "hardware" && hardwarePayload?.data) { + const hardware = this.computeHardwareStatus({ + currentStatus: newStatus, + reachabilityDown: newStatus === "down", + metrics: hardwarePayload.data, + thresholds: { + cpu: updatedMonitor.cpuAlertThreshold, + memory: updatedMonitor.memoryAlertThreshold, + disk: updatedMonitor.diskAlertThreshold, + temp: updatedMonitor.tempAlertThreshold, + }, + counters: { + cpu: updatedMonitor.cpuAlertCounter, + memory: updatedMonitor.memoryAlertCounter, + disk: updatedMonitor.diskAlertCounter, + temp: updatedMonitor.tempAlertCounter, + }, + }); + + updatedMonitor.cpuAlertCounter = hardware.nextCounters.cpu; + updatedMonitor.memoryAlertCounter = hardware.nextCounters.memory; + updatedMonitor.diskAlertCounter = hardware.nextCounters.disk; + updatedMonitor.tempAlertCounter = hardware.nextCounters.temp; + thresholdBreaches = hardware.breaches; + if (hardware.transitioned) { + newStatus = hardware.nextStatus; + statusChanged = true; + } + } + + // Apply the final status + updatedMonitor.status = newStatus; + + const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, updatedMonitor); + + return { + monitor: updated, + statusChanged, + prevStatus, + code, + timestamp: new Date().getTime(), + thresholdBreaches, + }; + } catch (error: unknown) { + throw new AppError({ + message: `Failed to update monitor with id ${check.metadata.monitorId} with status: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "updateMonitorStatus", + }); + } + }; updateMonitorStatus = async ( statusResponse: MonitorStatusResponse< | PingStatusPayload