From 380fdf470f2e8958606540b09da8d16a289d7d3b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 17 Jun 2026 11:35:15 -0700 Subject: [PATCH 1/9] backoff excessive polling --- server/src/worker/worker.db-queue.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index f44d91c98..2b42b8ca2 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -15,7 +15,8 @@ import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.rep import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; const SERVICE_NAME = "JobQueue"; -const POLL_MS = 250; +const POLL_MS = 250; // base poll interval while a loop is actively claiming work +const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals @@ -240,6 +241,7 @@ export class DBQueueWorker implements IWorker { }; private startLoop = (type: JobType) => { + let pollMs = POLL_MS; const tick = async () => { if (this.stopped) return; try { @@ -252,6 +254,8 @@ export class DBQueueWorker implements IWorker { this.inFlight[type]--; // job done, free the slot }); } + // Back off if no jobs founds, reset to base if jobs found + pollMs = capacity > 0 && jobs.length === 0 ? Math.min(pollMs * 2, POLL_MAX_MS) : POLL_MS; } catch (error: unknown) { this.logger.error({ message: error instanceof Error ? error.message : String(error), @@ -259,7 +263,7 @@ export class DBQueueWorker implements IWorker { method: `loop:${type}`, }); } - if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down + if (!this.stopped) this.timers.set(type, setTimeout(tick, pollMs)); // re-arm unless shutting down }; tick(); }; From 5efd64d44d56cb0c21fc70a1285709288da84363 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:05:17 -0700 Subject: [PATCH 2/9] thread prefetched monitor trough status service --- server/src/service/statusService.ts | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 47aa8d89a..f7d2b0b99 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -1,4 +1,3 @@ -import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import type { Check, CheckDiskInfo, CheckSnapshot } from "@/domain/checks/check.type.js"; @@ -17,7 +16,6 @@ import type { } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { ILogger } from "@/utils/logger.js"; -import { IBufferService } from "./bufferService.js"; import type { HardwareStatusMetrics } from "@/types/network.js"; import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.types.js"; @@ -42,30 +40,21 @@ export interface IStatusService { | GrpcStatusPayload | undefined >, - check: Check + check: Check, + monitor: Monitor ): Promise; } export class StatusService implements IStatusService { static SERVICE_NAME = SERVICE_NAME; private logger: ILogger; - private buffer: IBufferService; private monitorsRepository: IMonitorsRepository; private monitorStatsRepository: IMonitorStatsRepository; - private checksRepository: IChecksRepository; - constructor( - logger: ILogger, - buffer: IBufferService, - monitorsRepository: IMonitorsRepository, - monitorStatsRepository: IMonitorStatsRepository, - checksRepository: IChecksRepository - ) { + constructor(logger: ILogger, monitorsRepository: IMonitorsRepository, monitorStatsRepository: IMonitorStatsRepository) { this.logger = logger; - this.buffer = buffer; this.monitorsRepository = monitorsRepository; this.monitorStatsRepository = monitorStatsRepository; - this.checksRepository = checksRepository; } get serviceName() { @@ -212,11 +201,11 @@ export class StatusService implements IStatusService { | GrpcStatusPayload | undefined >, - check: Check + check: Check, + monitor: Monitor ): Promise => { try { - const { monitorId, teamId, status, code } = statusResponse; - const monitor = await this.monitorsRepository.findById(monitorId, teamId); + const { status, code } = statusResponse; // Update running stats await this.tryUpdateRunningStats(monitor, statusResponse); @@ -262,7 +251,7 @@ export class StatusService implements IStatusService { }; } - // First evaluate reachability-based status changes, which apply to all monitor types + // First evaluate reachability status changes, which apply to all monitor types // and take precedence over hardware breaches. const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold); if (reachabilityResult.transitioned) { From f740eef0fad0c45676b103fc3068586475e2fb5e Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:05:57 -0700 Subject: [PATCH 3/9] thread prefetched worker, remove dead code --- server/src/worker/worker.check-evaluator.ts | 7 +-- server/src/worker/worker.check-pipeline.ts | 18 -------- server/src/worker/worker.db-queue.ts | 13 +++--- server/src/worker/worker.helper.ts | 49 +-------------------- 4 files changed, 13 insertions(+), 74 deletions(-) diff --git a/server/src/worker/worker.check-evaluator.ts b/server/src/worker/worker.check-evaluator.ts index bbae96d69..34c444271 100644 --- a/server/src/worker/worker.check-evaluator.ts +++ b/server/src/worker/worker.check-evaluator.ts @@ -1,11 +1,12 @@ import { MonitorStatusResponse } from "@/types/network.js"; import { MonitorEvaluation } from "@/worker/worker.interface.js"; import { Check } from "@/domain/checks/check.type.js"; +import { Monitor } from "@/domain/monitors/monitor.types.js"; import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; import { IStatusService } from "@/service/statusService.js"; export interface ICheckEvaluator { - evaluate(status: MonitorStatusResponse, check: Check): Promise; + evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise; } export class CheckEvaluator implements ICheckEvaluator { @@ -13,11 +14,11 @@ export class CheckEvaluator implements ICheckEvaluator { private statusService: IStatusService, private monitorStatusPolicy: IMonitorStatusPolicy ) {} - evaluate = async (status: MonitorStatusResponse, check: Check) => { + evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => { // **************************** // Step 3: Evaluate and return result to reactors // **************************** - const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); + const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor); // Step 5. Get decisions and create an evaluation obj const decision = this.monitorStatusPolicy.evaluate(statusChangeResult); diff --git a/server/src/worker/worker.check-pipeline.ts b/server/src/worker/worker.check-pipeline.ts index c472c4bfa..045cea1ad 100644 --- a/server/src/worker/worker.check-pipeline.ts +++ b/server/src/worker/worker.check-pipeline.ts @@ -7,8 +7,6 @@ import { IBufferService } from "@/service/bufferService.js"; import { AppError } from "@/utils/AppError.js"; import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; -import { ICheckProducer } from "@/worker/worker.check-producer.js"; -import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; const SERVICE_NAME = "CheckPipeline"; @@ -16,22 +14,6 @@ export interface ICheckPipeline { run(monitor: Monitor): Promise; // null = skipped } -export class CheckPipeline implements ICheckPipeline { - constructor( - private checkProducer: ICheckProducer, - private checkEvaluator: ICheckEvaluator - ) {} - - run = async (monitor: Monitor): Promise => { - const checkResult = await this.checkProducer.produce(monitor); - if (!checkResult) { - return null; // skipped due to maintenance / geo guards / other guards - } - const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check); - return evaluation; - }; -} - export class GeoChecksPipeline implements ICheckPipeline { constructor( private maintenanceWindowsRepository: IMaintenanceWindowsRepository, diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 9088dc3b3..bef7a838e 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -151,7 +151,7 @@ export class DBQueueWorker implements IWorker { private runCheck = async (job: Job) => { if (!job.refId) return; - const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId + const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId if (!monitor) return; await this.checkProducer.produce(monitor); }; @@ -162,19 +162,22 @@ export class DBQueueWorker implements IWorker { private runEvaluate = async (job: Job) => { if (!job.refId) return; - const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId + const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId if (!monitor) return; const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt); + + let current = monitor; for (const check of checks) { const status = this.checkService.toStatusResponse(check); - const evaluation = await this.checkEvaluator.evaluate(status, check); + const evaluation = await this.checkEvaluator.evaluate(status, check, current); await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications - await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) }); + await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) }); + current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check } }; private runGeoCheck = async (job: Job) => { - const [monitor] = await this.monitorsRepository.findByIds([job.refId!]); + const monitor = await this.monitorsRepository.findByIdLean(job.refId!); if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff }; diff --git a/server/src/worker/worker.helper.ts b/server/src/worker/worker.helper.ts index 7b60a26ba..3d2d2e183 100644 --- a/server/src/worker/worker.helper.ts +++ b/server/src/worker/worker.helper.ts @@ -1,5 +1,4 @@ const SERVICE_NAME = "JobQueueHelper"; -import type { Monitor } from "@/domain/monitors/monitor.types.js"; import { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; import { ICheckService } from "@/domain/checks/check.service.js"; import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js"; @@ -11,13 +10,9 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; import { ILogger } from "@/utils/logger.js"; -import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; -import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; export interface IWorkerHelper { readonly serviceName: string; - getHeartbeatJob(): (monitor: Monitor) => Promise; - getHeartbeatGeoJob(): (monitor: Monitor) => Promise; getCleanupOrphanedJob(): () => Promise; getCleanupRetentionJob(): () => Promise; } @@ -49,9 +44,6 @@ export class WorkerHelper implements IWorkerHelper { private checksRepository: IChecksRepository; private incidentsRepository: IIncidentsRepository; private geoChecksRepository: IGeoChecksRepository; - private reactorDispatcher: IReactorDispatcher; - private checkPipeline: ICheckPipeline; - private geoCheckPipeline: ICheckPipeline; constructor( logger: ILogger, @@ -63,10 +55,7 @@ export class WorkerHelper implements IWorkerHelper { monitorStatsRepository: IMonitorStatsRepository, checksRepository: IChecksRepository, incidentsRepository: IIncidentsRepository, - geoChecksRepository: IGeoChecksRepository, - reactorDispatcher: IReactorDispatcher, - checkPipeline: ICheckPipeline, - geoCheckPipeline: ICheckPipeline + geoChecksRepository: IGeoChecksRepository ) { this.logger = logger; this.checkService = checkService; @@ -78,48 +67,12 @@ export class WorkerHelper implements IWorkerHelper { this.checksRepository = checksRepository; this.incidentsRepository = incidentsRepository; this.geoChecksRepository = geoChecksRepository; - this.reactorDispatcher = reactorDispatcher; - this.checkPipeline = checkPipeline; - this.geoCheckPipeline = geoCheckPipeline; } get serviceName() { return WorkerHelper.SERVICE_NAME; } - getHeartbeatJob = () => { - return async (monitor: Monitor) => { - try { - const evaluation = await this.checkPipeline.run(monitor); - if (evaluation) await this.reactorDispatcher.dispatch(evaluation); - } catch (error: unknown) { - this.logger.warn({ - message: error instanceof Error ? error.message : "Unknown error", - service: SERVICE_NAME, - method: "getMonitorJob", - stack: error instanceof Error ? error.stack : undefined, - }); - throw error; - } - }; - }; - - getHeartbeatGeoJob = () => { - return async (monitor: Monitor) => { - try { - await this.geoCheckPipeline.run(monitor); - } catch (error: unknown) { - this.logger.error({ - message: error instanceof Error ? error.message : "Unknown error", - service: SERVICE_NAME, - method: "getHeartbeatGeoJob", - stack: error instanceof Error ? error.stack : undefined, - }); - // Don't throw - geo check failures shouldn't crash the job scheduler - } - }; - }; - getCleanupOrphanedJob = () => { return async () => { try { From 190a3b47dad3cd69ff09305833c2a16d4100cd2b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:06:19 -0700 Subject: [PATCH 4/9] add findByIdLean for workers --- server/src/domain/monitors/monitor.repository.interface.ts | 2 ++ server/src/domain/monitors/monitor.repository.mongo.ts | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/server/src/domain/monitors/monitor.repository.interface.ts b/server/src/domain/monitors/monitor.repository.interface.ts index e5a0fa585..46f3df396 100644 --- a/server/src/domain/monitors/monitor.repository.interface.ts +++ b/server/src/domain/monitors/monitor.repository.interface.ts @@ -22,6 +22,8 @@ export interface IMonitorsRepository { createMonitors(monitors: Monitor[]): Promise; // single fetch findById(monitorId: string, teamId: string): Promise; + // for workers, don't need all details + findByIdLean(monitorId: string): Promise; // collection fetch findAll(): Promise; diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index e41c8a6f2..0ce0d48a6 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -38,6 +38,11 @@ class MongoMonitorsRepository implements IMonitorsRepository { return this.toEntity(monitor); }; + findByIdLean = async (monitorId: string): Promise => { + const monitor = await MonitorModel.findOne({ _id: monitorId }); + return monitor ? this.toEntity(monitor) : null; + }; + findAll = async (): Promise => { const monitors = await MonitorModel.find(); return this.mapDocuments(monitors); From e32590425b5379e73b14f01def85e30d4659d21e Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:06:43 -0700 Subject: [PATCH 5/9] remove unsed from constructor --- server/src/config/services.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 346f77209..9de13f27b 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -103,7 +103,7 @@ import { ILogger } from "@/utils/logger.js"; import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; -import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; +import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; import { CheckProducer } from "@/worker/worker.check-producer.js"; import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { DBQueueWorker } from "@/worker/worker.db-queue.js"; @@ -229,7 +229,7 @@ export const initializeServices = async ({ const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository); - const statusService = new StatusService(logger, bufferService, monitorsRepository, monitorStatsRepository, checksRepository); + const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); // Notification providers const webhookProvider = new WebhookProvider(logger); @@ -276,8 +276,6 @@ export const initializeServices = async ({ const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy); // pipelines - const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator); - const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger); const workerHelper = new WorkerHelper( @@ -290,10 +288,7 @@ export const initializeServices = async ({ monitorStatsRepository, checksRepository, incidentsRepository, - geoChecksRepository, - reactorDispatcher, - checkPipeline, - geoCheckPipeline + geoChecksRepository ); const worker = await DBQueueWorker.create( From 5ed9ab780ae3d5c697ef14c84a56bf39b4f99fe0 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:06:57 -0700 Subject: [PATCH 6/9] update tests, add check producer tests --- .../helpers/InMemoryMonitorsRepository.ts | 5 + server/test/helpers/heartbeatTestHarness.ts | 54 ++---- .../test/unit/services/checkPipeline.test.ts | 113 +----------- .../test/unit/services/checkProducer.test.ts | 120 +++++++++++++ .../test/unit/services/dbQueueWorker.test.ts | 34 +++- .../test/unit/services/statusService.test.ts | 162 +++++++++--------- .../test/unit/services/workerHelper.test.ts | 93 +--------- 7 files changed, 253 insertions(+), 328 deletions(-) create mode 100644 server/test/unit/services/checkProducer.test.ts diff --git a/server/test/helpers/InMemoryMonitorsRepository.ts b/server/test/helpers/InMemoryMonitorsRepository.ts index c53f270a7..45db5b3f7 100644 --- a/server/test/helpers/InMemoryMonitorsRepository.ts +++ b/server/test/helpers/InMemoryMonitorsRepository.ts @@ -40,6 +40,11 @@ export class InMemoryMonitorsRepository implements IMonitorsRepository { return this.monitors.filter((m) => monitorIds.includes(m.id)).map((m) => ({ ...m })); } + async findByIdLean(monitorId: string): Promise { + const monitor = this.monitors.find((m) => m.id === monitorId); + return monitor ? { ...monitor } : null; + } + async findByIdsWithChecks(monitorIds: string[], _checksCount?: number): Promise { return this.findByIds(monitorIds); } diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index baf7eeee3..5711f5aa9 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -1,7 +1,5 @@ import { jest } from "@jest/globals"; -import { WorkerHelper } from "../../src/worker/worker.helper.ts"; import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts"; -import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts"; import { CheckProducer } from "../../src/worker/worker.check-producer.ts"; import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts"; import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts"; @@ -71,20 +69,6 @@ const createStubMonitorStatsRepo = () => ({ deleteByMonitorIdsNotIn: jest.fn(), }); -const createStubChecksRepo = () => ({ - create: jest.fn(), - createChecks: jest.fn(), - findByMonitorId: jest.fn(), - findByTeamId: jest.fn(), - findLatestByMonitorIds: jest.fn(), - findByDateRangeAndMonitorId: jest.fn(), - findSummaryByTeamId: jest.fn(), - deleteByMonitorId: jest.fn(), - deleteByTeamId: jest.fn(), - deleteByMonitorIdsNotIn: jest.fn(), - deleteOlderThan: jest.fn(), -}); - export interface HeartbeatTestHarness { monitorsRepo: InMemoryMonitorsRepository; incidentsRepo: InMemoryIncidentsRepository; @@ -108,13 +92,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { const logger = createMockLogger() as any; const bufferStub = { addToBuffer: jest.fn(), addGeoCheckToBuffer: jest.fn(), scheduleNextFlush: jest.fn() }; - const statusService = new StatusService( - logger, - bufferStub as any, - monitorsRepo as any, - createStubMonitorStatsRepo() as any, - createStubChecksRepo() as any - ); + const statusService = new StatusService(logger, monitorsRepo as any, createStubMonitorStatsRepo() as any); const messageBuilder = { extractThresholdBreaches: jest.fn().mockReturnValue([]) }; const incidentService = new IncidentService(logger, incidentsRepo, monitorsRepo as any, { findById: jest.fn() } as any, messageBuilder as any); @@ -149,7 +127,6 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { }; const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) }; - const geoChecksService = { buildGeoCheck: jest.fn() }; const checkProducer = new CheckProducer( monitorsRepo as any, @@ -160,28 +137,21 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { logger ); const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy()); - const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator); - const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger); const notificationReactor = new NotificationReactor(notificationsService as any); const incidentReactor = new IncidentReactor(incidentService as any); const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); - const helper = new WorkerHelper( - logger, - checkService as any, - { getDBSettings: jest.fn() } as any, - monitorsRepo as any, - { deleteByMonitorIdsNotIn: jest.fn() } as any, - { findAllTeamIds: jest.fn() } as any, - createStubMonitorStatsRepo() as any, - createStubChecksRepo() as any, - incidentsRepo as any, - { deleteByMonitorIdsNotIn: jest.fn() } as any, - reactorDispatcher, - checkPipeline, - geoCheckPipeline - ); + // Mirror the production check→evaluate flow: read the monitor fresh each cycle (so the + // accumulated statusWindow is visible), produce a check, then evaluate against that monitor. + const heartbeatJob = async (seedMonitor: Monitor) => { + const monitor = await monitorsRepo.findByIdLean(seedMonitor.id); + if (!monitor) return; + const result = await checkProducer.produce(monitor); + if (!result) return; // skipped (e.g. maintenance window) + const evaluation = await checkEvaluator.evaluate(result.status, result.check, monitor); + await reactorDispatcher.dispatch(evaluation); + }; return { monitorsRepo, @@ -193,7 +163,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { bufferStub, maintenanceWindowsRepo, messageBuilder, - heartbeatJob: helper.getHeartbeatJob(), + heartbeatJob, setNextResponse, setNextResponseFull, }; diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/checkPipeline.test.ts index 9b1fffd8b..008e879fa 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/checkPipeline.test.ts @@ -1,8 +1,5 @@ import { describe, expect, it, jest } from "@jest/globals"; -import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts"; -import { CheckProducer } from "../../../src/worker/worker.check-producer.ts"; -import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts"; -import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; +import { GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; @@ -23,114 +20,6 @@ const activeWindow = () => { return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; }; -// ── CheckPipeline ──────────────────────────────────────────────────────────── - -const createCheckPipeline = (overrides?: Record) => { - const defaults = { - logger: createMockLogger(), - networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, - statusService: { - updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }), - }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - buffer: { addToBuffer: jest.fn() }, - monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, - monitorStatusPolicy: new MonitorStatusPolicy(), - ...overrides, - }; - const checkProducer = new CheckProducer( - defaults.monitorsRepository as any, - defaults.maintenanceWindowsRepository as any, - defaults.checkService as any, - defaults.networkService as any, - defaults.buffer as any, - defaults.logger as any - ); - const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); - const pipeline = new CheckPipeline(checkProducer, checkEvaluator); - return { pipeline, defaults }; -}; - -describe("CheckPipeline", () => { - it("throws when monitor id is missing", async () => { - const { pipeline } = createCheckPipeline(); - await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id"); - }); - - it("skips and sets maintenance status when in maintenance window", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - const result = await pipeline.run(makeMonitor({ status: "up" })); - - expect(result).toBeNull(); - expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" }); - }); - - it("does not re-write status when already in maintenance", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - await pipeline.run(makeMonitor({ status: "maintenance" })); - - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("throws when network response is null", async () => { - const { pipeline } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, - }); - await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response"); - }); - - it("returns null and warns when toCheck returns null", async () => { - const { pipeline, defaults } = createCheckPipeline({ - checkService: { toCheck: jest.fn().mockReturnValue(null) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(result).toBeNull(); - expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); - expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled(); - }); - - it("buffers the check, updates status, and returns an evaluation", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; - const statusChange = { monitor: { id: "m1", status: "down" }, statusChanged: true, prevStatus: "up", code: 500 }; - const { pipeline, defaults } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, - statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith({ id: "check-1" }); - expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalled(); - expect(result).toMatchObject({ - monitor: statusChange.monitor, - status, - check: { id: "check-1" }, - statusChange, - decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, incidentReason: "status_down" }), - }); - }); - - it("proceeds with the check when no maintenance window is active", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, - }); - - await pipeline.run(makeMonitor()); - - expect(defaults.networkService.requestStatus).toHaveBeenCalled(); - }); -}); - // ── GeoChecksPipeline ────────────────────────────────────────────────────────── const createGeoPipeline = (overrides?: Record) => { diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts new file mode 100644 index 000000000..bf93c8ea2 --- /dev/null +++ b/server/test/unit/services/checkProducer.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { CheckProducer } from "../../../src/worker/worker.check-producer.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "m1", + teamId: "team", + type: "http", + interval: 60000, + status: "up", + ...overrides, + }) as Monitor; + +// A maintenance window straddling "now" — active per isWindowActive. +const activeWindow = () => { + const now = Date.now(); + return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; +}; + +const createProducer = (overrides?: Record) => { + const defaults = { + logger: createMockLogger(), + monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, + checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, + networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, + buffer: { addToBuffer: jest.fn() }, + ...overrides, + }; + const producer = new CheckProducer( + defaults.monitorsRepository as any, + defaults.maintenanceWindowsRepository as any, + defaults.checkService as any, + defaults.networkService as any, + defaults.buffer as any, + defaults.logger as any + ); + return { producer, defaults }; +}; + +describe("CheckProducer", () => { + it("throws when monitor id is missing", async () => { + const { producer } = createProducer(); + await expect(producer.produce({} as Monitor)).rejects.toThrow("No monitor id"); + }); + + // ── maintenance gate ────────────────────────────────────────────────────── + + it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => { + const { producer, defaults } = createProducer({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, + }); + + const result = await producer.produce(makeMonitor({ status: "up" })); + + expect(result).toBeNull(); + expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" }); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); + }); + + it("does not re-write status when already in maintenance", async () => { + const { producer, defaults } = createProducer({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, + }); + + const result = await producer.produce(makeMonitor({ status: "maintenance" })); + + expect(result).toBeNull(); + expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("proceeds with the check when the maintenance window is inactive", async () => { + const { producer, defaults } = createProducer({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, + }); + + await producer.produce(makeMonitor()); + + expect(defaults.networkService.requestStatus).toHaveBeenCalled(); + }); + + // ── acquire / record ────────────────────────────────────────────────────── + + it("throws when the network response is null", async () => { + const { producer } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, + }); + + await expect(producer.produce(makeMonitor())).rejects.toThrow("No network response"); + }); + + it("returns null, warns, and does not buffer when toCheck yields nothing", async () => { + const { producer, defaults } = createProducer({ + checkService: { toCheck: jest.fn().mockReturnValue(null) }, + }); + + const result = await producer.produce(makeMonitor()); + + expect(result).toBeNull(); + expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); + expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); + }); + + it("buffers the built check and returns the status and check on success", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; + const check = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + }); + + const result = await producer.produce(makeMonitor()); + + expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); + expect(result).toEqual({ status, check }); + }); +}); diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index e60f82a95..1769ed573 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -80,6 +80,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses }; const monitorsRepository = { findByIds: jest.fn().mockResolvedValue([makeMonitor()]), + findByIdLean: jest.fn().mockResolvedValue(makeMonitor()), findAll: jest.fn().mockResolvedValue([]), updateById: jest.fn().mockResolvedValue({}), }; @@ -94,7 +95,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses produce: jest.fn().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }), }; const checkEvaluator = { - evaluate: jest.fn().mockResolvedValue({ monitor: makeMonitor(), decision: {} }), + evaluate: jest.fn().mockResolvedValue({ monitor: makeMonitor(), statusChange: { monitor: makeMonitor() }, decision: {} }), }; const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) }; const dispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) }; @@ -268,7 +269,7 @@ describe("DBQueueWorker", () => { const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 }); const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); - expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]); + expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1"); expect(mocks.checkProducer.produce).toHaveBeenCalled(); // The check stage only produces; the buffer arms evaluate once the check is durably stored expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); @@ -307,6 +308,35 @@ describe("DBQueueWorker", () => { expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 }); }); + it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => { + const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null }); + const initialMonitor = makeMonitor(); + const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1 + const monitorsRepository = { + findByIdLean: jest.fn().mockResolvedValue(initialMonitor), + updateById: jest.fn().mockResolvedValue({}), + }; + const checksRepository = { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) }; + const checkEvaluator = { + evaluate: jest.fn().mockResolvedValue({ monitor: postWriteMonitor, statusChange: { monitor: postWriteMonitor }, decision: {} }), + }; + const { mocks } = await start({ + mocks: { + jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, + monitorsRepository, + checksRepository, + checkEvaluator, + }, + }); + + // The monitor is read once for the whole backlog, not once per check + expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1); + expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2); + // First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first + expect(mocks.checkEvaluator.evaluate.mock.calls[0][2]).toBe(initialMonitor); + expect(mocks.checkEvaluator.evaluate.mock.calls[1][2]).toBe(postWriteMonitor); + }); + it("cleanup-orphaned job runs the helper's cleanup function", async () => { const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 }); const cleanupFn = jest.fn().mockResolvedValue(undefined); diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 2bd5fa74c..f011c183c 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -7,21 +7,15 @@ import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; -import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts"; -import type { IBufferService } from "../../../src/service/bufferService.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── -const createBuffer = (): jest.Mocked> => ({ - addToBuffer: jest.fn(), -}); - const createMonitorsRepo = () => { const findById = jest.fn(); const updateStatusWindowAndChecks = jest .fn() .mockImplementation( - ( + async ( _id: unknown, _tid: unknown, status: boolean, @@ -30,27 +24,23 @@ const createMonitorsRepo = () => { maxRecentChecks: number, statusPatch?: Partial ) => { - // Grab the monitor that findById was configured to resolve with - const monitor = findById.mock.results.at(-1)?.value; - if (monitor && typeof monitor.then === "function") { - return monitor.then((m: any) => { - m.statusWindow = m.statusWindow || []; - m.statusWindow.push(status); - while (m.statusWindow.length > windowSize) { - m.statusWindow.shift(); - } - m.recentChecks = m.recentChecks || []; - m.recentChecks.push(checkSnapshot); - while (m.recentChecks.length > maxRecentChecks) { - m.recentChecks.shift(); - } - if (statusPatch) { - Object.assign(m, statusPatch); - } - return { ...m }; - }); + // updateMonitorStatus no longer reads the monitor itself — tests seed it via + // findById.mockResolvedValue, so the mock resolves it here as the post-write monitor. + const m: any = await findById(); + m.statusWindow = m.statusWindow || []; + m.statusWindow.push(status); + while (m.statusWindow.length > windowSize) { + m.statusWindow.shift(); } - return Promise.reject(new Error("findById not mocked")); + m.recentChecks = m.recentChecks || []; + m.recentChecks.push(checkSnapshot); + while (m.recentChecks.length > maxRecentChecks) { + m.recentChecks.shift(); + } + if (statusPatch) { + Object.assign(m, statusPatch); + } + return { ...m }; } ); return { @@ -67,23 +57,17 @@ const createMonitorStatsRepo = () => updateByMonitorId: jest.fn(), }) as unknown as jest.Mocked; -const createChecksRepo = () => ({}) as unknown as jest.Mocked; - const createService = (overrides?: { logger?: ReturnType; - buffer?: ReturnType; monitorsRepository?: ReturnType; monitorStatsRepository?: ReturnType; - checksRepository?: ReturnType; }) => { const logger = overrides?.logger ?? createMockLogger(); - const buffer = overrides?.buffer ?? createBuffer(); const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo(); const monitorStatsRepository = overrides?.monitorStatsRepository ?? createMonitorStatsRepo(); - const checksRepository = overrides?.checksRepository ?? createChecksRepo(); - const service = new StatusService(logger as any, buffer as any, monitorsRepository, monitorStatsRepository, checksRepository); - return { service, logger, buffer, monitorsRepository, monitorStatsRepository, checksRepository }; + const service = new StatusService(logger as any, monitorsRepository, monitorStatsRepository); + return { service, logger, monitorsRepository, monitorStatsRepository }; }; const makeMonitor = (overrides?: Partial): Monitor => @@ -264,19 +248,31 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); expect(result.statusChanged).toBe(false); expect(result.prevStatus).toBe("up"); }); + it("uses the prefetched monitor and skips findById when one is passed", async () => { + const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); + + expect(monitorsRepository.findById).not.toHaveBeenCalled(); + expect(result.prevStatus).toBe("up"); + expect(result.monitor).toBe(monitor); + }); + it("pushes to statusWindow and trims to statusWindowSize", async () => { const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); // Atomic push should have been called with the correct status and window size expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith( @@ -297,7 +293,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" })); + await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }), monitor); expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS); expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" })); @@ -315,7 +311,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -333,7 +329,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); @@ -350,7 +346,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -370,7 +366,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -387,7 +383,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -399,7 +395,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck()); + await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); expect(monitor.statusWindow).toEqual([true]); }); @@ -410,23 +406,25 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck()); + await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); expect(monitor.recentChecks).toHaveLength(1); }); it("throws AppError when repository throws", async () => { + const monitor = makeMonitor(); const { service, monitorsRepository } = createService(); - (monitorsRepository.findById as jest.Mock).mockRejectedValue(new Error("db error")); + (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error")); - await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Failed to update monitor"); + await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Failed to update monitor"); }); it("throws AppError with 'Unknown error' for non-Error exceptions", async () => { + const monitor = makeMonitor(); const { service, monitorsRepository } = createService(); - (monitorsRepository.findById as jest.Mock).mockRejectedValue("string error"); + (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error"); - await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Unknown error"); + await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Unknown error"); }); it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => { @@ -444,7 +442,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -462,7 +460,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -484,7 +482,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); @@ -502,7 +500,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); }); @@ -518,7 +516,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -535,7 +533,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -550,7 +548,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -567,7 +565,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -580,7 +578,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down")); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); expect(result.monitor.status).toBe("up"); expect(logger.warn).toHaveBeenCalledWith( @@ -598,7 +596,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck(), monitor); expect(result.code).toBe(201); expect(result.timestamp).toBeGreaterThan(0); @@ -639,7 +637,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.cpu).toBe(true); expect(result.thresholdBreaches?.memory).toBe(false); @@ -656,7 +654,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.memory).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -673,7 +671,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -690,7 +688,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.temp).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -712,7 +710,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} }, } as any); - await service.updateMonitorStatus(response, makeCheck()); + await service.updateMonitorStatus(response, makeCheck(), monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(5); @@ -731,7 +729,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.monitor.status).toBe("breached"); expect(result.statusChanged).toBe(false); @@ -746,7 +744,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); @@ -769,7 +767,7 @@ describe("StatusService", () => { status: false, data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck({ status: false })); + const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); }); @@ -781,7 +779,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.cpu).toBe(false); }); @@ -793,7 +791,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); @@ -807,7 +805,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); @@ -821,7 +819,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); @@ -833,7 +831,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse(undefined as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); @@ -845,7 +843,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({} as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); @@ -860,7 +858,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); @@ -874,7 +872,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -888,7 +886,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -902,7 +900,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.cpu).toBe(false); expect(result.thresholdBreaches?.temp).toBe(false); @@ -917,7 +915,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); @@ -931,7 +929,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -945,7 +943,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(2); @@ -971,7 +969,7 @@ describe("StatusService", () => { status: false, data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck({ status: false })); + const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -991,7 +989,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {} }, } as any); - await service.updateMonitorStatus(response, makeCheck()); + await service.updateMonitorStatus(response, makeCheck(), monitor); expect(monitor.cpuAlertCounter).toBe(0); expect(monitor.memoryAlertCounter).toBe(0); diff --git a/server/test/unit/services/workerHelper.test.ts b/server/test/unit/services/workerHelper.test.ts index 27f06ab6c..96b45d3b3 100644 --- a/server/test/unit/services/workerHelper.test.ts +++ b/server/test/unit/services/workerHelper.test.ts @@ -1,14 +1,11 @@ import { describe, expect, it, jest } from "@jest/globals"; import { WorkerHelper } from "../../../src/worker/worker.helper.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── // -// WorkerHelper is now thin glue: it wires the scheduler's job functions to the -// CheckPipeline / GeoChecksPipeline (acquire→record→decide) and the -// ReactorDispatcher (react), and owns the cleanup jobs. The pipeline and -// dispatcher internals are tested in checkPipeline.test.ts / reactorDispatcher.test.ts; +// WorkerHelper is now thin glue: it wires the geo job to the GeoChecksPipeline and +// owns the cleanup jobs. The pipeline internals are tested in checkPipeline.test.ts; // here we test only the glue + cleanup jobs. const createHelper = (overrides?: Record) => { @@ -42,10 +39,6 @@ const createHelper = (overrides?: Record) => { deleteOlderThan: jest.fn().mockResolvedValue(0), }; - const checkPipeline = { run: jest.fn().mockResolvedValue(null) }; - const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) }; - const reactorDispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) }; - const defaults = { logger: createMockLogger(), checkService, @@ -57,9 +50,6 @@ const createHelper = (overrides?: Record) => { checksRepository, incidentsRepository, geoChecksRepository, - reactorDispatcher, - checkPipeline, - geoCheckPipeline, ...overrides, }; @@ -73,26 +63,11 @@ const createHelper = (overrides?: Record) => { defaults.monitorStatsRepository as any, defaults.checksRepository as any, defaults.incidentsRepository as any, - defaults.geoChecksRepository as any, - defaults.reactorDispatcher as any, - defaults.checkPipeline as any, - defaults.geoCheckPipeline as any + defaults.geoChecksRepository as any ); return { helper, defaults }; }; -const makeMonitor = (overrides?: Partial): Monitor => - ({ - id: "m1", - teamId: "team", - type: "http", - interval: 60000, - status: "up", - geoCheckEnabled: false, - geoCheckLocations: [], - ...overrides, - }) as Monitor; - // ── Tests ──────────────────────────────────────────────────────────────────── describe("WorkerHelper", () => { @@ -103,68 +78,6 @@ describe("WorkerHelper", () => { }); }); - // ── getHeartbeatJob (glue) ──────────────────────────────────────────────── - - describe("getHeartbeatJob", () => { - it("runs the pipeline and dispatches the evaluation to reactors", async () => { - const evaluation = { monitor: { id: "m1" }, decision: {} }; - const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(evaluation) } }); - - await helper.getHeartbeatJob()(makeMonitor()); - - expect(defaults.checkPipeline.run).toHaveBeenCalled(); - expect(defaults.reactorDispatcher.dispatch).toHaveBeenCalledWith(evaluation); - }); - - it("does not dispatch when the pipeline returns null (skipped)", async () => { - const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(null) } }); - - await helper.getHeartbeatJob()(makeMonitor()); - - expect(defaults.reactorDispatcher.dispatch).not.toHaveBeenCalled(); - }); - - it("logs and rethrows when the pipeline throws", async () => { - const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue(new Error("boom")) } }); - - await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toThrow("boom"); - expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "boom" })); - }); - - it("rethrows non-Error failures", async () => { - const { helper } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue("unexpected") } }); - - await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toBe("unexpected"); - }); - }); - - // ── getHeartbeatGeoJob (glue) ───────────────────────────────────────────── - - describe("getHeartbeatGeoJob", () => { - it("runs the geo pipeline", async () => { - const { helper, defaults } = createHelper(); - - await helper.getHeartbeatGeoJob()(makeMonitor()); - - expect(defaults.geoCheckPipeline.run).toHaveBeenCalled(); - }); - - it("catches and logs a pipeline error without rethrowing", async () => { - const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue(new Error("api timeout")) } }); - - await expect(helper.getHeartbeatGeoJob()(makeMonitor())).resolves.toBeUndefined(); - expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "api timeout" })); - }); - - it("logs 'Unknown error' for non-Error failures", async () => { - const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue("string error") } }); - - await helper.getHeartbeatGeoJob()(makeMonitor()); - - expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined })); - }); - }); - // ── getCleanupOrphanedJob ──────────────────────────────────────────────── describe("getCleanupOrphanedJob", () => { From 08e9c02a3a006a8cdc1de5c0885911418581b28b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 19 Jun 2026 10:02:06 -0700 Subject: [PATCH 7/9] update tests --- .../test/unit/services/checkEvaluator.test.ts | 87 ++++++++++++++ .../test/unit/services/checkPipeline.test.ts | 108 ------------------ .../test/unit/services/checkProducer.test.ts | 2 +- .../test/unit/services/statusService.test.ts | 6 +- 4 files changed, 91 insertions(+), 112 deletions(-) create mode 100644 server/test/unit/services/checkEvaluator.test.ts diff --git a/server/test/unit/services/checkEvaluator.test.ts b/server/test/unit/services/checkEvaluator.test.ts new file mode 100644 index 000000000..a22ceb1c7 --- /dev/null +++ b/server/test/unit/services/checkEvaluator.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts"; +import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "m1", + teamId: "team", + type: "http", + interval: 60000, + status: "up", + ...overrides, + }) as Monitor; + +const createEvaluator = (overrides?: Record) => { + const defaults = { + statusService: { + updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }), + }, + monitorStatusPolicy: new MonitorStatusPolicy(), + ...overrides, + }; + const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); + return { evaluator, defaults }; +}; + +describe("CheckEvaluator", () => { + it("threads status, check, and monitor through to the status service", async () => { + const { evaluator, defaults } = createEvaluator(); + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const check = { id: "check-1" } as any; + const monitor = makeMonitor({ status: "up" }); + + await evaluator.evaluate(status, check, monitor); + + expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor); + }); + + it("returns an evaluation that opens an incident on a down transition", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; + const check = { id: "check-1" } as any; + const statusChange = { monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 }; + const { evaluator } = createEvaluator({ + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, + }); + + const result = await evaluator.evaluate(status, check, makeMonitor()); + + expect(result).toMatchObject({ + monitor: statusChange.monitor, + status, + check, + statusChange, + decision: expect.objectContaining({ + shouldCreateIncident: true, + shouldSendNotification: true, + incidentReason: "status_down", + }), + }); + }); + + it("returns an evaluation that resolves an incident on recovery", async () => { + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const statusChange = { monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down", code: 200 }; + const { evaluator } = createEvaluator({ + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, + }); + + const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor({ status: "down" })); + + expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); + }); + + it("produces a no-op decision when status did not change", async () => { + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const { evaluator } = createEvaluator(); + + const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor()); + + expect(result.decision).toMatchObject({ + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + }); + }); +}); diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/checkPipeline.test.ts index 645d58787..008e879fa 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/checkPipeline.test.ts @@ -20,114 +20,6 @@ const activeWindow = () => { return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; }; -// ── CheckPipeline ──────────────────────────────────────────────────────────── - -const createCheckPipeline = (overrides?: Record) => { - const defaults = { - logger: createMockLogger(), - networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, - statusService: { - updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }), - }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - buffer: { addToBuffer: jest.fn() }, - monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, - monitorStatusPolicy: new MonitorStatusPolicy(), - ...overrides, - }; - const checkProducer = new CheckProducer( - defaults.monitorsRepository as any, - defaults.maintenanceWindowsRepository as any, - defaults.checkService as any, - defaults.networkService as any, - defaults.buffer as any, - defaults.logger as any - ); - const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); - const pipeline = new CheckPipeline(checkProducer, checkEvaluator); - return { pipeline, defaults }; -}; - -describe("CheckPipeline", () => { - it("throws when monitor id is missing", async () => { - const { pipeline } = createCheckPipeline(); - await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id"); - }); - - it("skips and sets maintenance status when in maintenance window", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - const result = await pipeline.run(makeMonitor({ status: "up" })); - - expect(result).toBeNull(); - expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] }); - }); - - it("does not re-write status when already in maintenance", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - await pipeline.run(makeMonitor({ status: "maintenance" })); - - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("throws when network response is null", async () => { - const { pipeline } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, - }); - await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response"); - }); - - it("returns null and warns when toCheck returns null", async () => { - const { pipeline, defaults } = createCheckPipeline({ - checkService: { toCheck: jest.fn().mockReturnValue(null) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(result).toBeNull(); - expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); - expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled(); - }); - - it("buffers the check, updates status, and returns an evaluation", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; - const statusChange = { monitor: { id: "m1", status: "down" }, statusChanged: true, prevStatus: "up", code: 500 }; - const { pipeline, defaults } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, - statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith({ id: "check-1" }); - expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalled(); - expect(result).toMatchObject({ - monitor: statusChange.monitor, - status, - check: { id: "check-1" }, - statusChange, - decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, incidentReason: "status_down" }), - }); - }); - - it("proceeds with the check when no maintenance window is active", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, - }); - - await pipeline.run(makeMonitor()); - - expect(defaults.networkService.requestStatus).toHaveBeenCalled(); - }); -}); - // ── GeoChecksPipeline ────────────────────────────────────────────────────────── const createGeoPipeline = (overrides?: Record) => { diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts index bf93c8ea2..4edd12bb5 100644 --- a/server/test/unit/services/checkProducer.test.ts +++ b/server/test/unit/services/checkProducer.test.ts @@ -56,7 +56,7 @@ describe("CheckProducer", () => { const result = await producer.produce(makeMonitor({ status: "up" })); expect(result).toBeNull(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" }); + expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] }); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); }); diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 99db9072a..5712de301 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -564,7 +564,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -579,7 +579,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -599,7 +599,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); From 3b919044e6b7bf3616c4965cc1be0e9e2abb3e96 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 19 Jun 2026 10:25:59 -0700 Subject: [PATCH 8/9] wake worker on job change --- server/src/config/services.ts | 1 + .../maintenance-window.service.ts | 8 ++ server/src/worker/worker.db-queue.ts | 46 ++++++++++-- server/src/worker/worker.interface.ts | 2 + .../test/unit/services/dbQueueWorker.test.ts | 74 +++++++++++++++++++ .../services/maintenanceWindowService.test.ts | 27 ++++++- 6 files changed, 149 insertions(+), 9 deletions(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 9de13f27b..2f1826241 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -334,6 +334,7 @@ export const initializeServices = async ({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, + worker, }); const monitorService = new MonitorService({ worker, diff --git a/server/src/domain/maintenance-windows/maintenance-window.service.ts b/server/src/domain/maintenance-windows/maintenance-window.service.ts index aef0f9fb4..b8601321f 100644 --- a/server/src/domain/maintenance-windows/maintenance-window.service.ts +++ b/server/src/domain/maintenance-windows/maintenance-window.service.ts @@ -1,6 +1,7 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; +import { IWorker } from "@/worker/worker.interface.js"; import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js"; import { AppError } from "@/utils/AppError.js"; import { isWindowActive } from "@/utils/maintenanceWindow.js"; @@ -42,19 +43,23 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { private monitorsRepository: IMonitorsRepository; private maintenanceWindowsRepository: IMaintenanceWindowsRepository; private jobsRepository: IJobsRepository; + private worker: IWorker; constructor({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, + worker, }: { monitorsRepository: IMonitorsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; jobsRepository: IJobsRepository; + worker: IWorker; }) { this.monitorsRepository = monitorsRepository; this.maintenanceWindowsRepository = maintenanceWindowsRepository; this.jobsRepository = jobsRepository; + this.worker = worker; } get serviceName() { @@ -78,6 +83,9 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]); // Rearm jobs to run soon (now + jitter to avoid herding) await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime()); + // Wake the (possibly idle, backed-off) loops so the rearmed jobs run promptly instead of waiting up to POLL_MAX_MS + this.worker.wake("check"); + this.worker.wake("geo-check"); } }; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index bef7a838e..24138695f 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -41,6 +41,17 @@ export class DBQueueWorker implements IWorker { "cleanup-retention": 0, }; + // Per job type poll intervals. These back off on idle and reset on work or wake() + private pollMs: Record = { + check: POLL_MS, + "geo-check": POLL_MS, + evaluate: POLL_MS, + "cleanup-orphaned": POLL_MS, + "cleanup-retention": POLL_MS, + }; + private ticking: Partial> = {}; // Whether a loop is mid tick + private tickFns = new Map void>(); // Store tick fns so we can run them immediately on wake + constructor( private logger: ILogger, private jobsRepository: IJobsRepository, @@ -239,9 +250,10 @@ export class DBQueueWorker implements IWorker { }; private startLoop = (type: JobType) => { - let pollMs = POLL_MS; + this.pollMs[type] = POLL_MS; const tick = async () => { if (this.stopped) return; + this.ticking[type] = true; try { // Claim a batch sized to the free capacity const capacity = CONCURRENCY[type] - this.inFlight[type]; @@ -252,8 +264,8 @@ export class DBQueueWorker implements IWorker { this.inFlight[type]--; // job done, free the slot }); } - // Back off if no jobs founds, reset to base if jobs found - pollMs = capacity > 0 && jobs.length === 0 ? Math.min(pollMs * 2, POLL_MAX_MS) : POLL_MS; + // Back off if no jobs found, reset to base if jobs found + this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS; } catch (error: unknown) { this.logger.error({ message: error instanceof Error ? error.message : String(error), @@ -261,11 +273,26 @@ export class DBQueueWorker implements IWorker { method: `loop:${type}`, }); } - if (!this.stopped) this.timers.set(type, setTimeout(tick, pollMs)); // re-arm unless shutting down + // Clearing the flag and re-arming run synchronously (no await between), so wake() can never interleave here. + this.ticking[type] = false; + if (!this.stopped) this.timers.set(type, setTimeout(tick, this.pollMs[type])); // re-arm unless shutting down }; + this.tickFns.set(type, tick); tick(); }; + // Wake a worker loop after adding a job that's due immediately. + // Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off + wake = (type: JobType) => { + this.pollMs[type] = POLL_MS; + if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing + const tick = this.tickFns.get(type); + if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done + const pending = this.timers.get(type); + if (pending) clearTimeout(pending); // Clear out scheduled tick if any + this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick + }; + // Seed queue private reconcile = async () => { const now = Date.now(); @@ -315,7 +342,11 @@ export class DBQueueWorker implements IWorker { addJob = async (_monitorId: string, monitor: Monitor) => { const now = Date.now(); await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true)); - if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); + this.wake("check"); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { + await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); + this.wake("geo-check"); + } }; deleteJob = async (monitor: Monitor) => { await this.jobsRepository.deleteById(monitor.id); @@ -324,12 +355,15 @@ export class DBQueueWorker implements IWorker { await this.jobsRepository.setActiveById(monitor.id, false); }; resumeJob = async (monitor: Monitor) => { - await this.jobsRepository.setActiveById(monitor.id, true); + await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately + this.wake("check"); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check"); }; updateJob = async (monitor: Monitor) => { await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval); if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away + this.wake("geo-check"); } else { await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check"); } diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index ccaa79d12..fd4160124 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,5 +1,6 @@ import { Monitor } from "@/domain/monitors/monitor.types.js"; import { Check } from "@/domain/checks/check.type.js"; +import { JobType } from "@/domain/jobs/job.type.js"; import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js"; import { MonitorActionDecision } from "@/worker/worker.helper.js"; import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; @@ -70,6 +71,7 @@ export interface IWorker { pauseJob(monitor: Monitor): Promise; resumeJob(monitor: Monitor): Promise; updateJob(monitor: Monitor): Promise; + wake(type: JobType): void; shutdown(): Promise; getMetrics(): Promise; getJobs(pagination: WorkerJobsPagination): Promise; diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index 1769ed573..cf15db8c0 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -479,6 +479,80 @@ describe("DBQueueWorker", () => { }); }); + // ── poll backoff + wake() ─────────────────────────────────────────────────── + // + // An idle loop grows its poll interval up to POLL_MAX_MS (5s). wake() fast-paths + // it back so a freshly-enqueued "run now" job doesn't wait out that backoff. + + describe("poll backoff and wake", () => { + const POLL_MAX_MS = 5000; + const claims = (mocks: any, type: string) => mocks.jobsRepository.claimDueBatch.mock.calls.filter((c: any[]) => c[0] === type).length; + + it("backs off the idle poll interval toward POLL_MAX_MS", async () => { + const { mocks } = await start({ queueMode: "worker" }); // claimDueBatch returns [] → always idle + await jest.advanceTimersByTimeAsync(15000); // let the check loop grow to the cap + const before = claims(mocks, "check"); + await jest.advanceTimersByTimeAsync(POLL_MAX_MS * 2); // ~2 polls at a 5s cadence + const delta = claims(mocks, "check") - before; + // At the cap that's ~2 polls; at the un-backed-off 250ms cadence it would be ~40. + expect(delta).toBeGreaterThanOrEqual(1); + expect(delta).toBeLessThanOrEqual(4); + }); + + it("wake() re-fires an idle, backed-off loop immediately", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // back off the check loop to the cap + const before = claims(mocks, "check"); + + worker.wake("check"); + await jest.advanceTimersByTimeAsync(1); + + expect(claims(mocks, "check")).toBe(before + 1); // fired now, not 5s later + }); + + it("wake() resets the cadence so the loop polls fast again, not at the 5s cap", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // backed off to the cap + worker.wake("check"); + await jest.advanceTimersByTimeAsync(1); // immediate tick (still idle → next poll at ~POLL_MS*2) + const before = claims(mocks, "check"); + + await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms + + // Polled again well within the old 5s cap, proving the interval was reset. + expect(claims(mocks, "check")).toBe(before + 1); + }); + + it("wake() is a no-op for a loop that isn't running in this process", async () => { + const { worker, mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false }); // no loops + expect(() => worker.wake("check")).not.toThrow(); + await jest.advanceTimersByTimeAsync(1); + expect(claims(mocks, "check")).toBe(0); + }); + + it("addJob wakes the check loop so a new monitor's first check runs promptly", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // backed off to the cap + const before = claims(mocks, "check"); + + await worker.addJob("m2", makeMonitor({ id: "m2" })); + await jest.advanceTimersByTimeAsync(1); + + expect(claims(mocks, "check")).toBe(before + 1); + }); + + it("resumeJob wakes the check loop so a resumed monitor runs promptly", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // backed off to the cap + const before = claims(mocks, "check"); + + await worker.resumeJob(makeMonitor()); + await jest.advanceTimersByTimeAsync(1); + + expect(claims(mocks, "check")).toBe(before + 1); + }); + }); + // ── shutdown ──────────────────────────────────────────────────────────────── describe("shutdown", () => { diff --git a/server/test/unit/services/maintenanceWindowService.test.ts b/server/test/unit/services/maintenanceWindowService.test.ts index 07d19e405..5c7387da8 100644 --- a/server/test/unit/services/maintenanceWindowService.test.ts +++ b/server/test/unit/services/maintenanceWindowService.test.ts @@ -3,6 +3,7 @@ import { MaintenanceWindowService } from "../../../src/domain/maintenance-window import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts"; +import type { IWorker } from "../../../src/worker/worker.interface.ts"; import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -33,16 +34,23 @@ const createJobsRepo = () => markMonitorsDue: jest.fn().mockResolvedValue(0), }) as unknown as jest.Mocked; +const createWorker = () => + ({ + wake: jest.fn(), + }) as unknown as jest.Mocked; + const createService = (overrides?: { monitorsRepository?: ReturnType; maintenanceWindowsRepository?: ReturnType; jobsRepository?: ReturnType; + worker?: ReturnType; }) => { const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo(); const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo(); const jobsRepository = overrides?.jobsRepository ?? createJobsRepo(); - const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository }); - return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository }; + const worker = overrides?.worker ?? createWorker(); + const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker }); + return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker }; }; const makeWindow = (overrides?: Partial): MaintenanceWindow => ({ @@ -275,7 +283,7 @@ describe("MaintenanceWindowService", () => { it("flips covered monitors to initializing when deleting an active window", async () => { const maintenanceWindowsRepository = createMaintenanceWindowsRepo(); (maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] })); - const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository }); + const { service, monitorsRepository, jobsRepository, worker } = createService({ maintenanceWindowsRepository }); await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" }); @@ -283,6 +291,19 @@ describe("MaintenanceWindowService", () => { expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]); // Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number)); + // ...and the idle loops are woken so the re-armed jobs run promptly + expect(worker.wake).toHaveBeenCalledWith("check"); + expect(worker.wake).toHaveBeenCalledWith("geo-check"); + }); + + it("does not wake the loops when no monitor leaves maintenance", async () => { + const maintenanceWindowsRepository = createMaintenanceWindowsRepo(); + (maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeWindow({ active: false, monitorIds: ["mon-1"] })); + const { service, worker } = createService({ maintenanceWindowsRepository }); + + await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" }); + + expect(worker.wake).not.toHaveBeenCalled(); }); it("excludes the deleted window from the overlap check", async () => { From 755134ee7f9c525bb28ab20935be868cf5c3f667 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 09:15:39 -0700 Subject: [PATCH 9/9] add poll backoff tests --- .../test/unit/services/dbQueueWorker.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index cf15db8c0..21515c34f 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -499,6 +499,52 @@ describe("DBQueueWorker", () => { expect(delta).toBeLessThanOrEqual(4); }); + it("resets the poll interval to POLL_MS once a tick finds work after being idle", async () => { + let busy = false; + const claimDueBatch = jest.fn(async (type: string) => (busy && type === "check" ? [makeJob()] : [])); + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch } } }); + + await jest.advanceTimersByTimeAsync(15000); // idle → backed off to the 5s cap + busy = true; + await jest.advanceTimersByTimeAsync(POLL_MAX_MS); // the next (backed-off) tick fires, finds work → resets pollMs to POLL_MS + const before = claims(mocks, "check"); + + await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms + + // Polling fast again (~2 in 500ms), not once per 5s — proves the interval reset on finding work. + expect(claims(mocks, "check") - before).toBeGreaterThanOrEqual(1); + }); + + it("does not back off a loop that is at capacity (capacity === 0)", async () => { + // cleanup-orphaned has concurrency 1; hold its one job in-flight so capacity stays 0. + // The backoff condition is `capacity > 0 && jobs.length === 0`, so an at-capacity loop + // must keep polling at POLL_MS (a freed slot should be picked up promptly, not after 5s). + const gate = deferred(); + let handed = false; + const claimDueBatch = jest.fn(async (type: string) => { + if (type === "cleanup-orphaned" && !handed) { + handed = true; + return [makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 })]; + } + return []; + }); + const helper = { + getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn(() => gate.promise)), // blocks → keeps the slot full + getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + }; + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, helper } }); + + const before = claims(mocks, "cleanup-orphaned"); // the blocking job is now in-flight → capacity 0 + await jest.advanceTimersByTimeAsync(1000); + const delta = claims(mocks, "cleanup-orphaned") - before; + + // ~4 polls at POLL_MS; if an at-capacity loop wrongly backed off it would be ~1. + expect(delta).toBeGreaterThanOrEqual(3); + + gate.resolve(); + await jest.advanceTimersByTimeAsync(1); // let the held job finish for a clean shutdown + }); + it("wake() re-fires an idle, backed-off loop immediately", async () => { const { worker, mocks } = await start({ queueMode: "worker" }); await jest.advanceTimersByTimeAsync(15000); // back off the check loop to the cap