diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index 88b5a18e2..21c653e8a 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -17,12 +17,9 @@ import { INetworkService, NetworkService } from "@/service/networkService.js"; import { IBufferService, BufferService } from "@/service/bufferService.js"; import { IStatusService, StatusService } from "@/service/statusService.js"; import { SecretsRotationService } from "@/service/encryption/secretsRotationService.js"; -import { IQueueWorker } from "@/worker/worker.interface.js"; -import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; +import { IQueueWorker, JobHandlers } from "@/worker/worker.interface.js"; import { WorkerHelper } from "@/worker/worker.helper.js"; -import { CheckProducer } from "@/worker/worker.check-producer.js"; -import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; -import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; +import { GeoChecksPipeline } from "@/worker/worker.geo-pipeline.js"; import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; @@ -42,6 +39,7 @@ import { GrpcProvider } from "@/service/network/GrpcProvider.js"; import { WebSocketProvider } from "@/service/network/WebSocketProvider.js"; import { DNSProvider } from "@/service/network/DNSProvider.js"; import { AppError } from "@/utils/AppError.js"; +import { WorkerPipeline } from "@/worker/worker.pipeline.js"; export interface WorkerServices { worker: IQueueWorker; networkService: INetworkService; @@ -103,37 +101,33 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig ]); const proxyResolver = new ProxyResolver(proxiesRepository, settingsService, logger); - const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository); + const bufferService = new BufferService(logger, geoChecksService, dockerLogsService, settingsService, (checks): Promise => + pipeline.ingestChecks(checks) + ); const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); - const monitorStatusPolicy = new MonitorStatusPolicy(); const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger); - // *********************** - // Reactors and dispatcher - // Handles notifications and incidents - // *********************** - const notificationReactor = new NotificationReactor(notificationsService); const incidentReactor = new IncidentReactor(incidentService); const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); - // *********************** - // Check producer/evaluator - // Handles creating and evaluatiog checks - // *********************** - const checkProducer = new CheckProducer( + const pipeline = new WorkerPipeline({ + logger, monitorsRepository, maintenanceWindowsRepository, + checksRepository, + jobsRepository, checkService, networkService, proxyResolver, bufferService, dockerLogsService, egressService, - logger - ); - const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy, logger); - const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger); + statusService, + dispatcher: reactorDispatcher, + }); + + const geoCheckPipeline = new GeoChecksPipeline(monitorsRepository, maintenanceWindowsRepository, geoChecksService, bufferService, logger); // *********************** // Worker @@ -154,19 +148,22 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig egressService ); + const handlers: JobHandlers = { + check: pipeline.handleCheck, + evaluate: pipeline.handleEvaluate, + "geo-check": geoCheckPipeline.handle, + "cleanup-orphaned": workerHelper.getCleanupOrphanedJob(), + "cleanup-retention": workerHelper.getCleanupRetentionJob(), + egress: workerHelper.getEgressRecoveryJob(), + }; + const worker = await DBQueueWorker.create({ logger, isDbConnected: () => mongoose.connection.readyState === 1, jobsRepository, monitorsRepository, - checksRepository, - checkService, bufferService, - checkProducer, - checkEvaluator, - geoCheckPipeline, - dispatcher: reactorDispatcher, - helper: workerHelper, + handlers, queueWorkersRepository, queueMode: envSettings.queueMode, queuePrimaryProcesses: envSettings.queuePrimaryProcesses, diff --git a/server/src/domain/incidents/incident.service.ts b/server/src/domain/incidents/incident.service.ts index a9851d85e..f8c79cb80 100644 --- a/server/src/domain/incidents/incident.service.ts +++ b/server/src/domain/incidents/incident.service.ts @@ -8,7 +8,7 @@ import type { IMonitorsRepository } from "@/domain/monitors/monitor.repository.i import type { IUsersRepository } from "@/domain/users/user.repository.interface.js"; import type { Incident, IncidentSummary } from "@/domain/incidents/incident.type.js"; import type { User } from "@/domain/users/user.type.js"; -import type { MonitorActionDecision } from "@/worker/worker.helper.js"; +import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; import type { ILogger } from "@/utils/logger.js"; import { DateRange } from "@/types/query.js"; diff --git a/server/src/domain/notifications/notification.message-builder.ts b/server/src/domain/notifications/notification.message-builder.ts index 92f54ada8..641d9527c 100644 --- a/server/src/domain/notifications/notification.message-builder.ts +++ b/server/src/domain/notifications/notification.message-builder.ts @@ -1,6 +1,6 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js"; -import type { MonitorActionDecision } from "@/worker/worker.helper.js"; +import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { NotificationMessage, NotificationType, diff --git a/server/src/domain/notifications/notification.service.ts b/server/src/domain/notifications/notification.service.ts index eb0e333d4..affc0ea8a 100644 --- a/server/src/domain/notifications/notification.service.ts +++ b/server/src/domain/notifications/notification.service.ts @@ -5,7 +5,7 @@ import type { NotificationMessage } from "@/domain/notifications/notification.ty import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js"; import { INotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js"; -import type { MonitorActionDecision } from "@/worker/worker.helper.js"; +import type { MonitorActionDecision } from "@/worker/worker.interface.js"; import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; import { ILogger } from "@/utils/logger.js"; import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; diff --git a/server/src/service/bufferService.ts b/server/src/service/bufferService.ts index 576b07ab1..3b5619748 100755 --- a/server/src/service/bufferService.ts +++ b/server/src/service/bufferService.ts @@ -3,18 +3,15 @@ import type { GeoCheck } from "@/domain/geo-checks/geo-check.type.js"; import type { IGeoChecksService } from "../domain/geo-checks/geo-check.service.js"; import type { ILogger } from "@/utils/logger.js"; import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; -import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; -import { ICheckService } from "../domain/checks/check.service.js"; import { DockerLog } from "@/domain/docker/docker-log.type.js"; import { IDockerLogsService } from "@/domain/docker/docker-log.service.js"; -import { PendingCheck } from "@/domain/jobs/job.type.js"; const SERVICE_NAME = "BufferService"; +export type CheckIngest = (checks: Check[]) => Promise; export interface IBufferService { addToBuffer(check: Check): void; addGeoCheckToBuffer(geoCheck: GeoCheck): void; addDockerLogToBuffer(dockerLog: DockerLog): void; - ingestChecks(checks: Check[]): Promise; scheduleNextFlush(): void; flushBuffer(): Promise; flushGeoBuffer(): Promise; @@ -31,28 +28,20 @@ export class BufferService implements IBufferService { private geoBuffer: GeoCheck[]; private dockerLogBuffer: DockerLog[]; private bufferTimer: NodeJS.Timeout | null = null; - private checksService: ICheckService; private geoChecksService: IGeoChecksService; private dockerLogsService: IDockerLogsService; - private jobsRepository: IJobsRepository; - - // Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated. - private unarmed: Map = new Map(); constructor( logger: ILogger, - checkService: ICheckService, geoChecksService: IGeoChecksService, dockerLogsService: IDockerLogsService, settingsService: ISettingsService, - jobsRepository: IJobsRepository + private ingest: CheckIngest ) { this.BUFFER_TIMEOUT = settingsService.getSettings().nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute this.logger = logger; - this.checksService = checkService; this.geoChecksService = geoChecksService; this.dockerLogsService = dockerLogsService; - this.jobsRepository = jobsRepository; this.SERVICE_NAME = SERVICE_NAME; this.buffer = []; this.geoBuffer = []; @@ -104,38 +93,6 @@ export class BufferService implements IBufferService { } } - ingestChecks = async (checks: Check[]) => { - if (checks.length > 0) await this.checksService.createChecks(checks); - - // Group by monitor, starting from any entries a previous flush failed to arm - const byMonitor = this.unarmed; - this.unarmed = new Map(); - for (const check of checks) { - const pending = byMonitor.get(check.metadata.monitorId) ?? []; - pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() }); - byMonitor.set(check.metadata.monitorId, pending); - } - if (byMonitor.size === 0) return; - - const now = Date.now(); - const ops: Promise[] = []; - for (const [monitorId, pendingChecks] of byMonitor) { - ops.push( - this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => { - // The checks are already stored, so a failed arm is retried on the next flush rather than dropped - this.unarmed.set(monitorId, pendingChecks); - this.logger.error({ - message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`, - service: this.SERVICE_NAME, - method: "ingestChecks", - }); - return false; - }) - ); - } - await Promise.all(ops); - }; - scheduleNextFlush() { if (this.bufferTimer) { clearTimeout(this.bufferTimer); @@ -160,19 +117,14 @@ export class BufferService implements IBufferService { } async flushBuffer() { - if (this.buffer.length === 0 && this.unarmed.size === 0) { - return; - } // Take the batch first so a write that drains it can't be appended to mid-flush const batch = this.buffer; this.buffer = []; try { - this.logger.debug({ - message: `Flushing ${batch.length} checks to database`, - service: this.SERVICE_NAME, - method: "flushBuffer", - }); - await this.ingestChecks(batch); + if (batch.length > 0) { + this.logger.debug({ message: `Flushing ${batch.length} checks to database`, service: this.SERVICE_NAME, method: "flushBuffer" }); + } + await this.ingest(batch); } catch (error: unknown) { this.logger.error({ message: error instanceof Error ? error.message : "Unknown error", diff --git a/server/src/worker/worker.check-evaluator.ts b/server/src/worker/worker.check-evaluator.ts deleted file mode 100644 index 4c4c4a7ac..000000000 --- a/server/src/worker/worker.check-evaluator.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { MonitorStatusResponse } from "@/types/network.js"; -import { MonitorEvaluation } from "@/worker/worker.interface.js"; -import { Check } from "@/domain/checks/check.type.js"; -import { Monitor } from "@/domain/monitors/monitor.type.js"; -import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; -import { IStatusService } from "@/service/statusService.js"; -import { MonitorActionDecision } from "@/worker/worker.helper.js"; -import { ILogger } from "@/utils/logger.js"; - -export interface ICheckEvaluator { - evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise; -} - -const SERVICE_NAME = "CheckEvaluator"; - -export class CheckEvaluator implements ICheckEvaluator { - constructor( - private statusService: IStatusService, - private monitorStatusPolicy: IMonitorStatusPolicy, - private logger: ILogger - ) {} - - // A check taken while the instance had no outbound connectivity says nothing about the target. - // It is left out of the status window, running stats and monitor status entirely, so that no incident - // opens for it and no spurious "resolved"/"up" fires once egress returns. - private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => { - this.logger.debug({ - message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`, - service: SERVICE_NAME, - method: "evaluate", - }); - const decision: MonitorActionDecision = { - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }; - return { - monitor, - status, - check, - statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() }, - decision, - }; - }; - - evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => { - if (check.egressStatus === "degraded") { - return this.skipDegradedEgressCheck(status, check, monitor); - } - - // **************************** - // Step 3: Evaluate and return result to reactors - // **************************** - const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor); - - // Step 5. Get decisions and create an evaluation obj - const decision = this.monitorStatusPolicy.evaluate(statusChangeResult); - const evaluation: MonitorEvaluation = { - monitor: statusChangeResult.monitor, - status, - check, - statusChange: statusChangeResult, - decision, - }; - return evaluation; - }; -} diff --git a/server/src/worker/worker.check-producer.ts b/server/src/worker/worker.check-producer.ts deleted file mode 100644 index 1d25bdd92..000000000 --- a/server/src/worker/worker.check-producer.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { Monitor } from "@/domain/monitors/monitor.type.js"; -import { Check } from "@/domain/checks/check.type.js"; -import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js"; -import { AppError } from "@/utils/AppError.js"; -import { ILogger } from "@/utils/logger.js"; -import { IBufferService } from "@/service/bufferService.js"; -import { INetworkService } from "@/service/networkService.js"; -import { ICheckService } from "@/domain/checks/check.service.js"; -import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; -import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; -import { isWindowActive } from "@/utils/maintenanceWindow.js"; -import { IProxyResolver } from "@/service/network/ProxyResolver.js"; -import { IDockerLogsService } from "@/domain/docker/docker-log.service.js"; -import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js"; -import type { EgressStatus } from "@/domain/egress/egress.type.js"; -import { isEgressAttributable } from "@/domain/monitors/monitor.type.js"; - -export interface ICheckProducer { - produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>; -} - -const SERVICE_NAME = "CheckProducer"; - -export class CheckProducer implements ICheckProducer { - constructor( - private monitorsRepository: IMonitorsRepository, - private maintenanceWindowsRepository: IMaintenanceWindowsRepository, - private checkService: ICheckService, - private networkService: INetworkService, - private proxyResolver: IProxyResolver, - private bufferService: IBufferService, - private dockerLogsService: IDockerLogsService, - private egressService: IEgressService, - private logger: ILogger - ) {} - - private async isInMaintenanceWindow(monitorId: string, teamId: string) { - const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId); - const now = new Date(); - return windows.some((w) => isWindowActive(w, now)); - } - - private async resolveDockerTlsKey(monitor: Monitor): Promise { - if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined; - const dockerTlsKey = await this.monitorsRepository.findDockerTlsKeyById(monitor.id); - return dockerTlsKey ?? undefined; - } - - // Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the - // two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx, - // or a 200 with a content mismatch). - private peerAnswered(status: MonitorStatusResponse): boolean { - return status.peerResponded ?? isHttpStatusCode(status.code); - } - - private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean { - return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status); - } - - // The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown. - private async assessEgress(monitorId: string): Promise { - try { - return await this.egressService.assessAfterFailure(); - } catch (error: unknown) { - this.logger.warn({ - message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "assessEgress", - }); - return null; - } - } - - produce = async (monitor: Monitor) => { - if (!monitor.id) { - throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" }); - } - // **************************** - // Step 1: Acquire - // **************************** - - // Step 1a: Maintenance window gate - skip if in maintenance - const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId); - if (maintenanceWindowActive) { - this.logger.debug({ - message: `Monitor ${monitor.id} is in maintenance window`, - service: SERVICE_NAME, - method: "produce", - }); - if (monitor.status !== "maintenance") { - // Clear the status window to avoid incidents being created on next check - await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] }); - } - return null; - } - - // Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the - // degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is - // evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage. - if (monitor.status === "maintenance") { - monitor = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" }); - } - - // Step 1b: Acquire status - const proxyUrl = await this.proxyResolver.resolve(monitor); - const dockerTlsKey = await this.resolveDockerTlsKey(monitor); - - const status = await this.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey }); - if (!status) { - throw new Error("No network response"); - } - - // **************************** - // Step 2: Record - // **************************** - - // Step 2a: Create & record a check, return null if fail. - // Built before the egress assessment below so that the probe's duration stays out of `createdAt`. - const check = this.checkService.toCheck(status); - if (!check) { - this.logger.warn({ - message: `No check could be built for monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "produce", - details: { code: status.code, message: status.message }, - }); - return null; - } - - // Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the - // target. Null means the egress check is disabled (or failed internally) and the check is treated as usual. - const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null; - if (egressStatus !== null) { - check.egressStatus = egressStatus; - } - - // Step 2c: Add to buffer - this.bufferService.addToBuffer(check); - - // Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops - // being evaluated, so the results either side of a spell never end up adjacent and cross the threshold - // on stale data. The length guard keeps this to one write per spell rather than one per degraded check, - // and it runs after the check is buffered so that a failed write cannot cost us the check. - if (egressStatus === "degraded" && monitor.statusWindow?.length) { - await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] }); - } - - // Step 2e: Handle docker logs - if (status.type === "docker") { - const dockerLogs = await this.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse); - for (const dockerLog of dockerLogs) { - this.bufferService.addDockerLogToBuffer(dockerLog); - } - } - return { status, check }; - }; -} diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 05da1ee74..eb419e67b 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -1,15 +1,8 @@ -import { IQueueWorker } from "@/worker/worker.interface.js"; +import { IQueueWorker, JobHandlers } from "@/worker/worker.interface.js"; import { type Job, type JobType, LOCK_MS } from "@/domain/jobs/job.type.js"; import { ILogger } from "@/utils/logger.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; -import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; -import { ICheckService } from "@/domain/checks/check.service.js"; -import { ICheckProducer } from "@/worker/worker.check-producer.js"; -import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; -import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; -import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; -import { IWorkerHelper } from "@/worker/worker.helper.js"; import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; @@ -36,14 +29,8 @@ export interface DBQueueWorkerDependencies { isDbConnected: () => boolean; jobsRepository: IJobsRepository; // forwarded monitorsRepository: IMonitorsRepository; // forwarded - checksRepository: IChecksRepository; - checkService: ICheckService; + handlers: JobHandlers; bufferService: IBufferService; - checkProducer: ICheckProducer; - checkEvaluator: ICheckEvaluator; - geoCheckPipeline: ICheckPipeline; - dispatcher: IReactorDispatcher; - helper: IWorkerHelper; queueWorkersRepository: IQueueWorkersRepository; // forwarded queueMode: QueueMode; // forwarded queuePrimaryProcesses: boolean; // consumed immediately @@ -65,15 +52,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { egress: 0, }; - private checksRepository: IChecksRepository; - private checkProducer: ICheckProducer; - private checkService: ICheckService; - private checkEvaluator: ICheckEvaluator; - private dispatcher: IReactorDispatcher; - private geoCheckPipeline: ICheckPipeline; - private helper: IWorkerHelper; private bufferService: IBufferService; private isDbConnected: () => boolean; + private handlers: JobHandlers; constructor(dependencies: DBQueueWorkerDependencies) { super( @@ -85,13 +66,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { dependencies.workerId ); this.processesJobs = dependencies.queuePrimaryProcesses === true || dependencies.queueMode === "worker"; - this.checksRepository = dependencies.checksRepository; - this.checkProducer = dependencies.checkProducer; - this.checkService = dependencies.checkService; - this.checkEvaluator = dependencies.checkEvaluator; - this.dispatcher = dependencies.dispatcher; - this.geoCheckPipeline = dependencies.geoCheckPipeline; - this.helper = dependencies.helper; + this.handlers = dependencies.handlers; this.bufferService = dependencies.bufferService; this.isDbConnected = dependencies.isDbConnected; } @@ -104,64 +79,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { private getInFlightCount = () => Object.values(this.inFlight).reduce((sum, n) => sum + n, 0); - // ******************** - // Stage 1: This can eventually be offloaded to a separate service - // ******************** - - private runCheck = async (job: Job) => { - if (!job.refId) return; - const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId - if (!monitor) return; - await this.checkProducer.produce(monitor); - }; - - // ******************** - // Stage 2: Evaluator loop stays here - // ******************** - - private runEvaluate = async (job: Job) => { - if (!job.refId || job.pendingChecks.length === 0) return; - const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId); - - const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId - if (!monitor) { - await this.jobsRepository.pullEvaluated(job.id, pendingCheckIds); - return; - } - const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks); - - let current = monitor; - for (const check of checks) { - const status = this.checkService.toStatusResponse(check); - const evaluation = await this.checkEvaluator.evaluate(status, check, current); - await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications - const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]); - if (!leaseHeld) { - // Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice - this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" }); - return; - } - current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check - } - - // Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever - const found = new Set(checks.map((check) => check.id)); - const missing = pendingCheckIds.filter((checkId) => !found.has(checkId)); - if (missing.length > 0) { - this.logger.warn({ - message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`, - service: SERVICE_NAME, - method: "runEvaluate", - }); - await this.jobsRepository.pullEvaluated(job.id, missing); - } - }; - - private runGeoCheck = async (job: Job) => { - const monitor = await this.monitorsRepository.findByIdLean(job.refId!); - if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff - }; - // Extend the lock while a job runs so a slow-but-alive job is never reclaimed. private renewLock = async (job: Job): Promise => { try { @@ -183,27 +100,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { if (!renewLock) clearInterval(renewTimer); }, LOCK_RENEW_MS); try { - switch (job.type) { - case "check": - await this.runCheck(job); - break; - case "evaluate": - await this.runEvaluate(job); - break; - case "geo-check": - await this.runGeoCheck(job); - break; - case "cleanup-orphaned": - await this.helper.getCleanupOrphanedJob()(); // Get job and execute - break; - case "cleanup-retention": - await this.helper.getCleanupRetentionJob()(); // Get job and execute - break; - case "egress": - await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery - break; - } - + await this.handlers[job.type](job); if (job.intervalMs === null) { // One-shot job await this.jobsRepository.recordOneShot(job.id, Date.now()); diff --git a/server/src/worker/worker.check-pipeline.ts b/server/src/worker/worker.geo-pipeline.ts similarity index 79% rename from server/src/worker/worker.check-pipeline.ts rename to server/src/worker/worker.geo-pipeline.ts index 9b97de354..4787b24fd 100644 --- a/server/src/worker/worker.check-pipeline.ts +++ b/server/src/worker/worker.geo-pipeline.ts @@ -1,5 +1,5 @@ import { Monitor } from "@/domain/monitors/monitor.type.js"; -import { MonitorEvaluation } from "@/worker/worker.interface.js"; +import { JobHandler, MonitorEvaluation } from "@/worker/worker.interface.js"; import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; import { isWindowActive } from "@/utils/maintenanceWindow.js"; import { ILogger } from "@/utils/logger.js"; @@ -7,15 +7,19 @@ import { IBufferService } from "@/service/bufferService.js"; import { AppError } from "@/utils/AppError.js"; import { supportsGeoCheck } from "@/domain/monitors/monitor.type.js"; import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; +import { Job } from "@/domain/jobs/job.type.js"; +import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; -const SERVICE_NAME = "CheckPipeline"; +const SERVICE_NAME = "GeoChecksPipeline"; -export interface ICheckPipeline { +export interface IGeoChecksPipeline { + handle: JobHandler; run(monitor: Monitor): Promise; // null = skipped } -export class GeoChecksPipeline implements ICheckPipeline { +export class GeoChecksPipeline implements IGeoChecksPipeline { constructor( + private monitorsRepository: IMonitorsRepository, private maintenanceWindowsRepository: IMaintenanceWindowsRepository, private geoChecksService: IGeoChecksService, private bufferService: IBufferService, @@ -28,6 +32,12 @@ export class GeoChecksPipeline implements ICheckPipeline { return maintenanceWindows.some((window) => isWindowActive(window, now)); } + handle: JobHandler = async (job: Job) => { + if (!job.refId) return; + const monitor = await this.monitorsRepository.findByIdLean(job.refId); + if (monitor) await this.run(monitor); + }; + run = async (monitor: Monitor): Promise => { // **************************** // Step 1: Acquire @@ -36,7 +46,7 @@ export class GeoChecksPipeline implements ICheckPipeline { // Step 1a: Guards - skip if unsupported or not enabled if (!monitor.id) { - throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getHeartbeatGeoJob" }); + throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "run" }); } if (!monitor.geoCheckEnabled) { @@ -46,7 +56,7 @@ export class GeoChecksPipeline implements ICheckPipeline { this.logger.debug({ message: `Monitor ${monitor.id} type does not support geo checks, skipping`, service: SERVICE_NAME, - method: "runGeoChecksPipeline", + method: "run", }); return null; } @@ -55,7 +65,7 @@ export class GeoChecksPipeline implements ICheckPipeline { this.logger.warn({ message: `No geo check locations configured for monitor ${monitor.id}`, service: SERVICE_NAME, - method: "runGeoChecksPipeline", + method: "run", }); return null; } @@ -66,7 +76,7 @@ export class GeoChecksPipeline implements ICheckPipeline { this.logger.debug({ message: `Monitor ${monitor.id} is in maintenance window, skipping geo check`, service: SERVICE_NAME, - method: "runGeoChecksPipeline", + method: "run", }); return null; } @@ -81,7 +91,7 @@ export class GeoChecksPipeline implements ICheckPipeline { this.logger.warn({ message: `No geo check could be built for monitor ${monitor.id}`, service: SERVICE_NAME, - method: "runGeoChecksPipeline", + method: "run", }); return null; } @@ -92,7 +102,7 @@ export class GeoChecksPipeline implements ICheckPipeline { this.logger.debug({ message: `Geo check job executed for monitor ${monitor.id}`, service: SERVICE_NAME, - method: "runGeoChecksPipeline", + method: "run", }); return null; }; diff --git a/server/src/worker/worker.helper.ts b/server/src/worker/worker.helper.ts index 9e3a6aa8e..fdb3eea59 100644 --- a/server/src/worker/worker.helper.ts +++ b/server/src/worker/worker.helper.ts @@ -20,20 +20,6 @@ export interface IWorkerHelper { getEgressRecoveryJob(): (job: Job) => Promise; } -export interface MonitorActionDecision { - shouldCreateIncident: boolean; - shouldResolveIncident: boolean; - shouldSendNotification: boolean; - incidentReason: "status_down" | "threshold_breach" | null; - notificationReason: "status_change" | "threshold_breach" | null; - thresholdBreaches?: { - cpu?: boolean; - memory?: boolean; - disk?: boolean; - temp?: boolean; - }; -} - export class WorkerHelper implements IWorkerHelper { static SERVICE_NAME = SERVICE_NAME; diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index bb6cb0c53..379040010 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,17 +1,32 @@ import { Monitor } from "@/domain/monitors/monitor.type.js"; import { Check } from "@/domain/checks/check.type.js"; -import { JobType } from "@/domain/jobs/job.type.js"; +import { Job, JobType } from "@/domain/jobs/job.type.js"; import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js"; -import { MonitorActionDecision } from "@/worker/worker.helper.js"; import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; +export type JobHandler = (job: Job) => Promise; +export type JobHandlers = Record; +export interface MonitorActionDecision { + shouldCreateIncident: boolean; + shouldResolveIncident: boolean; + shouldSendNotification: boolean; + incidentReason: "status_down" | "threshold_breach" | null; + notificationReason: "status_change" | "threshold_breach" | null; + thresholdBreaches?: { + cpu?: boolean; + memory?: boolean; + disk?: boolean; + temp?: boolean; + }; +} + export type MonitorEvaluation = { monitor: Monitor; status: MonitorStatusResponse; // raw result from networkService.requestStatus check: Check; statusChange: StatusChangeResult; // from statusService.updateMonitorStatus - decision: MonitorActionDecision; // from MonitorStatusPolicy + decision: MonitorActionDecision; }; export type WorkerJobFailure = { diff --git a/server/src/worker/worker.monitor-status-policy.ts b/server/src/worker/worker.monitor-status-policy.ts deleted file mode 100644 index 4ef36b2a9..000000000 --- a/server/src/worker/worker.monitor-status-policy.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { StatusChangeResult } from "@/types/network.js"; -import { MonitorActionDecision } from "@/worker/worker.helper.js"; - -export interface IMonitorStatusPolicy { - evaluate(statusChange: StatusChangeResult): MonitorActionDecision; -} - -export class MonitorStatusPolicy implements IMonitorStatusPolicy { - evaluate = (statusChangeResult: StatusChangeResult): MonitorActionDecision => { - const { monitor, statusChanged, prevStatus } = statusChangeResult; - - // Initialize result - const decision: MonitorActionDecision = { - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }; - - if (!statusChanged) { - return decision; - } - - if (monitor.status === "down") { - // Monitor went down (unreachable) - decision.shouldCreateIncident = true; - decision.shouldSendNotification = true; - decision.incidentReason = "status_down"; - decision.notificationReason = "status_change"; - } else if (monitor.status === "breached") { - // Hardware monitor exceeded thresholds - decision.shouldCreateIncident = true; - decision.shouldSendNotification = true; - decision.incidentReason = "threshold_breach"; - decision.notificationReason = "threshold_breach"; - } else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) { - // Monitor recovered from down or breached state - decision.shouldResolveIncident = true; - decision.shouldSendNotification = true; - decision.notificationReason = "status_change"; - } - - return decision; - }; -} diff --git a/server/src/worker/worker.pipeline.ts b/server/src/worker/worker.pipeline.ts new file mode 100644 index 000000000..bdcb86c00 --- /dev/null +++ b/server/src/worker/worker.pipeline.ts @@ -0,0 +1,336 @@ +import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; +import { ICheckService } from "@/domain/checks/check.service.js"; +import { Check } from "@/domain/checks/check.type.js"; +import { IDockerLogsService } from "@/domain/docker/docker-log.service.js"; +import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js"; +import type { EgressStatus } from "@/domain/egress/egress.type.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; +import { PendingCheck } from "@/domain/jobs/job.type.js"; +import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; +import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { isEgressAttributable, Monitor } from "@/domain/monitors/monitor.type.js"; +import { IBufferService } from "@/service/bufferService.js"; +import { IProxyResolver } from "@/service/network/ProxyResolver.js"; +import { INetworkService } from "@/service/networkService.js"; +import { IStatusService } from "@/service/statusService.js"; +import { DockerStatusPayload, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js"; +import { AppError } from "@/utils/AppError.js"; +import { ILogger } from "@/utils/logger.js"; +import { isWindowActive } from "@/utils/maintenanceWindow.js"; +import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; +import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js"; + +const SERVICE_NAME = "WorkerPipeline"; + +// The three entry points: the queue calls the two handlers, the buffer calls ingestChecks. +// produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer. +export interface IWorkerPipeline { + handleCheck: JobHandler; + ingestChecks(checks: Check[]): Promise; + handleEvaluate: JobHandler; +} + +export interface WorkerPipelineDependencies { + logger: ILogger; + monitorsRepository: IMonitorsRepository; + maintenanceWindowsRepository: IMaintenanceWindowsRepository; + checksRepository: IChecksRepository; + jobsRepository: IJobsRepository; + checkService: ICheckService; + networkService: INetworkService; + proxyResolver: IProxyResolver; + bufferService: IBufferService; + dockerLogsService: IDockerLogsService; + egressService: IEgressService; + statusService: IStatusService; + dispatcher: IReactorDispatcher; +} + +export class WorkerPipeline implements IWorkerPipeline { + static SERVICE_NAME = SERVICE_NAME; + + private unarmed: Map = new Map(); + + constructor(private deps: WorkerPipelineDependencies) {} + + private isInMaintenanceWindow = async (monitorId: string, teamId: string) => { + const windows = await this.deps.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId); + const now = new Date(); + return windows.some((window) => isWindowActive(window, now)); + }; + + // **************************************************************** + // Stage 1: Check job. Probe the host and buffer the result. + // **************************************************************** + + handleCheck: JobHandler = async (job) => { + if (!job.refId) return; + const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId); + if (!monitor) return; + await this.produce(monitor); + }; + + produce = async (monitor: Monitor) => { + if (!monitor.id) { + throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" }); + } + // **************************** + // Step 1: Acquire + // **************************** + + // Step 1a: Maintenance window gate - skip if in maintenance + const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId); + if (maintenanceWindowActive) { + this.deps.logger.debug({ + message: `Monitor ${monitor.id} is in maintenance window`, + service: SERVICE_NAME, + method: "produce", + }); + if (monitor.status !== "maintenance") { + // Clear the status window to avoid incidents being created on next check + await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] }); + } + return null; + } + + // Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the + // degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is + // evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage. + if (monitor.status === "maintenance") { + monitor = await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" }); + } + + // Step 1b: Acquire status + const proxyUrl = await this.deps.proxyResolver.resolve(monitor); + const dockerTlsKey = await this.resolveDockerTlsKey(monitor); + + const status = await this.deps.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey }); + if (!status) { + throw new Error("No network response"); + } + + // **************************** + // Step 2: Record + // **************************** + + // Step 2a: Create & record a check, return null if fail. + // Built before the egress assessment below so that the probe's duration stays out of `createdAt`. + const check = this.deps.checkService.toCheck(status); + if (!check) { + this.deps.logger.warn({ + message: `No check could be built for monitor ${monitor.id}`, + service: SERVICE_NAME, + method: "produce", + details: { code: status.code, message: status.message }, + }); + return null; + } + + // Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the + // target. Null means the egress check is disabled (or failed internally) and the check is treated as usual. + const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null; + if (egressStatus !== null) { + check.egressStatus = egressStatus; + } + + // Step 2c: Add to buffer + this.deps.bufferService.addToBuffer(check); + + // Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops + // being evaluated, so the results either side of a spell never end up adjacent and cross the threshold + // on stale data. The length guard keeps this to one write per spell rather than one per degraded check, + // and it runs after the check is buffered so that a failed write cannot cost us the check. + if (egressStatus === "degraded" && monitor.statusWindow?.length) { + await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] }); + } + + // Step 2e: Handle docker logs + if (status.type === "docker") { + const dockerLogs = await this.deps.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse); + for (const dockerLog of dockerLogs) { + this.deps.bufferService.addDockerLogToBuffer(dockerLog); + } + } + return { status, check }; + }; + + private async resolveDockerTlsKey(monitor: Monitor): Promise { + if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined; + const dockerTlsKey = await this.deps.monitorsRepository.findDockerTlsKeyById(monitor.id); + return dockerTlsKey ?? undefined; + } + + // Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the + // two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx, + // or a 200 with a content mismatch). + private peerAnswered(status: MonitorStatusResponse): boolean { + return status.peerResponded ?? isHttpStatusCode(status.code); + } + + private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean { + return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status); + } + + // The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown. + private async assessEgress(monitorId: string): Promise { + try { + return await this.deps.egressService.assessAfterFailure(); + } catch (error: unknown) { + this.deps.logger.warn({ + message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`, + service: SERVICE_NAME, + method: "assessEgress", + }); + return null; + } + } + + // **************************************************************** + // Stage 2: Ingestion + // Called by the buffer, checks are inserted and evaluation jobs created + // **************************************************************** + + ingestChecks = async (checks: Check[]) => { + if (checks.length > 0) await this.deps.checkService.createChecks(checks); + + // Group by monitor, starting from any entries a previous flush failed to arm + const byMonitor = this.unarmed; + this.unarmed = new Map(); + for (const check of checks) { + const pending = byMonitor.get(check.metadata.monitorId) ?? []; + pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() }); + byMonitor.set(check.metadata.monitorId, pending); + } + if (byMonitor.size === 0) return; + + const now = Date.now(); + const ops: Promise[] = []; + for (const [monitorId, pendingChecks] of byMonitor) { + ops.push( + this.deps.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => { + // The checks are already stored, so a failed arm is retried on the next ingest rather than dropped + this.unarmed.set(monitorId, pendingChecks); + this.deps.logger.error({ + message: `Could not arm evaluation for monitor ${monitorId}, retrying on next ingest: ${error instanceof Error ? error.message : String(error)}`, + service: SERVICE_NAME, + method: "ingestChecks", + }); + return false; + }) + ); + } + await Promise.all(ops); + }; + + // **************************************************************** + // Stage 3: Evaluation + // Apply each pending check to the monitor -> decide -> react. + // **************************************************************** + + handleEvaluate: JobHandler = async (job) => { + if (!job.refId || job.pendingChecks.length === 0) return; + const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId); + + const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId); // job row has no teamId + if (!monitor) { + await this.deps.jobsRepository.pullEvaluated(job.id, pendingCheckIds); + return; + } + const checks = await this.deps.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks); + + let current = monitor; + for (const check of checks) { + const status = this.deps.checkService.toStatusResponse(check); + const evaluation = await this.evaluateCheck(status, check, current); + await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications + const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]); + if (!leaseHeld) { + // Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice + this.deps.logger.warn({ + message: `Lost lease on ${job.id} after check ${check.id}, stopping`, + service: SERVICE_NAME, + method: "handleEvaluate", + }); + return; + } + current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check + } + + // Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever + const found = new Set(checks.map((check) => check.id)); + const missing = pendingCheckIds.filter((checkId) => !found.has(checkId)); + if (missing.length > 0) { + this.deps.logger.warn({ + message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`, + service: SERVICE_NAME, + method: "handleEvaluate", + }); + await this.deps.jobsRepository.pullEvaluated(job.id, missing); + } + }; + + // A check taken while the instance had no outbound connectivity says nothing about the target. + // It is left out of the status window, running stats and monitor status entirely, so that no incident + // opens for it and no spurious "resolved"/"up" fires once egress returns. + private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => { + this.deps.logger.debug({ + message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`, + service: SERVICE_NAME, + method: "evaluateCheck", + }); + const decision: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }; + return { + monitor, + status, + check, + statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() }, + decision, + }; + }; + + evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise => { + if (check.egressStatus === "degraded") { + return this.skipDegradedEgressCheck(status, check, monitor); + } + + const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor); + + const decision = this.decide(statusChange); + return { monitor: statusChange.monitor, status, check, statusChange, decision }; + }; + + private decide = (statusChange: StatusChangeResult): MonitorActionDecision => { + const { monitor, statusChanged, prevStatus } = statusChange; + const decision: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }; + if (!statusChanged) return decision; + + if (monitor.status === "down") { + decision.shouldCreateIncident = true; + decision.shouldSendNotification = true; + decision.incidentReason = "status_down"; + decision.notificationReason = "status_change"; + } else if (monitor.status === "breached") { + decision.shouldCreateIncident = true; + decision.shouldSendNotification = true; + decision.incidentReason = "threshold_breach"; + decision.notificationReason = "threshold_breach"; + } else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) { + decision.shouldResolveIncident = true; + decision.shouldSendNotification = true; + decision.notificationReason = "status_change"; + } + return decision; + }; +} diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index 47370df0b..c4de7f6ec 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -1,7 +1,5 @@ import { jest } from "@jest/globals"; -import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts"; -import { CheckProducer } from "../../src/worker/worker.check-producer.ts"; -import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts"; +import { WorkerPipeline } from "../../src/worker/worker.pipeline.ts"; import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts"; import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts"; import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts"; @@ -139,31 +137,35 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { nextEgressStatus = status; }; - const checkProducer = new CheckProducer( - monitorsRepo as any, - maintenanceWindowsRepo as any, - checkService as any, - networkService as any, - proxyResolver as any, - bufferStub as any, - dockerLogsService as any, - egressService as any, - logger - ); - const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger); - const notificationReactor = new NotificationReactor(notificationsService as any); const incidentReactor = new IncidentReactor(incidentService as any); const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); + // The harness drives stages 1 and 3 directly, so the stage 2 repositories are never reached. + const pipeline = new WorkerPipeline({ + logger, + monitorsRepository: monitorsRepo as any, + maintenanceWindowsRepository: maintenanceWindowsRepo as any, + checksRepository: { findUnevaluatedByMonitorId: jest.fn() } as any, + jobsRepository: { upsertEvaluate: jest.fn(), pullEvaluated: jest.fn() } as any, + checkService: checkService as any, + networkService: networkService as any, + proxyResolver: proxyResolver as any, + bufferService: bufferStub as any, + dockerLogsService: dockerLogsService as any, + egressService: egressService as any, + statusService, + dispatcher: reactorDispatcher, + }); + // Mirror the production check→evaluate flow: read the monitor fresh each cycle (so the // accumulated statusWindow is visible), produce a check, then evaluate against that monitor. const heartbeatJob = async (seedMonitor: Monitor) => { const monitor = await monitorsRepo.findByIdLean(seedMonitor.id); if (!monitor) return; - const result = await checkProducer.produce(monitor); + const result = await pipeline.produce(monitor); if (!result) return; // skipped (e.g. maintenance window) - const evaluation = await checkEvaluator.evaluate(result.status, result.check, monitor); + const evaluation = await pipeline.evaluateCheck(result.status, result.check, monitor); await reactorDispatcher.dispatch(evaluation); }; diff --git a/server/test/integration/evaluateCursorMultiWorker.test.ts b/server/test/integration/evaluateCursorMultiWorker.test.ts index 57ce10e46..50be5ea72 100644 --- a/server/test/integration/evaluateCursorMultiWorker.test.ts +++ b/server/test/integration/evaluateCursorMultiWorker.test.ts @@ -8,6 +8,7 @@ import JobModel from "../../src/domain/jobs/job.model.ts"; import { CheckService } from "../../src/domain/checks/check.service.ts"; import { BufferService } from "../../src/service/bufferService.ts"; import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts"; +import { WorkerPipeline } from "../../src/worker/worker.pipeline.ts"; import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts"; import { createMockLogger } from "../helpers/createMockLogger.ts"; import type { Job } from "../../src/domain/jobs/job.type.ts"; @@ -29,8 +30,9 @@ import type { ILogger } from "../../src/utils/logger.ts"; // // The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each // is applied, so a late batch is always evaluated. This drives the real time-series query, the -// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an -// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's. +// real jobs collection, the real buffer flush, the real pipeline ingest and evaluate stages and the +// real DBQueueWorker job path against an in-process mongod, so the handoff semantics under test +// are MongoDB's, not a mock's. let mongod: MongoMemoryServer; @@ -105,16 +107,39 @@ describe("Evaluate cursor with more than one processing worker", () => { let evaluator: DBQueueWorker; let evaluatorJobsRepository: MongoJobsRepository; - // One BufferService per worker process, each flushing into the same collection and jobs table. - const createWorkerBuffer = (workerId: string) => - new BufferService( + // One pipeline and BufferService per worker process, each flushing into the same collection and + // jobs table. Only stages 2 and 3 are exercised: the status service is stubbed to record which + // checks reached evaluation, and stage 1's network dependencies are never called. + const createWorkerPipeline = (workerId: string) => { + const jobsRepository = new MongoJobsRepository(workerId); + const bufferService = new BufferService( logger, - checkService, stubGeoChecksService as any, stubDockerLogsService as any, stubSettingsService as any, - new MongoJobsRepository(workerId) + (checks): Promise => pipeline.ingestChecks(checks) ); + const pipeline = new WorkerPipeline({ + logger, + monitorsRepository: monitorsRepository as any, + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) } as any, + checksRepository, + jobsRepository, + checkService, + networkService: { requestStatus: jest.fn() } as any, + proxyResolver: { resolve: jest.fn() } as any, + bufferService, + dockerLogsService: { buildDockerLogs: jest.fn() } as any, + statusService: { + updateMonitorStatus: async (_status: MonitorStatusResponse, check: Check, monitor: Monitor) => { + evaluatedCheckIds.push(check.id); + return { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 }; + }, + } as any, + dispatcher: { dispatch: jest.fn().mockResolvedValue(undefined) } as any, + }); + return { pipeline, bufferService, jobsRepository }; + }; // Claim every due evaluate job and run it through the worker's real per-job path. const runDueEvaluateJobs = async () => { @@ -130,35 +155,28 @@ describe("Evaluate cursor with more than one processing worker", () => { monitorsRepository.seed(makeMonitor()); checksRepository = new MongoChecksRepository(logger); checkService = new CheckService(monitorsRepository, logger, checksRepository); - bufferA = createWorkerBuffer("worker-a"); - bufferB = createWorkerBuffer("worker-b"); - evaluatedCheckIds = []; - evaluatorJobsRepository = new MongoJobsRepository("worker-b"); + const workerA = createWorkerPipeline("worker-a"); + const workerB = createWorkerPipeline("worker-b"); + bufferA = workerA.bufferService; + bufferB = workerB.bufferService; + + const unused = jest.fn().mockResolvedValue(undefined); + evaluatorJobsRepository = workerB.jobsRepository; evaluator = new DBQueueWorker({ logger, isDbConnected: () => true, jobsRepository: evaluatorJobsRepository, monitorsRepository, - checksRepository, - checkService, bufferService: bufferB, - checkProducer: { produce: jest.fn() }, - checkEvaluator: { - evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => { - evaluatedCheckIds.push(check.id); - return { - monitor, - status, - check, - statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 }, - decision: {}, - }; - }, - } as any, - geoCheckPipeline: { run: jest.fn() }, - dispatcher: { dispatch: jest.fn().mockResolvedValue(undefined) }, - helper: {} as any, + handlers: { + check: unused, + evaluate: workerB.pipeline.handleEvaluate, + "geo-check": unused, + "cleanup-orphaned": unused, + "cleanup-retention": unused, + egress: unused, + }, queueWorkersRepository: {} as any, queueMode: "worker", queuePrimaryProcesses: true, diff --git a/server/test/integration/incidentLifecycle.test.ts b/server/test/integration/incidentLifecycle.test.ts index 6f7d8bc9a..037713b36 100644 --- a/server/test/integration/incidentLifecycle.test.ts +++ b/server/test/integration/incidentLifecycle.test.ts @@ -6,7 +6,7 @@ import type { IMonitorsRepository } from "../../src/domain/monitors/monitor.repo import type { IUsersRepository } from "../../src/domain/users/user.repository.interface.ts"; import type { INotificationMessageBuilder } from "../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../src/domain/monitors/monitor.type.ts"; -import type { MonitorActionDecision } from "../../src/worker/worker.helper.ts"; +import type { MonitorActionDecision } from "../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── diff --git a/server/test/unit/services/bufferService.test.ts b/server/test/unit/services/bufferService.test.ts index 2788af4be..bfcba9eaa 100644 --- a/server/test/unit/services/bufferService.test.ts +++ b/server/test/unit/services/bufferService.test.ts @@ -1,10 +1,8 @@ import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals"; import { BufferService } from "../../../src/service/bufferService.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; -import type { ICheckService } from "../../../src/domain/checks/check.service.ts"; import type { IGeoChecksService } from "../../../src/domain/geo-checks/geo-check.service.ts"; import type { ISettingsService } from "../../../src/domain/app-settings/app-settings.service.ts"; -import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { GeoCheck } from "../../../src/domain/geo-checks/geo-check.type.ts"; import type { IDockerLogsService } from "../../../src/domain/docker/docker-log.service.ts"; @@ -12,11 +10,6 @@ import type { DockerLog } from "../../../src/domain/docker/docker-log.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── -const createMockCheckService = () => - ({ - createChecks: jest.fn().mockResolvedValue([]), - }) as unknown as jest.Mocked; - const createMockGeoChecksService = () => ({ createGeoChecks: jest.fn().mockResolvedValue([]), @@ -32,11 +25,6 @@ const createMockSettingsService = (nodeEnv: string = "development") => getSettings: jest.fn().mockReturnValue({ nodeEnv }), }) as unknown as jest.Mocked; -const createMockJobsRepository = () => - ({ - upsertEvaluate: jest.fn().mockResolvedValue(true), - }) as unknown as jest.Mocked; - const makeCheck = (overrides?: Partial): Check => ({ id: "check-1", @@ -72,13 +60,12 @@ const makeDockerLog = (overrides?: Partial): DockerLog => const createService = (nodeEnv: string = "development") => { const logger = createMockLogger(); - const checkService = createMockCheckService(); + const ingest = jest.fn<(checks: Check[]) => Promise>().mockResolvedValue(undefined); const geoChecksService = createMockGeoChecksService(); const dockerLogsService = createMockDockerLogsService(); const settingsService = createMockSettingsService(nodeEnv); - const jobsRepository = createMockJobsRepository(); - const service = new BufferService(logger as any, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository); - return { service, logger, checkService, geoChecksService, dockerLogsService, jobsRepository }; + const service = new BufferService(logger as any, geoChecksService, dockerLogsService, settingsService, ingest); + return { service, logger, ingest, geoChecksService, dockerLogsService }; }; // ── Tests ──────────────────────────────────────────────────────────────────── @@ -127,17 +114,17 @@ describe("BufferService", () => { describe("addToBuffer", () => { it("adds a check to the buffer", async () => { - const { service, checkService } = createService(); + const { service, ingest } = createService(); const check = makeCheck(); service.addToBuffer(check); await service.flushBuffer(); - expect(checkService.createChecks).toHaveBeenCalledWith([check]); + expect(ingest).toHaveBeenCalledWith([check]); }); it("adds multiple checks to the buffer", async () => { - const { service, checkService } = createService(); + const { service, ingest } = createService(); const check1 = makeCheck({ id: "c1" }); const check2 = makeCheck({ id: "c2" }); @@ -145,7 +132,7 @@ describe("BufferService", () => { service.addToBuffer(check2); await service.flushBuffer(); - expect(checkService.createChecks).toHaveBeenCalledWith([check1, check2]); + expect(ingest).toHaveBeenCalledWith([check1, check2]); }); it("logs error if push throws", () => { @@ -268,21 +255,20 @@ describe("BufferService", () => { }); it("flushes all buffers when timer fires", async () => { - const { service, checkService, geoChecksService, dockerLogsService } = createService(); + const { service, ingest, geoChecksService, dockerLogsService } = createService(); service.addToBuffer(makeCheck()); service.addGeoCheckToBuffer(makeGeoCheck()); service.addDockerLogToBuffer(makeDockerLog()); await jest.advanceTimersByTimeAsync(1000); - expect(checkService.createChecks).toHaveBeenCalled(); + expect(ingest).toHaveBeenCalledWith([expect.objectContaining({ id: "check-1" })]); expect(geoChecksService.createGeoChecks).toHaveBeenCalled(); expect(dockerLogsService.createDockerLogs).toHaveBeenCalled(); }); it("reschedules after flush completes", async () => { - const { service, checkService } = createService(); - (checkService.createChecks as jest.Mock).mockResolvedValue([]); + const { service, ingest } = createService(); service.addToBuffer(makeCheck()); await jest.advanceTimersByTimeAsync(1000); @@ -291,12 +277,12 @@ describe("BufferService", () => { service.addToBuffer(makeCheck({ id: "c2" })); await jest.advanceTimersByTimeAsync(1000); - expect(checkService.createChecks).toHaveBeenCalledTimes(2); + expect(ingest).toHaveBeenCalledTimes(2); }); it("reschedules even when flush throws", async () => { - const { service, checkService, logger } = createService(); - (checkService.createChecks as jest.Mock).mockRejectedValueOnce(new Error("DB down")); + const { service, ingest, logger } = createService(); + ingest.mockRejectedValueOnce(new Error("DB down")); service.addToBuffer(makeCheck()); await jest.advanceTimersByTimeAsync(1000); @@ -309,11 +295,10 @@ describe("BufferService", () => { ); // Should still reschedule — add another check and flush - (checkService.createChecks as jest.Mock).mockResolvedValue([]); service.addToBuffer(makeCheck({ id: "c2" })); await jest.advanceTimersByTimeAsync(1000); - expect(checkService.createChecks).toHaveBeenCalledTimes(2); + expect(ingest).toHaveBeenCalledTimes(2); }); it("logs error and reschedules when flush throws past its own catch", async () => { @@ -350,25 +335,26 @@ describe("BufferService", () => { // ── flushBuffer ────────────────────────────────────────────────────── describe("flushBuffer", () => { - it("does nothing when buffer is empty", async () => { - const { service, checkService } = createService(); + it("hands an empty batch to ingest so the pipeline can retry anything it left unarmed", async () => { + const { service, ingest, logger } = createService(); await service.flushBuffer(); - expect(checkService.createChecks).not.toHaveBeenCalled(); + expect(ingest).toHaveBeenCalledWith([]); + expect(logger.debug).not.toHaveBeenCalled(); }); - it("flushes checks to checksService and clears buffer", async () => { - const { service, checkService } = createService(); + it("hands the batch to ingest and clears the buffer", async () => { + const { service, ingest } = createService(); const check = makeCheck(); service.addToBuffer(check); await service.flushBuffer(); - expect(checkService.createChecks).toHaveBeenCalledWith([check]); + expect(ingest).toHaveBeenNthCalledWith(1, [check]); // Buffer should be empty now await service.flushBuffer(); - expect(checkService.createChecks).toHaveBeenCalledTimes(1); + expect(ingest).toHaveBeenNthCalledWith(2, []); }); it("logs debug message before flushing", async () => { @@ -386,68 +372,11 @@ describe("BufferService", () => { ); }); - it("arms the evaluate stage once per distinct monitor after the checks are stored", async () => { - const { service, jobsRepository } = createService(); - service.addToBuffer(makeCheck({ id: "c1", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } })); - service.addToBuffer(makeCheck({ id: "c2", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } })); - service.addToBuffer(makeCheck({ id: "c3", metadata: { monitorId: "mon-2", teamId: "team-1", type: "http" } })); - - await service.flushBuffer(); - - const createdAt = Date.parse("2026-01-01T00:00:00.000Z"); - expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2); - expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith( - "mon-1", - [ - { checkId: "c1", createdAt }, - { checkId: "c2", createdAt }, - ], - expect.any(Number) - ); - expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number)); - }); - - it("retries a failed arm on the next flush without re-storing the checks", async () => { - const { service, checkService, jobsRepository, logger } = createService(); - (jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed")); - service.addToBuffer(makeCheck({ id: "c1" })); - - await service.flushBuffer(); - expect(checkService.createChecks).toHaveBeenCalledTimes(1); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" })); - - await service.flushBuffer(); // buffer is empty, only the retry runs - expect(checkService.createChecks).toHaveBeenCalledTimes(1); - expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2); - expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number)); - - await service.flushBuffer(); // nothing left to retry - expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2); - }); - - it("ingestChecks with an empty batch neither writes nor arms", async () => { - const { service, checkService, jobsRepository } = createService(); - - await service.ingestChecks([]); - - expect(checkService.createChecks).not.toHaveBeenCalled(); - expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); - }); - - it("does not arm the evaluate stage when the check write fails", async () => { - const { service, checkService, jobsRepository } = createService(); - (checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed")); - service.addToBuffer(makeCheck()); - - await service.flushBuffer(); - - expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); - }); - it("clears buffer even on error to prevent infinite retries", async () => { - const { service, checkService, logger } = createService(); - (checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed")); - service.addToBuffer(makeCheck()); + const { service, ingest, logger } = createService(); + ingest.mockRejectedValueOnce(new Error("DB write failed")); + const check = makeCheck(); + service.addToBuffer(check); await service.flushBuffer(); @@ -457,14 +386,14 @@ describe("BufferService", () => { method: "flushBuffer", }) ); - // Buffer should be cleared + // Buffer should be cleared: the failed batch is not handed over again await service.flushBuffer(); - expect(checkService.createChecks).toHaveBeenCalledTimes(1); + expect(ingest).toHaveBeenNthCalledWith(2, []); }); it("logs 'Unknown error' for non-Error thrown values", async () => { - const { service, checkService, logger } = createService(); - (checkService.createChecks as jest.Mock).mockRejectedValue(null); + const { service, ingest, logger } = createService(); + ingest.mockRejectedValue(null); service.addToBuffer(makeCheck()); await service.flushBuffer(); @@ -609,28 +538,28 @@ describe("BufferService", () => { describe("shutdown", () => { it("stops the flush timer and flushes all buffers", async () => { - const { service, checkService, geoChecksService, dockerLogsService } = createService(); + const { service, ingest, geoChecksService, dockerLogsService } = createService(); service.addToBuffer(makeCheck()); service.addGeoCheckToBuffer(makeGeoCheck()); service.addDockerLogToBuffer(makeDockerLog()); await service.shutdown(); - expect(checkService.createChecks).toHaveBeenCalledTimes(1); + expect(ingest).toHaveBeenCalledTimes(1); expect(geoChecksService.createGeoChecks).toHaveBeenCalledTimes(1); expect(dockerLogsService.createDockerLogs).toHaveBeenCalledTimes(1); // Timer is cleared: advancing past the flush interval triggers no further flush. await jest.advanceTimersByTimeAsync(60 * 1000); - expect(checkService.createChecks).toHaveBeenCalledTimes(1); + expect(ingest).toHaveBeenCalledTimes(1); }); it("is safe to call with empty buffers", async () => { - const { service, checkService, geoChecksService, dockerLogsService } = createService(); + const { service, ingest, geoChecksService, dockerLogsService } = createService(); await service.shutdown(); - expect(checkService.createChecks).not.toHaveBeenCalled(); + expect(ingest).toHaveBeenCalledWith([]); expect(geoChecksService.createGeoChecks).not.toHaveBeenCalled(); expect(dockerLogsService.createDockerLogs).not.toHaveBeenCalled(); }); diff --git a/server/test/unit/services/checkEvaluator.test.ts b/server/test/unit/services/checkEvaluator.test.ts deleted file mode 100644 index 23d64a0b0..000000000 --- a/server/test/unit/services/checkEvaluator.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { describe, expect, it, jest } from "@jest/globals"; -import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts"; -import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; -import { createMockLogger } from "../../helpers/createMockLogger.ts"; - -const makeMonitor = (overrides?: Partial): Monitor => - ({ - id: "m1", - teamId: "team", - type: "http", - interval: 60000, - status: "up", - ...overrides, - }) as Monitor; - -const createEvaluator = (overrides?: Record) => { - const defaults = { - statusService: { - updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }), - }, - monitorStatusPolicy: new MonitorStatusPolicy(), - logger: createMockLogger(), - ...overrides, - }; - const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any); - return { evaluator, defaults }; -}; - -describe("CheckEvaluator", () => { - it("threads status, check, and monitor through to the status service", async () => { - const { evaluator, defaults } = createEvaluator(); - const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; - const check = { id: "check-1" } as any; - const monitor = makeMonitor({ status: "up" }); - - await evaluator.evaluate(status, check, monitor); - - expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor); - }); - - it("returns an evaluation that opens an incident on a down transition", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; - const check = { id: "check-1" } as any; - const statusChange = { monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 }; - const { evaluator } = createEvaluator({ - statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, - }); - - const result = await evaluator.evaluate(status, check, makeMonitor()); - - expect(result).toMatchObject({ - monitor: statusChange.monitor, - status, - check, - statusChange, - decision: expect.objectContaining({ - shouldCreateIncident: true, - shouldSendNotification: true, - incidentReason: "status_down", - }), - }); - }); - - it("returns an evaluation that resolves an incident on recovery", async () => { - const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; - const statusChange = { monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down", code: 200 }; - const { evaluator } = createEvaluator({ - statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, - }); - - const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor({ status: "down" })); - - expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); - }); - - it("produces a no-op decision when status did not change", async () => { - const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; - const { evaluator } = createEvaluator(); - - const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor()); - - expect(result.decision).toMatchObject({ - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - }); - }); - - // ── degraded egress ─────────────────────────────────────────────────────── - - it("short-circuits a check flagged with degraded egress without touching monitor status", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; - const check = { id: "check-1", egressStatus: "degraded" } as any; - const monitor = makeMonitor({ status: "up" }); - const { evaluator, defaults } = createEvaluator(); - - const result = await evaluator.evaluate(status, check, monitor); - - expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled(); - expect(result.monitor).toBe(monitor); - expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 }); - expect(result.statusChange.timestamp).toEqual(expect.any(Number)); - expect(result.decision).toEqual({ - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }); - expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") })); - }); - - it("evaluates a check flagged with egress ok as normal", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; - const check = { id: "check-1", egressStatus: "ok" } as any; - const { evaluator, defaults } = createEvaluator(); - - await evaluator.evaluate(status, check, makeMonitor()); - - expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything()); - }); -}); diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts deleted file mode 100644 index 460f54ee0..000000000 --- a/server/test/unit/services/checkProducer.test.ts +++ /dev/null @@ -1,461 +0,0 @@ -import { describe, expect, it, jest } from "@jest/globals"; -import { CheckProducer } from "../../../src/worker/worker.check-producer.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; -import { createMockLogger } from "../../helpers/createMockLogger.ts"; -import { NETWORK_ERROR } from "../../../src/types/network.ts"; - -const makeMonitor = (overrides?: Partial): Monitor => - ({ - id: "m1", - teamId: "team", - type: "http", - interval: 60000, - status: "up", - ...overrides, - }) as Monitor; - -// A maintenance window straddling "now" — active per isWindowActive. -const activeWindow = () => { - const now = Date.now(); - return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; -}; - -const createProducer = (overrides?: Record) => { - const defaults = { - logger: createMockLogger(), - monitorsRepository: { updateById: jest.fn().mockResolvedValue({}), findDockerTlsKeyById: jest.fn().mockResolvedValue(null) }, - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, - proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) }, - buffer: { addToBuffer: jest.fn() }, - dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) }, - ...overrides, - }; - const producer = new CheckProducer( - defaults.monitorsRepository as any, - defaults.maintenanceWindowsRepository as any, - defaults.checkService as any, - defaults.networkService as any, - defaults.proxyResolver as any, - defaults.buffer as any, - defaults.dockerLogsService as any, - defaults.egressService as any, - defaults.logger as any - ); - return { producer, defaults }; -}; - -describe("CheckProducer", () => { - it("throws when monitor id is missing", async () => { - const { producer } = createProducer(); - await expect(producer.produce({} as Monitor)).rejects.toThrow("No monitor id"); - }); - - // ── maintenance gate ────────────────────────────────────────────────────── - - it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => { - const { producer, defaults } = createProducer({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - const result = await producer.produce(makeMonitor({ status: "up" })); - - expect(result).toBeNull(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] }); - expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); - expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); - }); - - it("does not re-write status when already in maintenance", async () => { - const { producer, defaults } = createProducer({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - const result = await producer.produce(makeMonitor({ status: "maintenance" })); - - expect(result).toBeNull(); - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("moves a monitor out of maintenance explicitly once the window is over", async () => { - const monitor = makeMonitor({ status: "maintenance" }); - const { producer, defaults } = createProducer({ - monitorsRepository: { - updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }), - findDockerTlsKeyById: jest.fn().mockResolvedValue(null), - }, - }); - - await producer.produce(monitor); - - // The exit used to be a side effect of evaluating the resulting check, which the degraded-egress - // short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage. - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" }); - expect(defaults.networkService.requestStatus).toHaveBeenCalled(); - }); - - it("does not touch the status of a monitor that was not in maintenance", async () => { - const { producer, defaults } = createProducer(); - - await producer.produce(makeMonitor({ status: "up" })); - - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("proceeds with the check when the maintenance window is inactive", async () => { - const { producer, defaults } = createProducer({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, - }); - - await producer.produce(makeMonitor()); - - expect(defaults.networkService.requestStatus).toHaveBeenCalled(); - }); - - // ── proxy resolution ────────────────────────────────────────────────────── - - it("passes the resolved proxy url into requestStatus", async () => { - const { producer, defaults } = createProducer({ - proxyResolver: { resolve: jest.fn().mockResolvedValue("http://proxy.example.com:8080") }, - }); - const monitor = makeMonitor(); - - await producer.produce(monitor); - - expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: "http://proxy.example.com:8080" }); - }); - - it("still produces a check when the resolver returns undefined", async () => { - const { producer, defaults } = createProducer(); - const monitor = makeMonitor(); - - const result = await producer.produce(monitor); - - expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined }); - expect(result).toEqual({ status: expect.objectContaining({ monitorId: "m1" }), check: { id: "check-1" } }); - }); - - // ── acquire / record ────────────────────────────────────────────────────── - - // ── docker tls key resolution ───────────────────────────────────────────── - - it("fetches the stored key for a docker monitor with a key set and passes it in the context", async () => { - const { producer, defaults } = createProducer({ - monitorsRepository: { updateById: jest.fn(), findDockerTlsKeyById: jest.fn().mockResolvedValue("v1.abc123.iv.tag.data") }, - }); - const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true }); - - await producer.produce(monitor); - - expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1"); - expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: "v1.abc123.iv.tag.data" }); - }); - - it("does not fetch a key for a docker monitor without one set", async () => { - const { producer, defaults } = createProducer(); - const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKeySet: false }); - - await producer.produce(monitor); - - expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled(); - expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined }); - }); - - it("does not fetch a key for non-docker monitors even if the flag is set", async () => { - const { producer, defaults } = createProducer(); - - await producer.produce(makeMonitor({ type: "http", dockerTlsKeySet: true })); - - expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled(); - }); - - it("passes undefined when the flag is set but the repository has no key", async () => { - const { producer, defaults } = createProducer(); - const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true }); - - await producer.produce(monitor); - - expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1"); - expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined }); - }); - - it("throws when the network response is null", async () => { - const { producer } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, - }); - - await expect(producer.produce(makeMonitor())).rejects.toThrow("No network response"); - }); - - it("returns null, warns, and does not buffer when toCheck yields nothing", async () => { - const { producer, defaults } = createProducer({ - checkService: { toCheck: jest.fn().mockReturnValue(null) }, - }); - - const result = await producer.produce(makeMonitor()); - - expect(result).toBeNull(); - expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); - expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); - }); - - it("buffers the built check and returns the status and check on success", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; - const check = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - }); - - const result = await producer.produce(makeMonitor()); - - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); - expect(result).toEqual({ status, check }); - }); - - it("builds and buffers docker logs for a docker status", async () => { - const status = { type: "docker", monitorId: "m1", teamId: "team", status: true, code: 200, message: "OK", payload: {} }; - const dockerLogs = [{ id: "docker-log-1" }, { id: "docker-log-2" }]; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, - buffer: { addToBuffer: jest.fn(), addDockerLogToBuffer: jest.fn() }, - dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue(dockerLogs) }, - }); - - await producer.produce(makeMonitor({ type: "docker" })); - - expect(defaults.dockerLogsService.buildDockerLogs).toHaveBeenCalledWith(status); - expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(1, dockerLogs[0]); - expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(2, dockerLogs[1]); - }); - - it("does not build docker logs for a non-docker status", async () => { - const { producer, defaults } = createProducer(); - - await producer.produce(makeMonitor()); - - expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled(); - }); - - // ── egress self-check ───────────────────────────────────────────────────── - - // A transport failure: nothing came back from the target, so the code is the network-error sentinel. - const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" }; - - it("does not consult the egress service for a successful check and leaves the field unset", async () => { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - }); - - await producer.produce(makeMonitor()); - - expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled(); - expect(check).not.toHaveProperty("egressStatus"); - }); - - it("does not consult egress when an HTTP response was received", async () => { - for (const status of [ - { monitorId: "m1", status: false, code: 500, message: "Internal Server Error" }, - { monitorId: "m1", status: false, code: 200, message: "Content mismatch" }, - ]) { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(makeMonitor()); - - expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled(); - expect(check).not.toHaveProperty("egressStatus"); - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); - } - }); - - it("consults egress for a timeout or network error", async () => { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(makeMonitor()); - - expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); - expect(check.egressStatus).toBe("degraded"); - }); - - it("never consults egress for a docker daemon on a local socket", async () => { - for (const url of ["unix:///var/run/docker.sock", "/var/run/docker.sock"]) { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(makeMonitor({ type: "docker", url })); - - // Local IPC never leaves the instance, so it cannot fail through egress. - expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled(); - expect(check).not.toHaveProperty("egressStatus"); - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); - } - }); - - it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => { - for (const monitor of [ - makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }), - makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }), - ]) { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(monitor); - - // Both requests leave the instance, so our own loss of egress can be the cause. - expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); - expect(check.egressStatus).toBe("degraded"); - } - }); - - it("does not consult egress when the provider reports that the peer answered", async () => { - // gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates - // them from a connection failure. - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(makeMonitor({ type: "dns" })); - - expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled(); - expect(check).not.toHaveProperty("egressStatus"); - }); - - it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(makeMonitor({ type: "grpc" })); - - // An explicit report from the provider outranks the HTTP-status heuristic. - expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); - expect(check.egressStatus).toBe("degraded"); - }); - - it("flags a failing check as degraded when the egress service reports degraded egress", async () => { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - const result = await producer.produce(makeMonitor()); - - expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); - expect(check.egressStatus).toBe("degraded"); - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" })); - expect(result?.check.egressStatus).toBe("degraded"); - }); - - // ── status window across a spell ───────────────────────────────────────── - // Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either - // side of a spell are never adjacent. - - it("clears the status window on the first degraded check of a spell", async () => { - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] })); - - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] }); - }); - - it("does not re-write the status window for later degraded checks in the same spell", async () => { - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, - }); - - await producer.produce(makeMonitor({ statusWindow: [] })); - - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("leaves the status window alone when the probe found egress fine", async () => { - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") }, - }); - - await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] })); - - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("flags a failing check as ok when the probe found egress fine", async () => { - const check: Record = { id: "check-1" }; - const { producer } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") }, - }); - - await producer.produce(makeMonitor()); - - expect(check.egressStatus).toBe("ok"); - }); - - it("buffers the check when the egress assessment rejects", async () => { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) }, - }); - - const result = await producer.produce(makeMonitor()); - - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); - expect(check).not.toHaveProperty("egressStatus"); - expect(result).toEqual({ status: failingStatus, check }); - expect(defaults.logger.warn).toHaveBeenCalledWith( - expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" }) - ); - }); - - it("leaves the field unset on a failing check when the egress check is disabled", async () => { - const check: Record = { id: "check-1" }; - const { producer, defaults } = createProducer({ - networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, - checkService: { toCheck: jest.fn().mockReturnValue(check) }, - }); - - await producer.produce(makeMonitor()); - - expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); - expect(check).not.toHaveProperty("egressStatus"); - }); -}); diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index b8bf85688..7c9d368f2 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -9,10 +9,11 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; // ── Notes ────────────────────────────────────────────────────────────────────── // // DBQueueWorker is the orchestrator: it drives the polling loops, claims batches -// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and -// renews their locks, and registers itself in the worker heartbeat. The repository -// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we -// test only the orchestration around it, with the repository fully mocked. +// sized to free capacity, hands each job to the handler registered for its type, +// renews locks while a job runs, and registers itself in the worker heartbeat. What a +// handler does is covered in workerPipeline.test.ts and geoChecksPipeline.test.ts; the +// repository claim/lock atomicity in integration/jobsRepository.test.ts. Here we test +// only the orchestration, with the repository and every handler mocked. // // Fake timers are mandatory: init() starts self-re-arming setTimeout loops and a // heartbeat setInterval, so without them the loops would leak across tests. The @@ -73,6 +74,17 @@ const makeEvaluateJob = (...checkIds: string[]): Job => pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })), }); +// One resolved mock per job type; a test overrides the one it drives +const makeHandlers = (overrides?: Record) => ({ + check: jest.fn().mockResolvedValue(undefined), + evaluate: jest.fn().mockResolvedValue(undefined), + "geo-check": jest.fn().mockResolvedValue(undefined), + "cleanup-orphaned": jest.fn().mockResolvedValue(undefined), + "cleanup-retention": jest.fn().mockResolvedValue(undefined), + egress: jest.fn().mockResolvedValue(undefined), + ...overrides, +}); + const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record }) => { const jobsRepository = { claimDueBatch: jest.fn().mockResolvedValue([]), @@ -98,12 +110,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses findAllForScheduling: jest.fn().mockResolvedValue([]), updateById: jest.fn().mockResolvedValue({}), }; - const checksRepository = { - findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([]), - }; - const checkService = { - toStatusResponse: jest.fn().mockReturnValue({ status: "up" }), - }; const bufferService = { addToBuffer: jest.fn(), addGeoCheckToBuffer: jest.fn(), @@ -112,19 +118,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses flushGeoBuffer: jest.fn().mockResolvedValue(undefined), shutdown: jest.fn().mockResolvedValue(undefined), }; - const checkProducer = { - produce: jest.fn().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }), - }; - const checkEvaluator = { - evaluate: jest.fn().mockResolvedValue({ monitor: makeMonitor(), statusChange: { monitor: makeMonitor() }, decision: {} }), - }; - const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) }; - const dispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) }; - const helper = { - getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - getEgressRecoveryJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - }; + const handlers = makeHandlers(); const queueWorkersRepository = { upsert: jest.fn().mockResolvedValue(undefined), deleteById: jest.fn().mockResolvedValue(undefined), @@ -137,14 +131,8 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses isDbConnected, jobsRepository, monitorsRepository, - checksRepository, - checkService, bufferService, - checkProducer, - checkEvaluator, - geoCheckPipeline, - dispatcher, - helper, + handlers, queueWorkersRepository, logger, ...overrides?.mocks, @@ -282,19 +270,18 @@ describe("DBQueueWorker", () => { const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 }); const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); - expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1"); - expect(mocks.checkProducer.produce).toHaveBeenCalled(); - // The check stage only produces; the buffer arms evaluate once the check is durably stored - expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); + expect(mocks.handlers.check).toHaveBeenCalledWith(job); expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled(); }); it("records a failure (and not a success) when the job handler throws", async () => { const job = makeJob({ type: "check" }); - const failingProducer = { produce: jest.fn().mockRejectedValue(new Error("network down")) }; const { mocks } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: failingProducer }, + mocks: { + jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, + handlers: makeHandlers({ check: jest.fn().mockRejectedValue(new Error("network down")) }), + }, }); expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number)); @@ -310,7 +297,6 @@ describe("DBQueueWorker", () => { process.on("unhandledRejection", onUnhandled); try { const job = makeJob({ type: "check" }); - const failingProducer = { produce: jest.fn().mockRejectedValue(new Error("network down")) }; const { mocks, worker } = await start({ mocks: { jobsRepository: { @@ -318,7 +304,7 @@ describe("DBQueueWorker", () => { claimDueBatch: claimOnce(job), recordFailure: jest.fn().mockRejectedValue(new Error("mongo unavailable")), }, - checkProducer: failingProducer, + handlers: makeHandlers({ check: jest.fn().mockRejectedValue(new Error("network down")) }), }, }); await jest.advanceTimersByTimeAsync(1); // let the recordFailure rejection settle @@ -336,7 +322,7 @@ describe("DBQueueWorker", () => { }); it("swallows the failure when both the success and failure writes reject", async () => { - const job = makeJob({ type: "check", intervalMs: 60000 }); // produce succeeds → recordSuccess path + const job = makeJob({ type: "check", intervalMs: 60000 }); // handler succeeds → recordSuccess path const { mocks, worker } = await start({ mocks: { jobsRepository: { @@ -367,7 +353,7 @@ describe("DBQueueWorker", () => { claimDueBatch: claimOnce(job), recordFailure: jest.fn().mockRejectedValue(new Error("mongo unavailable")), }, - checkProducer: { produce: jest.fn().mockRejectedValue(new Error("network down")) }, + handlers: makeHandlers({ check: jest.fn().mockRejectedValue(new Error("network down")) }), }, }); await jest.advanceTimersByTimeAsync(1); @@ -386,173 +372,66 @@ describe("DBQueueWorker", () => { expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled(); }); - it("evaluate job loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => { - const job = makeEvaluateJob("c1"); - const checksRepository = { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }]) }; - const { mocks } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository }, + it("routes every job type to the handler registered for it, with the claimed job", async () => { + const jobs: Job[] = [ + makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null, pendingChecks: [{ checkId: "c1", createdAt: 1000 }] }), + makeJob({ id: "geo-check:m1", type: "geo-check", intervalMs: 300000 }), + makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 }), + makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 }), + makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 }), + ]; + const handed = new Set(); + const claimDueBatch = jest.fn(async (type: string) => { + const job = jobs.find((j) => j.type === type); + if (!job || handed.has(type)) return []; + handed.add(type); + return [job]; }); + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch } } }); - expect(checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks); - expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled(); - expect(mocks.dispatcher.dispatch).toHaveBeenCalled(); - expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]); - // The monitor cursor is gone: nothing writes lastEvaluatedAt - expect(mocks.monitorsRepository.updateById).not.toHaveBeenCalled(); + for (const job of jobs) { + expect(mocks.handlers[job.type]).toHaveBeenCalledWith(job); + } + expect(mocks.handlers.check).not.toHaveBeenCalled(); }); - it("evaluate job with no pending checks does nothing", async () => { - const job = makeEvaluateJob(); - const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); - - expect(mocks.monitorsRepository.findByIdLean).not.toHaveBeenCalled(); - expect(mocks.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled(); - expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number)); - }); - - it("evaluate job for a deleted monitor drops every pending id", async () => { - const job = makeEvaluateJob("c1", "c2"); - const monitorsRepository = { findByIdLean: jest.fn().mockResolvedValue(null), updateById: jest.fn() }; - const { mocks } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, monitorsRepository }, - }); - - expect(mocks.checkEvaluator.evaluate).not.toHaveBeenCalled(); - expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]); - }); - - it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => { - const job = makeEvaluateJob("c1", "c2", "c3"); - const checksRepository = { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) }; - const jobsRepository = { - ...createWorker().mocks.jobsRepository, - claimDueBatch: claimOnce(job), - pullEvaluated: jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2 - }; - const { mocks } = await start({ mocks: { jobsRepository, checksRepository } }); - - expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2); - expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2); - expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either - expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" })); - }); - - it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => { - const job = makeEvaluateJob("c1", "c-gone"); - const checksRepository = { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up - const { mocks } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository }, - }); - - expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(1); - expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]); - expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]); - }); - - it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => { - const job = makeEvaluateJob("c1", "c2"); - const initialMonitor = makeMonitor(); - const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1 - const monitorsRepository = { - findByIdLean: jest.fn().mockResolvedValue(initialMonitor), - updateById: jest.fn().mockResolvedValue({}), - }; - const checksRepository = { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) }; - const checkEvaluator = { - evaluate: jest.fn().mockResolvedValue({ monitor: postWriteMonitor, statusChange: { monitor: postWriteMonitor }, decision: {} }), - }; - const { mocks } = await start({ - mocks: { - jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, - monitorsRepository, - checksRepository, - checkEvaluator, - }, - }); - - // The monitor is read once for the whole backlog, not once per check - expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1); - expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2); - // First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first - expect(mocks.checkEvaluator.evaluate.mock.calls[0][2]).toBe(initialMonitor); - expect(mocks.checkEvaluator.evaluate.mock.calls[1][2]).toBe(postWriteMonitor); - }); - - it("cleanup-orphaned job runs the helper's cleanup function", async () => { - const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 }); - const cleanupFn = jest.fn().mockResolvedValue(undefined); - const helper = { - getCleanupOrphanedJob: jest.fn().mockReturnValue(cleanupFn), - getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - }; - await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } }); - - expect(cleanupFn).toHaveBeenCalled(); - }); - - it("cleanup-retention job runs the helper's retention function", async () => { - const job = makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 }); - const retentionFn = jest.fn().mockResolvedValue(undefined); - const helper = { - getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - getCleanupRetentionJob: jest.fn().mockReturnValue(retentionFn), - }; - await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } }); - - expect(retentionFn).toHaveBeenCalled(); - }); - - it("egress job runs the helper's egress recovery function with the claimed job", async () => { + it("reschedules an interval job after its handler succeeds", async () => { const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 }); - const egressFn = jest.fn().mockResolvedValue(undefined); - const helper = { - getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - getEgressRecoveryJob: jest.fn().mockReturnValue(egressFn), - }; - const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } }); - - expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" })); - expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); - }); - - it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => { - const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 }); const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); - expect(mocks.geoCheckPipeline.run).toHaveBeenCalled(); - expect(mocks.dispatcher.dispatch).not.toHaveBeenCalled(); + expect(mocks.handlers.egress).toHaveBeenCalledWith(job); expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); }); it("renews the lock while a slow job is still running", async () => { const job = makeJob({ type: "check" }); - const gate = deferred<{ status: unknown; check: unknown }>(); - const slowProducer = { produce: jest.fn().mockReturnValue(gate.promise) }; + const gate = deferred(); const { mocks } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer }, + mocks: { + jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, + handlers: makeHandlers({ check: jest.fn().mockReturnValue(gate.promise) }), + }, }); - // Job is parked on produce(); push past one renewal interval. + // Job is parked in its handler; push past one renewal interval. await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS); expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledWith([job.id], expect.any(Number)); // Let it finish; success is still recorded. - gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); + gate.resolve(); await jest.advanceTimersByTimeAsync(1); expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled(); }); it("stops renewing once the lock is lost to another worker", async () => { const job = makeJob({ type: "check" }); - const gate = deferred<{ status: unknown; check: unknown }>(); - const slowProducer = { produce: jest.fn().mockReturnValue(gate.promise) }; + const gate = deferred(); const jobsRepository = { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job), renewLocks: jest.fn().mockResolvedValue(0), }; - const { mocks } = await start({ mocks: { jobsRepository, checkProducer: slowProducer } }); + const { mocks } = await start({ mocks: { jobsRepository, handlers: makeHandlers({ check: jest.fn().mockReturnValue(gate.promise) }) } }); await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS); expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); // lease lost → renewal timer stops @@ -562,7 +441,7 @@ describe("DBQueueWorker", () => { await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS); expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); - gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); + gate.resolve(); await jest.advanceTimersByTimeAsync(1); }); }); @@ -683,11 +562,8 @@ describe("DBQueueWorker", () => { } return []; }); - const helper = { - getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn(() => gate.promise)), // blocks → keeps the slot full - getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), - }; - const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, helper } }); + const handlers = makeHandlers({ "cleanup-orphaned": jest.fn(() => gate.promise) }); // blocks → keeps the slot full + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, handlers } }); const before = claims(mocks, "cleanup-orphaned"); // the blocking job is now in-flight → capacity 0 await jest.advanceTimersByTimeAsync(1000); @@ -804,19 +680,21 @@ describe("DBQueueWorker", () => { it("waits for an in-flight job to finish before flushing", async () => { const job = makeJob({ type: "check" }); - const gate = deferred<{ status: unknown; check: unknown }>(); - const slowProducer = { produce: jest.fn().mockReturnValue(gate.promise) }; + const gate = deferred(); const { worker, mocks } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer }, + mocks: { + jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, + handlers: makeHandlers({ check: jest.fn().mockReturnValue(gate.promise) }), + }, }); - // Job is parked on produce(); drain must not flush while it is still in flight. + // Job is parked in its handler; drain must not flush while it is still in flight. const draining = worker.drain(); await jest.advanceTimersByTimeAsync(DRAIN_POLL_MS * 3); expect(mocks.bufferService.shutdown).not.toHaveBeenCalled(); // Let the job finish; drain observes inFlight == 0, then flushes. - gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); + gate.resolve(); await jest.advanceTimersByTimeAsync(DRAIN_POLL_MS); await draining; expect(mocks.bufferService.shutdown).toHaveBeenCalledTimes(1); @@ -824,10 +702,12 @@ describe("DBQueueWorker", () => { it("times out after DRAIN_TIMEOUT_MS, warns, and still flushes", async () => { const job = makeJob({ type: "check" }); - const gate = deferred<{ status: unknown; check: unknown }>(); - const stuckProducer = { produce: jest.fn().mockReturnValue(gate.promise) }; + const gate = deferred(); const { worker, mocks } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: stuckProducer }, + mocks: { + jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, + handlers: makeHandlers({ check: jest.fn().mockReturnValue(gate.promise) }), + }, }); const draining = worker.drain(); @@ -837,7 +717,7 @@ describe("DBQueueWorker", () => { expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("timed out") })); expect(mocks.bufferService.shutdown).toHaveBeenCalledTimes(1); - gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); // release the stuck job for cleanup + gate.resolve(); // release the stuck job for cleanup }); }); @@ -892,15 +772,17 @@ describe("DBQueueWorker", () => { it("counts in-flight jobs while they run and releases on completion", async () => { const job = makeJob({ type: "check" }); - const gate = deferred<{ status: unknown; check: unknown }>(); - const slowProducer = { produce: jest.fn().mockReturnValue(gate.promise) }; + const gate = deferred(); const { worker } = await start({ - mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOneCheck(job) }, checkProducer: slowProducer }, + mocks: { + jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOneCheck(job) }, + handlers: makeHandlers({ check: jest.fn().mockReturnValue(gate.promise) }), + }, }); - expect(worker.getHealth().inFlight).toBe(1); // parked on produce() + expect(worker.getHealth().inFlight).toBe(1); // parked in its handler - gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); + gate.resolve(); await jest.advanceTimersByTimeAsync(1); expect(worker.getHealth().inFlight).toBe(0); }); diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/geoChecksPipeline.test.ts similarity index 72% rename from server/test/unit/services/checkPipeline.test.ts rename to server/test/unit/services/geoChecksPipeline.test.ts index 2e7026f4d..867bc2f0a 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/geoChecksPipeline.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, jest } from "@jest/globals"; -import { GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts"; +import { GeoChecksPipeline } from "../../../src/worker/worker.geo-pipeline.ts"; +import type { Job } from "../../../src/domain/jobs/job.type.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; @@ -25,12 +26,14 @@ const activeWindow = () => { const createGeoPipeline = (overrides?: Record) => { const defaults = { logger: createMockLogger(), + monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(null) }, geoChecksService: { buildGeoCheck: jest.fn().mockResolvedValue(null) }, buffer: { addGeoCheckToBuffer: jest.fn() }, maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, ...overrides, }; const pipeline = new GeoChecksPipeline( + defaults.monitorsRepository as any, defaults.maintenanceWindowsRepository as any, defaults.geoChecksService as any, defaults.buffer as any, @@ -42,7 +45,56 @@ const createGeoPipeline = (overrides?: Record) => { const geoMonitor = (overrides?: Partial) => makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckLocations: ["us-east"], ...overrides }); +const makeJob = (overrides?: Partial): Job => ({ + id: "geo-check:m1", + type: "geo-check", + refId: "m1", + isActive: true, + nextScheduledAt: 1000, + intervalMs: 300000, + lockedBy: null, + lockedUntil: null, + pendingChecks: [], + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + ...overrides, +}); + describe("GeoChecksPipeline", () => { + describe("handle", () => { + it("loads the monitor by refId and runs the pipeline for it", async () => { + const geoCheck = { id: "gc-1", monitorId: "m1" }; + const { pipeline, defaults } = createGeoPipeline({ + monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(geoMonitor()) }, + geoChecksService: { buildGeoCheck: jest.fn().mockResolvedValue(geoCheck) }, + }); + + await pipeline.handle(makeJob()); + + expect(defaults.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1"); + expect(defaults.buffer.addGeoCheckToBuffer).toHaveBeenCalledWith(geoCheck); + }); + + it("does nothing for a job without a refId", async () => { + const { pipeline, defaults } = createGeoPipeline(); + + await pipeline.handle(makeJob({ refId: null })); + + expect(defaults.monitorsRepository.findByIdLean).not.toHaveBeenCalled(); + expect(defaults.geoChecksService.buildGeoCheck).not.toHaveBeenCalled(); + }); + + it("does nothing when the monitor no longer exists", async () => { + const { pipeline, defaults } = createGeoPipeline(); + + await pipeline.handle(makeJob()); + + expect(defaults.geoChecksService.buildGeoCheck).not.toHaveBeenCalled(); + }); + }); + it("throws when monitor id is missing", async () => { const { pipeline } = createGeoPipeline(); await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id"); diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index af73623df..73a422cf7 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -7,7 +7,7 @@ import type { IUsersRepository } from "../../../src/domain/users/user.repository import type { INotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import type { Incident } from "../../../src/domain/incidents/incident.type.ts"; -import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts"; +import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── diff --git a/server/test/unit/services/monitorStatusPolicy.test.ts b/server/test/unit/services/monitorStatusPolicy.test.ts deleted file mode 100644 index 7b340a3e9..000000000 --- a/server/test/unit/services/monitorStatusPolicy.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { describe, expect, it } from "@jest/globals"; -import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; -import type { StatusChangeResult } from "../../../src/types/network.ts"; - -// ── Helpers ────────────────────────────────────────────────────────────────── - -const makeStatusChange = (overrides: Partial & { status?: string }): StatusChangeResult => { - const { status = "up", ...rest } = overrides; - return { - monitor: { id: "m1", status } as any, - statusChanged: false, - prevStatus: "up", - code: 200, - timestamp: 0, - ...rest, - } as StatusChangeResult; -}; - -// ── Tests ──────────────────────────────────────────────────────────────────── - -describe("MonitorStatusPolicy", () => { - const policy = new MonitorStatusPolicy(); - - it("does nothing when statusChanged is false", () => { - const decision = policy.evaluate(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "up" })); - expect(decision).toEqual({ - shouldCreateIncident: false, - shouldResolveIncident: false, - shouldSendNotification: false, - incidentReason: null, - notificationReason: null, - }); - }); - - it("creates incident and notifies when monitor goes down", () => { - const decision = policy.evaluate(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 })); - expect(decision).toMatchObject({ - shouldCreateIncident: true, - shouldSendNotification: true, - incidentReason: "status_down", - notificationReason: "status_change", - }); - }); - - it("creates incident and notifies when monitor is breached", () => { - const decision = policy.evaluate(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" })); - expect(decision).toMatchObject({ - shouldCreateIncident: true, - shouldSendNotification: true, - incidentReason: "threshold_breach", - notificationReason: "threshold_breach", - }); - }); - - it("resolves incident when monitor recovers from down", () => { - const decision = policy.evaluate(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "down" })); - expect(decision).toMatchObject({ - shouldResolveIncident: true, - shouldSendNotification: true, - shouldCreateIncident: false, - notificationReason: "status_change", - }); - }); - - it("resolves incident when monitor recovers from breached", () => { - const decision = policy.evaluate(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" })); - expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); - }); - - it("does not create or resolve for unhandled status transitions", () => { - const decision = policy.evaluate(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" })); - expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false }); - }); -}); diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index 862697602..b0b8397e3 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, beforeEach } from "@jest/globals"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; -import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts"; +import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── diff --git a/server/test/unit/services/notificationsService.test.ts b/server/test/unit/services/notificationsService.test.ts index d0fdd61b2..adfdb6c90 100644 --- a/server/test/unit/services/notificationsService.test.ts +++ b/server/test/unit/services/notificationsService.test.ts @@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import type { Notification } from "../../../src/domain/notifications/notification.type.ts"; import type { MonitorStatusResponse } from "../../../src/types/network.ts"; -import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts"; +import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts new file mode 100644 index 000000000..e7c117152 --- /dev/null +++ b/server/test/unit/services/workerPipeline.test.ts @@ -0,0 +1,874 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { WorkerPipeline } from "../../../src/worker/worker.pipeline.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; +import type { Job } from "../../../src/domain/jobs/job.type.ts"; +import type { Check } from "../../../src/domain/checks/check.type.ts"; +import type { StatusChangeResult } from "../../../src/types/network.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; +import { NETWORK_ERROR } from "../../../src/types/network.ts"; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "m1", + teamId: "team", + type: "http", + interval: 60000, + status: "up", + ...overrides, + }) as Monitor; + +const makeJob = (overrides?: Partial): Job => ({ + id: "check:m1", + type: "check", + refId: "m1", + isActive: true, + nextScheduledAt: 1000, + intervalMs: 60000, + lockedBy: null, + lockedUntil: null, + pendingChecks: [], + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + ...overrides, +}); + +// An evaluate row as ingest leaves it: one pending entry per stored check +const makeEvaluateJob = (...checkIds: string[]): Job => + makeJob({ + id: "evaluate:m1", + type: "evaluate", + intervalMs: null, + pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })), + }); + +const makeCheck = (overrides?: Partial): Check => + ({ + id: "check-1", + metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" }, + status: true, + statusCode: 200, + responseTime: 100, + message: "OK", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + ...overrides, + }) as Check; + +const makeStatusChange = (overrides: Partial & { status?: string }): StatusChangeResult => { + const { status = "up", ...rest } = overrides; + return { + monitor: { id: "m1", status } as any, + statusChanged: false, + prevStatus: "up", + code: 200, + timestamp: 0, + ...rest, + } as StatusChangeResult; +}; + +// A maintenance window straddling "now" — active per isWindowActive. +const activeWindow = () => { + const now = Date.now(); + return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; +}; + +const createPipeline = (overrides?: Record) => { + const deps = { + logger: createMockLogger(), + monitorsRepository: { + findByIdLean: jest.fn().mockResolvedValue(makeMonitor()), + updateById: jest.fn().mockResolvedValue({}), + findDockerTlsKeyById: jest.fn().mockResolvedValue(null), + }, + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([]) }, + jobsRepository: { + upsertEvaluate: jest.fn().mockResolvedValue(true), + pullEvaluated: jest.fn().mockResolvedValue(true), + }, + checkService: { + toCheck: jest.fn().mockReturnValue({ id: "check-1" }), + toStatusResponse: jest.fn().mockReturnValue({ monitorId: "m1", status: true, code: 200, message: "OK" }), + createChecks: jest.fn().mockResolvedValue([]), + }, + networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, + proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) }, + bufferService: { addToBuffer: jest.fn(), addDockerLogToBuffer: jest.fn() }, + dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) }, + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(makeStatusChange({})) }, + dispatcher: { dispatch: jest.fn().mockResolvedValue(undefined) }, + ...overrides, + }; + const pipeline = new WorkerPipeline(deps as any); + return { pipeline, deps }; +}; + +// ── Stage 1: check job ─────────────────────────────────────────────────────── + +describe("WorkerPipeline", () => { + describe("handleCheck", () => { + it("loads the monitor by refId and produces a check for it", async () => { + const { pipeline, deps } = createPipeline(); + + await pipeline.handleCheck(makeJob()); + + expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1"); + expect(deps.networkService.requestStatus).toHaveBeenCalled(); + expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith({ id: "check-1" }); + }); + + it("does nothing for a job without a refId", async () => { + const { pipeline, deps } = createPipeline(); + + await pipeline.handleCheck(makeJob({ refId: null })); + + expect(deps.monitorsRepository.findByIdLean).not.toHaveBeenCalled(); + expect(deps.networkService.requestStatus).not.toHaveBeenCalled(); + }); + + it("does nothing when the monitor no longer exists", async () => { + const { pipeline, deps } = createPipeline({ + monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(null) }, + }); + + await pipeline.handleCheck(makeJob()); + + expect(deps.networkService.requestStatus).not.toHaveBeenCalled(); + }); + }); + + describe("produce", () => { + it("throws when monitor id is missing", async () => { + const { pipeline } = createPipeline(); + await expect(pipeline.produce({} as Monitor)).rejects.toThrow("No monitor id"); + }); + + // ── maintenance gate ────────────────────────────────────────────────── + + it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => { + const { pipeline, deps } = createPipeline({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, + }); + + const result = await pipeline.produce(makeMonitor({ status: "up" })); + + expect(result).toBeNull(); + expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] }); + expect(deps.networkService.requestStatus).not.toHaveBeenCalled(); + expect(deps.bufferService.addToBuffer).not.toHaveBeenCalled(); + }); + + it("does not re-write status when already in maintenance", async () => { + const { pipeline, deps } = createPipeline({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, + }); + + const result = await pipeline.produce(makeMonitor({ status: "maintenance" })); + + expect(result).toBeNull(); + expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("moves a monitor out of maintenance explicitly once the window is over", async () => { + const monitor = makeMonitor({ status: "maintenance" }); + const { pipeline, deps } = createPipeline({ + monitorsRepository: { + updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }), + findDockerTlsKeyById: jest.fn().mockResolvedValue(null), + }, + }); + + await pipeline.produce(monitor); + + // The exit used to be a side effect of evaluating the resulting check, which the degraded-egress + // short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage. + expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" }); + expect(deps.networkService.requestStatus).toHaveBeenCalled(); + }); + + it("does not touch the status of a monitor that was not in maintenance", async () => { + const { pipeline, deps } = createPipeline(); + + await pipeline.produce(makeMonitor({ status: "up" })); + + expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("proceeds with the check when the maintenance window is inactive", async () => { + const { pipeline, deps } = createPipeline({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, + }); + + await pipeline.produce(makeMonitor()); + + expect(deps.networkService.requestStatus).toHaveBeenCalled(); + }); + + // ── proxy resolution ────────────────────────────────────────────────── + + it("passes the resolved proxy url into requestStatus", async () => { + const { pipeline, deps } = createPipeline({ + proxyResolver: { resolve: jest.fn().mockResolvedValue("http://proxy.example.com:8080") }, + }); + const monitor = makeMonitor(); + + await pipeline.produce(monitor); + + expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: "http://proxy.example.com:8080" }); + }); + + it("still produces a check when the resolver returns undefined", async () => { + const { pipeline, deps } = createPipeline(); + const monitor = makeMonitor(); + + const result = await pipeline.produce(monitor); + + expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined }); + expect(result).toEqual({ status: expect.objectContaining({ monitorId: "m1" }), check: { id: "check-1" } }); + }); + + // ── docker tls key resolution ───────────────────────────────────────── + + it("fetches the stored key for a docker monitor with a key set and passes it in the context", async () => { + const { pipeline, deps } = createPipeline({ + monitorsRepository: { updateById: jest.fn(), findDockerTlsKeyById: jest.fn().mockResolvedValue("v1.abc123.iv.tag.data") }, + }); + const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true }); + + await pipeline.produce(monitor); + + expect(deps.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1"); + expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: "v1.abc123.iv.tag.data" }); + }); + + it("does not fetch a key for a docker monitor without one set", async () => { + const { pipeline, deps } = createPipeline(); + const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKeySet: false }); + + await pipeline.produce(monitor); + + expect(deps.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled(); + expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined }); + }); + + it("does not fetch a key for non-docker monitors even if the flag is set", async () => { + const { pipeline, deps } = createPipeline(); + + await pipeline.produce(makeMonitor({ type: "http", dockerTlsKeySet: true })); + + expect(deps.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled(); + }); + + it("passes undefined when the flag is set but the repository has no key", async () => { + const { pipeline, deps } = createPipeline(); + const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true }); + + await pipeline.produce(monitor); + + expect(deps.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1"); + expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined }); + }); + + // ── acquire / record ────────────────────────────────────────────────── + + it("throws when the network response is null", async () => { + const { pipeline } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, + }); + + await expect(pipeline.produce(makeMonitor())).rejects.toThrow("No network response"); + }); + + it("returns null, warns, and does not buffer when toCheck yields nothing", async () => { + const { pipeline, deps } = createPipeline({ + checkService: { toCheck: jest.fn().mockReturnValue(null) }, + }); + + const result = await pipeline.produce(makeMonitor()); + + expect(result).toBeNull(); + expect(deps.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); + expect(deps.bufferService.addToBuffer).not.toHaveBeenCalled(); + }); + + it("buffers the built check and returns the status and check on success", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; + const check = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + }); + + const result = await pipeline.produce(makeMonitor()); + + expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check); + expect(result).toEqual({ status, check }); + }); + + it("builds and buffers docker logs for a docker status", async () => { + const status = { type: "docker", monitorId: "m1", teamId: "team", status: true, code: 200, message: "OK", payload: {} }; + const dockerLogs = [{ id: "docker-log-1" }, { id: "docker-log-2" }]; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, + dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue(dockerLogs) }, + }); + + await pipeline.produce(makeMonitor({ type: "docker" })); + + expect(deps.dockerLogsService.buildDockerLogs).toHaveBeenCalledWith(status); + expect(deps.bufferService.addDockerLogToBuffer).toHaveBeenNthCalledWith(1, dockerLogs[0]); + expect(deps.bufferService.addDockerLogToBuffer).toHaveBeenNthCalledWith(2, dockerLogs[1]); + }); + + it("does not build docker logs for a non-docker status", async () => { + const { pipeline, deps } = createPipeline(); + + await pipeline.produce(makeMonitor()); + + expect(deps.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled(); + }); + + // ── egress self-check ───────────────────────────────────────────────── + + // A transport failure: nothing came back from the target, so the code is the network-error sentinel. + const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" }; + + it("does not consult the egress service for a successful check and leaves the field unset", async () => { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + }); + + await pipeline.produce(makeMonitor()); + + expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled(); + expect(check).not.toHaveProperty("egressStatus"); + }); + + it("does not consult egress when an HTTP response was received", async () => { + for (const status of [ + { monitorId: "m1", status: false, code: 500, message: "Internal Server Error" }, + { monitorId: "m1", status: false, code: 200, message: "Content mismatch" }, + ]) { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(makeMonitor()); + + expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled(); + expect(check).not.toHaveProperty("egressStatus"); + expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check); + } + }); + + it("consults egress for a timeout or network error", async () => { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(makeMonitor()); + + expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check.egressStatus).toBe("degraded"); + }); + + it("never consults egress for a docker daemon on a local socket", async () => { + for (const url of ["unix:///var/run/docker.sock", "/var/run/docker.sock"]) { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(makeMonitor({ type: "docker", url })); + + // Local IPC never leaves the instance, so it cannot fail through egress. + expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled(); + expect(check).not.toHaveProperty("egressStatus"); + expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check); + } + }); + + it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => { + for (const monitor of [ + makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }), + makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }), + ]) { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(monitor); + + // Both requests leave the instance, so our own loss of egress can be the cause. + expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check.egressStatus).toBe("degraded"); + } + }); + + it("does not consult egress when the provider reports that the peer answered", async () => { + // gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates + // them from a connection failure. + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(makeMonitor({ type: "dns" })); + + expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled(); + expect(check).not.toHaveProperty("egressStatus"); + }); + + it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(makeMonitor({ type: "grpc" })); + + // An explicit report from the provider outranks the HTTP-status heuristic. + expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check.egressStatus).toBe("degraded"); + }); + + it("flags a failing check as degraded when the egress service reports degraded egress", async () => { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + const result = await pipeline.produce(makeMonitor()); + + expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check.egressStatus).toBe("degraded"); + expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" })); + expect(result?.check.egressStatus).toBe("degraded"); + }); + + // ── status window across a spell ───────────────────────────────────── + // Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either + // side of a spell are never adjacent. + + it("clears the status window on the first degraded check of a spell", async () => { + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(makeMonitor({ statusWindow: [true, true, true, false, false] })); + + expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] }); + }); + + it("does not re-write the status window for later degraded checks in the same spell", async () => { + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await pipeline.produce(makeMonitor({ statusWindow: [] })); + + expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("leaves the status window alone when the probe found egress fine", async () => { + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") }, + }); + + await pipeline.produce(makeMonitor({ statusWindow: [true, true, true, false, false] })); + + expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("flags a failing check as ok when the probe found egress fine", async () => { + const check: Record = { id: "check-1" }; + const { pipeline } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") }, + }); + + await pipeline.produce(makeMonitor()); + + expect(check.egressStatus).toBe("ok"); + }); + + it("buffers the check when the egress assessment rejects", async () => { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) }, + }); + + const result = await pipeline.produce(makeMonitor()); + + expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check); + expect(check).not.toHaveProperty("egressStatus"); + expect(result).toEqual({ status: failingStatus, check }); + expect(deps.logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "WorkerPipeline", method: "assessEgress" }) + ); + }); + + it("leaves the field unset on a failing check when the egress check is disabled", async () => { + const check: Record = { id: "check-1" }; + const { pipeline, deps } = createPipeline({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + }); + + await pipeline.produce(makeMonitor()); + + expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check).not.toHaveProperty("egressStatus"); + }); + }); + + // ── Stage 2: ingestion ─────────────────────────────────────────────────── + + describe("ingestChecks", () => { + it("stores the batch, then arms the evaluate stage once per distinct monitor", async () => { + const { pipeline, deps } = createPipeline(); + const batch = [ + makeCheck({ id: "c1", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }), + makeCheck({ id: "c2", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }), + makeCheck({ id: "c3", metadata: { monitorId: "mon-2", teamId: "team-1", type: "http" } }), + ]; + + await pipeline.ingestChecks(batch); + + const createdAt = Date.parse("2026-01-01T00:00:00.000Z"); + expect(deps.checkService.createChecks).toHaveBeenCalledWith(batch); + expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2); + expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledWith( + "mon-1", + [ + { checkId: "c1", createdAt }, + { checkId: "c2", createdAt }, + ], + expect.any(Number) + ); + expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number)); + }); + + it("retries a failed arm on the next ingest without re-storing the checks", async () => { + const { pipeline, deps } = createPipeline(); + deps.jobsRepository.upsertEvaluate.mockRejectedValueOnce(new Error("arm failed")); + + await pipeline.ingestChecks([makeCheck({ id: "c1" })]); + expect(deps.checkService.createChecks).toHaveBeenCalledTimes(1); + expect(deps.logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" })); + + await pipeline.ingestChecks([]); // nothing new, only the retry runs + expect(deps.checkService.createChecks).toHaveBeenCalledTimes(1); + expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2); + expect(deps.jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith( + "mon-1", + [{ checkId: "c1", createdAt: expect.any(Number) }], + expect.any(Number) + ); + + await pipeline.ingestChecks([]); // nothing left to retry + expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2); + }); + + it("neither writes nor arms for an empty batch", async () => { + const { pipeline, deps } = createPipeline(); + + await pipeline.ingestChecks([]); + + expect(deps.checkService.createChecks).not.toHaveBeenCalled(); + expect(deps.jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); + }); + + it("does not arm the evaluate stage when the check write fails", async () => { + const { pipeline, deps } = createPipeline(); + deps.checkService.createChecks.mockRejectedValue(new Error("DB write failed")); + + await expect(pipeline.ingestChecks([makeCheck()])).rejects.toThrow("DB write failed"); + + expect(deps.jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); + }); + }); + + // ── Stage 3: evaluation ────────────────────────────────────────────────── + + describe("handleEvaluate", () => { + it("loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => { + const job = makeEvaluateJob("c1"); + const { pipeline, deps } = createPipeline({ + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }]) }, + }); + + await pipeline.handleEvaluate(job); + + expect(deps.checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks); + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(1); + expect(deps.dispatcher.dispatch).toHaveBeenCalledTimes(1); + expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]); + // The monitor cursor is gone: nothing writes lastEvaluatedAt + expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("does nothing when the row has no pending checks", async () => { + const { pipeline, deps } = createPipeline(); + + await pipeline.handleEvaluate(makeEvaluateJob()); + + expect(deps.monitorsRepository.findByIdLean).not.toHaveBeenCalled(); + expect(deps.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled(); + }); + + it("drops every pending id for a deleted monitor", async () => { + const job = makeEvaluateJob("c1", "c2"); + const { pipeline, deps } = createPipeline({ + monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(null) }, + }); + + await pipeline.handleEvaluate(job); + + expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled(); + expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]); + }); + + it("stops applying checks once the lease is lost so a second claimer does not double-apply", async () => { + const job = makeEvaluateJob("c1", "c2", "c3"); + const { pipeline, deps } = createPipeline({ + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) }, + jobsRepository: { + upsertEvaluate: jest.fn(), + pullEvaluated: jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2 + }, + }); + + await pipeline.handleEvaluate(job); + + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2); + expect(deps.dispatcher.dispatch).toHaveBeenCalledTimes(2); + expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either + expect(deps.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Lost lease") })); + }); + + it("pulls ids whose check no longer exists so they do not sit on the row forever", async () => { + const job = makeEvaluateJob("c1", "c-gone"); + const { pipeline, deps } = createPipeline({ + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }]) }, // c-gone was cleaned up + }); + + await pipeline.handleEvaluate(job); + + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(1); + expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]); + expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]); + }); + + it("reads the monitor once and threads the post-write monitor across the backlog", async () => { + const job = makeEvaluateJob("c1", "c2"); + const initialMonitor = makeMonitor(); + const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1 + const { pipeline, deps } = createPipeline({ + monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(initialMonitor), updateById: jest.fn() }, + checksRepository: { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) }, + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(makeStatusChange({ monitor: postWriteMonitor })) }, + }); + + await pipeline.handleEvaluate(job); + + // The monitor is read once for the whole backlog, not once per check + expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1); + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2); + // First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first + expect(deps.statusService.updateMonitorStatus.mock.calls[0][2]).toBe(initialMonitor); + expect(deps.statusService.updateMonitorStatus.mock.calls[1][2]).toBe(postWriteMonitor); + }); + }); + + describe("evaluateCheck", () => { + it("threads status, check, and monitor through to the status service", async () => { + const { pipeline, deps } = createPipeline(); + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const check = { id: "check-1" } as any; + const monitor = makeMonitor({ status: "up" }); + + await pipeline.evaluateCheck(status, check, monitor); + + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor); + }); + + it("returns an evaluation that opens an incident on a down transition", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; + const check = { id: "check-1" } as any; + const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 }); + const { pipeline } = createPipeline({ + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, + }); + + const result = await pipeline.evaluateCheck(status, check, makeMonitor()); + + expect(result).toMatchObject({ + monitor: statusChange.monitor, + status, + check, + statusChange, + decision: expect.objectContaining({ + shouldCreateIncident: true, + shouldSendNotification: true, + incidentReason: "status_down", + }), + }); + }); + + it("returns an evaluation that resolves an incident on recovery", async () => { + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down" }); + const { pipeline } = createPipeline({ + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, + }); + + const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor({ status: "down" })); + + expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); + }); + + it("produces a no-op decision when status did not change", async () => { + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const { pipeline } = createPipeline(); + + const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor()); + + expect(result.decision).toMatchObject({ + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + }); + }); + + // ── degraded egress ─────────────────────────────────────────────────── + + it("short-circuits a check flagged with degraded egress without touching monitor status", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; + const check = { id: "check-1", egressStatus: "degraded" } as any; + const monitor = makeMonitor({ status: "up" }); + const { pipeline, deps } = createPipeline(); + + const result = await pipeline.evaluateCheck(status, check, monitor); + + expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled(); + expect(result.monitor).toBe(monitor); + expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 }); + expect(result.statusChange.timestamp).toEqual(expect.any(Number)); + expect(result.decision).toEqual({ + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }); + expect(deps.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") })); + }); + + it("evaluates a check flagged with egress ok as normal", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; + const check = { id: "check-1", egressStatus: "ok" } as any; + const { pipeline, deps } = createPipeline(); + + await pipeline.evaluateCheck(status, check, makeMonitor()); + + expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything()); + }); + }); + + // The decision table, driven through evaluateCheck with the status service stubbed to return each transition. + + describe("decision", () => { + const decide = async (statusChange: StatusChangeResult) => { + const { pipeline } = createPipeline({ + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, + }); + const result = await pipeline.evaluateCheck( + { monitorId: "m1", status: true, code: 200, message: "OK" } as any, + { id: "check-1" } as any, + makeMonitor() + ); + return result.decision; + }; + + it("does nothing when statusChanged is false", async () => { + const decision = await decide(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "up" })); + expect(decision).toEqual({ + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }); + }); + + it("creates incident and notifies when monitor goes down", async () => { + const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 })); + expect(decision).toMatchObject({ + shouldCreateIncident: true, + shouldSendNotification: true, + incidentReason: "status_down", + notificationReason: "status_change", + }); + }); + + it("creates incident and notifies when monitor is breached", async () => { + const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" })); + expect(decision).toMatchObject({ + shouldCreateIncident: true, + shouldSendNotification: true, + incidentReason: "threshold_breach", + notificationReason: "threshold_breach", + }); + }); + + it("resolves incident when monitor recovers from down", async () => { + const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "down" })); + expect(decision).toMatchObject({ + shouldResolveIncident: true, + shouldSendNotification: true, + shouldCreateIncident: false, + notificationReason: "status_change", + }); + }); + + it("resolves incident when monitor recovers from breached", async () => { + const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" })); + expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); + }); + + it("does not create or resolve for unhandled status transitions", async () => { + const decision = await decide(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" })); + expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false }); + }); + }); +});