TRX/Checkmate
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3978 from bluewave-labs/chore/readability-refactor

Browse Source 
chore: refactor worker services into a pipeline for readability
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-09-21 09:38:56 -07:00
2 parents 199d68e8a2 461a5e35dd
commit d51fdbd772
26 files changed
+1525 -1505
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/config/services.worker.ts
+25 -28
View File
@@ -17,12 +17,9 @@ import { INetworkService, NetworkService } from "@/service/networkService.js";
import { IBufferService, BufferService } from "@/service/bufferService.js";
import { IStatusService, StatusService } from "@/service/statusService.js";
import { SecretsRotationService } from "@/service/encryption/secretsRotationService.js";
import { IQueueWorker } from "@/worker/worker.interface.js";
import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IQueueWorker, JobHandlers } from "@/worker/worker.interface.js";
import { WorkerHelper } from "@/worker/worker.helper.js";
import { CheckProducer } from "@/worker/worker.check-producer.js";
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
import { GeoChecksPipeline } from "@/worker/worker.geo-pipeline.js";
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
@@ -42,6 +39,7 @@ import { GrpcProvider } from "@/service/network/GrpcProvider.js";
import { WebSocketProvider } from "@/service/network/WebSocketProvider.js";
import { DNSProvider } from "@/service/network/DNSProvider.js";
import { AppError } from "@/utils/AppError.js";
import { WorkerPipeline } from "@/worker/worker.pipeline.js";
export interface WorkerServices {
worker: IQueueWorker;
networkService: INetworkService;
@@ -103,37 +101,33 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
]);
const proxyResolver = new ProxyResolver(proxiesRepository, settingsService, logger);
const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
const bufferService = new BufferService(logger, geoChecksService, dockerLogsService, settingsService, (checks): Promise<void> =>
pipeline.ingestChecks(checks)
);
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
const monitorStatusPolicy = new MonitorStatusPolicy();
const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger);
// ***********************
// Reactors and dispatcher
// Handles notifications and incidents
// ***********************
const notificationReactor = new NotificationReactor(notificationsService);
const incidentReactor = new IncidentReactor(incidentService);
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
// ***********************
// Check producer/evaluator
// Handles creating and evaluatiog checks
// ***********************
const checkProducer = new CheckProducer(
const pipeline = new WorkerPipeline({
logger,
monitorsRepository,
maintenanceWindowsRepository,
checksRepository,
jobsRepository,
checkService,
networkService,
proxyResolver,
bufferService,
dockerLogsService,
egressService,
logger
);
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy, logger);
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
statusService,
dispatcher: reactorDispatcher,
});
const geoCheckPipeline = new GeoChecksPipeline(monitorsRepository, maintenanceWindowsRepository, geoChecksService, bufferService, logger);
// ***********************
// Worker
@@ -154,19 +148,22 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
egressService
);
const handlers: JobHandlers = {
check: pipeline.handleCheck,
evaluate: pipeline.handleEvaluate,
"geo-check": geoCheckPipeline.handle,
"cleanup-orphaned": workerHelper.getCleanupOrphanedJob(),
"cleanup-retention": workerHelper.getCleanupRetentionJob(),
egress: workerHelper.getEgressRecoveryJob(),
};
const worker = await DBQueueWorker.create({
logger,
isDbConnected: () => mongoose.connection.readyState === 1,
jobsRepository,
monitorsRepository,
checksRepository,
checkService,
bufferService,
checkProducer,
checkEvaluator,
geoCheckPipeline,
dispatcher: reactorDispatcher,
helper: workerHelper,
handlers,
queueWorkersRepository,
queueMode: envSettings.queueMode,
queuePrimaryProcesses: envSettings.queuePrimaryProcesses,
server/src/domain/incidents/incident.service.ts
+1 -1
View File
@@ -8,7 +8,7 @@ import type { IMonitorsRepository } from "@/domain/monitors/monitor.repository.i
import type { IUsersRepository } from "@/domain/users/user.repository.interface.js";
import type { Incident, IncidentSummary } from "@/domain/incidents/incident.type.js";
import type { User } from "@/domain/users/user.type.js";
import type { MonitorActionDecision } from "@/worker/worker.helper.js";
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
import type { ILogger } from "@/utils/logger.js";
import { DateRange } from "@/types/query.js";
server/src/domain/notifications/notification.message-builder.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import type { MonitorActionDecision } from "@/worker/worker.helper.js";
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
import type {
NotificationMessage,
NotificationType,
server/src/domain/notifications/notification.service.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import type { NotificationMessage } from "@/domain/notifications/notification.ty
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
import { INotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { MonitorActionDecision } from "@/worker/worker.helper.js";
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
import { ILogger } from "@/utils/logger.js";
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
server/src/service/bufferService.ts
+6 -54
View File
@@ -3,18 +3,15 @@ import type { GeoCheck } from "@/domain/geo-checks/geo-check.type.js";
import type { IGeoChecksService } from "../domain/geo-checks/geo-check.service.js";
import type { ILogger } from "@/utils/logger.js";
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ICheckService } from "../domain/checks/check.service.js";
import { DockerLog } from "@/domain/docker/docker-log.type.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
const SERVICE_NAME = "BufferService";
export type CheckIngest = (checks: Check[]) => Promise<void>;
export interface IBufferService {
addToBuffer(check: Check): void;
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
addDockerLogToBuffer(dockerLog: DockerLog): void;
ingestChecks(checks: Check[]): Promise<void>;
scheduleNextFlush(): void;
flushBuffer(): Promise<void>;
flushGeoBuffer(): Promise<void>;
@@ -31,28 +28,20 @@ export class BufferService implements IBufferService {
private geoBuffer: GeoCheck[];
private dockerLogBuffer: DockerLog[];
private bufferTimer: NodeJS.Timeout | null = null;
private checksService: ICheckService;
private geoChecksService: IGeoChecksService;
private dockerLogsService: IDockerLogsService;
private jobsRepository: IJobsRepository;
// Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated.
private unarmed: Map<string, PendingCheck[]> = new Map();
constructor(
logger: ILogger,
checkService: ICheckService,
geoChecksService: IGeoChecksService,
dockerLogsService: IDockerLogsService,
settingsService: ISettingsService,
jobsRepository: IJobsRepository
private ingest: CheckIngest
) {
this.BUFFER_TIMEOUT = settingsService.getSettings().nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute
this.logger = logger;
this.checksService = checkService;
this.geoChecksService = geoChecksService;
this.dockerLogsService = dockerLogsService;
this.jobsRepository = jobsRepository;
this.SERVICE_NAME = SERVICE_NAME;
this.buffer = [];
this.geoBuffer = [];
@@ -104,38 +93,6 @@ export class BufferService implements IBufferService {
}
}
ingestChecks = async (checks: Check[]) => {
if (checks.length > 0) await this.checksService.createChecks(checks);
// Group by monitor, starting from any entries a previous flush failed to arm
const byMonitor = this.unarmed;
this.unarmed = new Map();
for (const check of checks) {
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
byMonitor.set(check.metadata.monitorId, pending);
}
if (byMonitor.size === 0) return;
const now = Date.now();
const ops: Promise<boolean>[] = [];
for (const [monitorId, pendingChecks] of byMonitor) {
ops.push(
this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
// The checks are already stored, so a failed arm is retried on the next flush rather than dropped
this.unarmed.set(monitorId, pendingChecks);
this.logger.error({
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`,
service: this.SERVICE_NAME,
method: "ingestChecks",
});
return false;
})
);
}
await Promise.all(ops);
};
scheduleNextFlush() {
if (this.bufferTimer) {
clearTimeout(this.bufferTimer);
@@ -160,19 +117,14 @@ export class BufferService implements IBufferService {
}
async flushBuffer() {
if (this.buffer.length === 0 && this.unarmed.size === 0) {
return;
}
// Take the batch first so a write that drains it can't be appended to mid-flush
const batch = this.buffer;
this.buffer = [];
try {
this.logger.debug({
message: `Flushing ${batch.length} checks to database`,
service: this.SERVICE_NAME,
method: "flushBuffer",
});
await this.ingestChecks(batch);
if (batch.length > 0) {
this.logger.debug({ message: `Flushing ${batch.length} checks to database`, service: this.SERVICE_NAME, method: "flushBuffer" });
}
await this.ingest(batch);
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : "Unknown error",
server/src/worker/worker.check-evaluator.ts
-69
View File
@@ -1,69 +0,0 @@
import { MonitorStatusResponse } from "@/types/network.js";
import { MonitorEvaluation } from "@/worker/worker.interface.js";
import { Check } from "@/domain/checks/check.type.js";
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IStatusService } from "@/service/statusService.js";
import { MonitorActionDecision } from "@/worker/worker.helper.js";
import { ILogger } from "@/utils/logger.js";
export interface ICheckEvaluator {
evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
}
const SERVICE_NAME = "CheckEvaluator";
export class CheckEvaluator implements ICheckEvaluator {
constructor(
private statusService: IStatusService,
private monitorStatusPolicy: IMonitorStatusPolicy,
private logger: ILogger
) {}
// A check taken while the instance had no outbound connectivity says nothing about the target.
// It is left out of the status window, running stats and monitor status entirely, so that no incident
// opens for it and no spurious "resolved"/"up" fires once egress returns.
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
this.logger.debug({
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
service: SERVICE_NAME,
method: "evaluate",
});
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
return {
monitor,
status,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
decision,
};
};
evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
if (check.egressStatus === "degraded") {
return this.skipDegradedEgressCheck(status, check, monitor);
}
// ****************************
// Step 3: Evaluate and return result to reactors
// ****************************
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor);
// Step 5. Get decisions and create an evaluation obj
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
const evaluation: MonitorEvaluation = {
monitor: statusChangeResult.monitor,
status,
check,
statusChange: statusChangeResult,
decision,
};
return evaluation;
};
}
server/src/worker/worker.check-producer.ts
-157
View File
@@ -1,157 +0,0 @@
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { Check } from "@/domain/checks/check.type.js";
import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import { IBufferService } from "@/service/bufferService.js";
import { INetworkService } from "@/service/networkService.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { isEgressAttributable } from "@/domain/monitors/monitor.type.js";
export interface ICheckProducer {
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
}
const SERVICE_NAME = "CheckProducer";
export class CheckProducer implements ICheckProducer {
constructor(
private monitorsRepository: IMonitorsRepository,
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
private checkService: ICheckService,
private networkService: INetworkService,
private proxyResolver: IProxyResolver,
private bufferService: IBufferService,
private dockerLogsService: IDockerLogsService,
private egressService: IEgressService,
private logger: ILogger
) {}
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
const now = new Date();
return windows.some((w) => isWindowActive(w, now));
}
private async resolveDockerTlsKey(monitor: Monitor): Promise<string | undefined> {
if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined;
const dockerTlsKey = await this.monitorsRepository.findDockerTlsKeyById(monitor.id);
return dockerTlsKey ?? undefined;
}
// Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the
// two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx,
// or a 200 with a content mismatch).
private peerAnswered(status: MonitorStatusResponse): boolean {
return status.peerResponded ?? isHttpStatusCode(status.code);
}
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
}
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
try {
return await this.egressService.assessAfterFailure();
} catch (error: unknown) {
this.logger.warn({
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "assessEgress",
});
return null;
}
}
produce = async (monitor: Monitor) => {
if (!monitor.id) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
}
// ****************************
// Step 1: Acquire
// ****************************
// Step 1a: Maintenance window gate - skip if in maintenance
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
if (maintenanceWindowActive) {
this.logger.debug({
message: `Monitor ${monitor.id} is in maintenance window`,
service: SERVICE_NAME,
method: "produce",
});
if (monitor.status !== "maintenance") {
// Clear the status window to avoid incidents being created on next check
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] });
}
return null;
}
// Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the
// degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is
// evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage.
if (monitor.status === "maintenance") {
monitor = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" });
}
// Step 1b: Acquire status
const proxyUrl = await this.proxyResolver.resolve(monitor);
const dockerTlsKey = await this.resolveDockerTlsKey(monitor);
const status = await this.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey });
if (!status) {
throw new Error("No network response");
}
// ****************************
// Step 2: Record
// ****************************
// Step 2a: Create & record a check, return null if fail.
// Built before the egress assessment below so that the probe's duration stays out of `createdAt`.
const check = this.checkService.toCheck(status);
if (!check) {
this.logger.warn({
message: `No check could be built for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "produce",
details: { code: status.code, message: status.message },
});
return null;
}
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
if (egressStatus !== null) {
check.egressStatus = egressStatus;
}
// Step 2c: Add to buffer
this.bufferService.addToBuffer(check);
// Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops
// being evaluated, so the results either side of a spell never end up adjacent and cross the threshold
// on stale data. The length guard keeps this to one write per spell rather than one per degraded check,
// and it runs after the check is buffered so that a failed write cannot cost us the check.
if (egressStatus === "degraded" && monitor.statusWindow?.length) {
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] });
}
// Step 2e: Handle docker logs
if (status.type === "docker") {
const dockerLogs = await this.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
for (const dockerLog of dockerLogs) {
this.bufferService.addDockerLogToBuffer(dockerLog);
}
}
return { status, check };
};
}
server/src/worker/worker.db-queue.ts
+5 -108
View File
@@ -1,15 +1,8 @@
import { IQueueWorker } from "@/worker/worker.interface.js";
import { IQueueWorker, JobHandlers } from "@/worker/worker.interface.js";
import { type Job, type JobType, LOCK_MS } from "@/domain/jobs/job.type.js";
import { ILogger } from "@/utils/logger.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { ICheckProducer } from "@/worker/worker.check-producer.js";
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
@@ -36,14 +29,8 @@ export interface DBQueueWorkerDependencies {
isDbConnected: () => boolean;
jobsRepository: IJobsRepository; // forwarded
monitorsRepository: IMonitorsRepository; // forwarded
checksRepository: IChecksRepository;
checkService: ICheckService;
handlers: JobHandlers;
bufferService: IBufferService;
checkProducer: ICheckProducer;
checkEvaluator: ICheckEvaluator;
geoCheckPipeline: ICheckPipeline;
dispatcher: IReactorDispatcher;
helper: IWorkerHelper;
queueWorkersRepository: IQueueWorkersRepository; // forwarded
queueMode: QueueMode; // forwarded
queuePrimaryProcesses: boolean; // consumed immediately
@@ -65,15 +52,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
egress: 0,
};
private checksRepository: IChecksRepository;
private checkProducer: ICheckProducer;
private checkService: ICheckService;
private checkEvaluator: ICheckEvaluator;
private dispatcher: IReactorDispatcher;
private geoCheckPipeline: ICheckPipeline;
private helper: IWorkerHelper;
private bufferService: IBufferService;
private isDbConnected: () => boolean;
private handlers: JobHandlers;
constructor(dependencies: DBQueueWorkerDependencies) {
super(
@@ -85,13 +66,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
dependencies.workerId
);
this.processesJobs = dependencies.queuePrimaryProcesses === true || dependencies.queueMode === "worker";
this.checksRepository = dependencies.checksRepository;
this.checkProducer = dependencies.checkProducer;
this.checkService = dependencies.checkService;
this.checkEvaluator = dependencies.checkEvaluator;
this.dispatcher = dependencies.dispatcher;
this.geoCheckPipeline = dependencies.geoCheckPipeline;
this.helper = dependencies.helper;
this.handlers = dependencies.handlers;
this.bufferService = dependencies.bufferService;
this.isDbConnected = dependencies.isDbConnected;
}
@@ -104,64 +79,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
private getInFlightCount = () => Object.values(this.inFlight).reduce((sum, n) => sum + n, 0);
// ********************
// Stage 1: This can eventually be offloaded to a separate service
// ********************
private runCheck = async (job: Job) => {
if (!job.refId) return;
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
if (!monitor) return;
await this.checkProducer.produce(monitor);
};
// ********************
// Stage 2: Evaluator loop stays here
// ********************
private runEvaluate = async (job: Job) => {
if (!job.refId || job.pendingChecks.length === 0) return;
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
if (!monitor) {
await this.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
return;
}
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
let current = monitor;
for (const check of checks) {
const status = this.checkService.toStatusResponse(check);
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]);
if (!leaseHeld) {
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" });
return;
}
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
}
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
const found = new Set(checks.map((check) => check.id));
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
if (missing.length > 0) {
this.logger.warn({
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
service: SERVICE_NAME,
method: "runEvaluate",
});
await this.jobsRepository.pullEvaluated(job.id, missing);
}
};
private runGeoCheck = async (job: Job) => {
const monitor = await this.monitorsRepository.findByIdLean(job.refId!);
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
};
// Extend the lock while a job runs so a slow-but-alive job is never reclaimed.
private renewLock = async (job: Job): Promise<boolean> => {
try {
@@ -183,27 +100,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
if (!renewLock) clearInterval(renewTimer);
}, LOCK_RENEW_MS);
try {
switch (job.type) {
case "check":
await this.runCheck(job);
break;
case "evaluate":
await this.runEvaluate(job);
break;
case "geo-check":
await this.runGeoCheck(job);
break;
case "cleanup-orphaned":
await this.helper.getCleanupOrphanedJob()(); // Get job and execute
break;
case "cleanup-retention":
await this.helper.getCleanupRetentionJob()(); // Get job and execute
break;
case "egress":
await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
break;
}
await this.handlers[job.type](job);
if (job.intervalMs === null) {
// One-shot job
await this.jobsRepository.recordOneShot(job.id, Date.now());
server/src/worker/worker.check-pipeline.ts → server/src/worker/worker.geo-pipeline.ts
+20 -10
View File
@@ -1,5 +1,5 @@
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { MonitorEvaluation } from "@/worker/worker.interface.js";
import { JobHandler, MonitorEvaluation } from "@/worker/worker.interface.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { ILogger } from "@/utils/logger.js";
@@ -7,15 +7,19 @@ import { IBufferService } from "@/service/bufferService.js";
import { AppError } from "@/utils/AppError.js";
import { supportsGeoCheck } from "@/domain/monitors/monitor.type.js";
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
import { Job } from "@/domain/jobs/job.type.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
const SERVICE_NAME = "CheckPipeline";
const SERVICE_NAME = "GeoChecksPipeline";
export interface ICheckPipeline {
export interface IGeoChecksPipeline {
handle: JobHandler;
run(monitor: Monitor): Promise<MonitorEvaluation | null>; // null = skipped
}
export class GeoChecksPipeline implements ICheckPipeline {
export class GeoChecksPipeline implements IGeoChecksPipeline {
constructor(
private monitorsRepository: IMonitorsRepository,
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
private geoChecksService: IGeoChecksService,
private bufferService: IBufferService,
@@ -28,6 +32,12 @@ export class GeoChecksPipeline implements ICheckPipeline {
return maintenanceWindows.some((window) => isWindowActive(window, now));
}
handle: JobHandler = async (job: Job) => {
if (!job.refId) return;
const monitor = await this.monitorsRepository.findByIdLean(job.refId);
if (monitor) await this.run(monitor);
};
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
// ****************************
// Step 1: Acquire
@@ -36,7 +46,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
// Step 1a: Guards - skip if unsupported or not enabled
if (!monitor.id) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getHeartbeatGeoJob" });
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "run" });
}
if (!monitor.geoCheckEnabled) {
@@ -46,7 +56,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
this.logger.debug({
message: `Monitor ${monitor.id} type does not support geo checks, skipping`,
service: SERVICE_NAME,
method: "runGeoChecksPipeline",
method: "run",
});
return null;
}
@@ -55,7 +65,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
this.logger.warn({
message: `No geo check locations configured for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "runGeoChecksPipeline",
method: "run",
});
return null;
}
@@ -66,7 +76,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
this.logger.debug({
message: `Monitor ${monitor.id} is in maintenance window, skipping geo check`,
service: SERVICE_NAME,
method: "runGeoChecksPipeline",
method: "run",
});
return null;
}
@@ -81,7 +91,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
this.logger.warn({
message: `No geo check could be built for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "runGeoChecksPipeline",
method: "run",
});
return null;
}
@@ -92,7 +102,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
this.logger.debug({
message: `Geo check job executed for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "runGeoChecksPipeline",
method: "run",
});
return null;
};
server/src/worker/worker.helper.ts
-14
View File
@@ -20,20 +20,6 @@ export interface IWorkerHelper {
getEgressRecoveryJob(): (job: Job) => Promise<void>;
}
export interface MonitorActionDecision {
shouldCreateIncident: boolean;
shouldResolveIncident: boolean;
shouldSendNotification: boolean;
incidentReason: "status_down" | "threshold_breach" | null;
notificationReason: "status_change" | "threshold_breach" | null;
thresholdBreaches?: {
cpu?: boolean;
memory?: boolean;
disk?: boolean;
temp?: boolean;
};
}
export class WorkerHelper implements IWorkerHelper {
static SERVICE_NAME = SERVICE_NAME;
server/src/worker/worker.interface.ts
+18 -3
View File
@@ -1,17 +1,32 @@
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { Check } from "@/domain/checks/check.type.js";
import { JobType } from "@/domain/jobs/job.type.js";
import { Job, JobType } from "@/domain/jobs/job.type.js";
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
import { MonitorActionDecision } from "@/worker/worker.helper.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
import type { QueueMode } from "@/domain/app-settings/app-settings.type.js";
export type JobHandler = (job: Job) => Promise<void>;
export type JobHandlers = Record<JobType, JobHandler>;
export interface MonitorActionDecision {
shouldCreateIncident: boolean;
shouldResolveIncident: boolean;
shouldSendNotification: boolean;
incidentReason: "status_down" | "threshold_breach" | null;
notificationReason: "status_change" | "threshold_breach" | null;
thresholdBreaches?: {
cpu?: boolean;
memory?: boolean;
disk?: boolean;
temp?: boolean;
};
}
export type MonitorEvaluation = {
monitor: Monitor;
status: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>; // raw result from networkService.requestStatus
check: Check;
statusChange: StatusChangeResult; // from statusService.updateMonitorStatus
decision: MonitorActionDecision; // from MonitorStatusPolicy
decision: MonitorActionDecision;
};
export type WorkerJobFailure = {
server/src/worker/worker.monitor-status-policy.ts
-46
View File
@@ -1,46 +0,0 @@
import { StatusChangeResult } from "@/types/network.js";
import { MonitorActionDecision } from "@/worker/worker.helper.js";
export interface IMonitorStatusPolicy {
evaluate(statusChange: StatusChangeResult): MonitorActionDecision;
}
export class MonitorStatusPolicy implements IMonitorStatusPolicy {
evaluate = (statusChangeResult: StatusChangeResult): MonitorActionDecision => {
const { monitor, statusChanged, prevStatus } = statusChangeResult;
// Initialize result
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
if (!statusChanged) {
return decision;
}
if (monitor.status === "down") {
// Monitor went down (unreachable)
decision.shouldCreateIncident = true;
decision.shouldSendNotification = true;
decision.incidentReason = "status_down";
decision.notificationReason = "status_change";
} else if (monitor.status === "breached") {
// Hardware monitor exceeded thresholds
decision.shouldCreateIncident = true;
decision.shouldSendNotification = true;
decision.incidentReason = "threshold_breach";
decision.notificationReason = "threshold_breach";
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
// Monitor recovered from down or breached state
decision.shouldResolveIncident = true;
decision.shouldSendNotification = true;
decision.notificationReason = "status_change";
}
return decision;
};
}
server/src/worker/worker.pipeline.ts
+336
View File
@@ -0,0 +1,336 @@
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { Check } from "@/domain/checks/check.type.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { isEgressAttributable, Monitor } from "@/domain/monitors/monitor.type.js";
import { IBufferService } from "@/service/bufferService.js";
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
import { INetworkService } from "@/service/networkService.js";
import { IStatusService } from "@/service/statusService.js";
import { DockerStatusPayload, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js";
const SERVICE_NAME = "WorkerPipeline";
// The three entry points: the queue calls the two handlers, the buffer calls ingestChecks.
// produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer.
export interface IWorkerPipeline {
handleCheck: JobHandler;
ingestChecks(checks: Check[]): Promise<void>;
handleEvaluate: JobHandler;
}
export interface WorkerPipelineDependencies {
logger: ILogger;
monitorsRepository: IMonitorsRepository;
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
checksRepository: IChecksRepository;
jobsRepository: IJobsRepository;
checkService: ICheckService;
networkService: INetworkService;
proxyResolver: IProxyResolver;
bufferService: IBufferService;
dockerLogsService: IDockerLogsService;
egressService: IEgressService;
statusService: IStatusService;
dispatcher: IReactorDispatcher;
}
export class WorkerPipeline implements IWorkerPipeline {
static SERVICE_NAME = SERVICE_NAME;
private unarmed: Map<string, PendingCheck[]> = new Map();
constructor(private deps: WorkerPipelineDependencies) {}
private isInMaintenanceWindow = async (monitorId: string, teamId: string) => {
const windows = await this.deps.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
const now = new Date();
return windows.some((window) => isWindowActive(window, now));
};
// ****************************************************************
// Stage 1: Check job. Probe the host and buffer the result.
// ****************************************************************
handleCheck: JobHandler = async (job) => {
if (!job.refId) return;
const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId);
if (!monitor) return;
await this.produce(monitor);
};
produce = async (monitor: Monitor) => {
if (!monitor.id) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
}
// ****************************
// Step 1: Acquire
// ****************************
// Step 1a: Maintenance window gate - skip if in maintenance
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
if (maintenanceWindowActive) {
this.deps.logger.debug({
message: `Monitor ${monitor.id} is in maintenance window`,
service: SERVICE_NAME,
method: "produce",
});
if (monitor.status !== "maintenance") {
// Clear the status window to avoid incidents being created on next check
await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] });
}
return null;
}
// Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the
// degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is
// evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage.
if (monitor.status === "maintenance") {
monitor = await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" });
}
// Step 1b: Acquire status
const proxyUrl = await this.deps.proxyResolver.resolve(monitor);
const dockerTlsKey = await this.resolveDockerTlsKey(monitor);
const status = await this.deps.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey });
if (!status) {
throw new Error("No network response");
}
// ****************************
// Step 2: Record
// ****************************
// Step 2a: Create & record a check, return null if fail.
// Built before the egress assessment below so that the probe's duration stays out of `createdAt`.
const check = this.deps.checkService.toCheck(status);
if (!check) {
this.deps.logger.warn({
message: `No check could be built for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "produce",
details: { code: status.code, message: status.message },
});
return null;
}
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
if (egressStatus !== null) {
check.egressStatus = egressStatus;
}
// Step 2c: Add to buffer
this.deps.bufferService.addToBuffer(check);
// Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops
// being evaluated, so the results either side of a spell never end up adjacent and cross the threshold
// on stale data. The length guard keeps this to one write per spell rather than one per degraded check,
// and it runs after the check is buffered so that a failed write cannot cost us the check.
if (egressStatus === "degraded" && monitor.statusWindow?.length) {
await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] });
}
// Step 2e: Handle docker logs
if (status.type === "docker") {
const dockerLogs = await this.deps.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
for (const dockerLog of dockerLogs) {
this.deps.bufferService.addDockerLogToBuffer(dockerLog);
}
}
return { status, check };
};
private async resolveDockerTlsKey(monitor: Monitor): Promise<string | undefined> {
if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined;
const dockerTlsKey = await this.deps.monitorsRepository.findDockerTlsKeyById(monitor.id);
return dockerTlsKey ?? undefined;
}
// Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the
// two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx,
// or a 200 with a content mismatch).
private peerAnswered(status: MonitorStatusResponse): boolean {
return status.peerResponded ?? isHttpStatusCode(status.code);
}
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
}
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
try {
return await this.deps.egressService.assessAfterFailure();
} catch (error: unknown) {
this.deps.logger.warn({
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "assessEgress",
});
return null;
}
}
// ****************************************************************
// Stage 2: Ingestion
// Called by the buffer, checks are inserted and evaluation jobs created
// ****************************************************************
ingestChecks = async (checks: Check[]) => {
if (checks.length > 0) await this.deps.checkService.createChecks(checks);
// Group by monitor, starting from any entries a previous flush failed to arm
const byMonitor = this.unarmed;
this.unarmed = new Map();
for (const check of checks) {
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
byMonitor.set(check.metadata.monitorId, pending);
}
if (byMonitor.size === 0) return;
const now = Date.now();
const ops: Promise<boolean>[] = [];
for (const [monitorId, pendingChecks] of byMonitor) {
ops.push(
this.deps.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
// The checks are already stored, so a failed arm is retried on the next ingest rather than dropped
this.unarmed.set(monitorId, pendingChecks);
this.deps.logger.error({
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next ingest: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "ingestChecks",
});
return false;
})
);
}
await Promise.all(ops);
};
// ****************************************************************
// Stage 3: Evaluation
// Apply each pending check to the monitor -> decide -> react.
// ****************************************************************
handleEvaluate: JobHandler = async (job) => {
if (!job.refId || job.pendingChecks.length === 0) return;
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
if (!monitor) {
await this.deps.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
return;
}
const checks = await this.deps.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
let current = monitor;
for (const check of checks) {
const status = this.deps.checkService.toStatusResponse(check);
const evaluation = await this.evaluateCheck(status, check, current);
await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications
const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]);
if (!leaseHeld) {
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
this.deps.logger.warn({
message: `Lost lease on ${job.id} after check ${check.id}, stopping`,
service: SERVICE_NAME,
method: "handleEvaluate",
});
return;
}
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
}
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
const found = new Set(checks.map((check) => check.id));
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
if (missing.length > 0) {
this.deps.logger.warn({
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
service: SERVICE_NAME,
method: "handleEvaluate",
});
await this.deps.jobsRepository.pullEvaluated(job.id, missing);
}
};
// A check taken while the instance had no outbound connectivity says nothing about the target.
// It is left out of the status window, running stats and monitor status entirely, so that no incident
// opens for it and no spurious "resolved"/"up" fires once egress returns.
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
this.deps.logger.debug({
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
service: SERVICE_NAME,
method: "evaluateCheck",
});
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
return {
monitor,
status,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
decision,
};
};
evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
if (check.egressStatus === "degraded") {
return this.skipDegradedEgressCheck(status, check, monitor);
}
const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor);
const decision = this.decide(statusChange);
return { monitor: statusChange.monitor, status, check, statusChange, decision };
};
private decide = (statusChange: StatusChangeResult): MonitorActionDecision => {
const { monitor, statusChanged, prevStatus } = statusChange;
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
if (!statusChanged) return decision;
if (monitor.status === "down") {
decision.shouldCreateIncident = true;
decision.shouldSendNotification = true;
decision.incidentReason = "status_down";
decision.notificationReason = "status_change";
} else if (monitor.status === "breached") {
decision.shouldCreateIncident = true;
decision.shouldSendNotification = true;
decision.incidentReason = "threshold_breach";
decision.notificationReason = "threshold_breach";
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
decision.shouldResolveIncident = true;
decision.shouldSendNotification = true;
decision.notificationReason = "status_change";
}
return decision;
};
}
server/test/helpers/heartbeatTestHarness.ts
+20 -18
View File
@@ -1,7 +1,5 @@
import { jest } from "@jest/globals";
import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts";
import { CheckProducer } from "../../src/worker/worker.check-producer.ts";
import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts";
import { WorkerPipeline } from "../../src/worker/worker.pipeline.ts";
import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts";
import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts";
import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts";
@@ -139,31 +137,35 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
nextEgressStatus = status;
};
const checkProducer = new CheckProducer(
monitorsRepo as any,
maintenanceWindowsRepo as any,
checkService as any,
networkService as any,
proxyResolver as any,
bufferStub as any,
dockerLogsService as any,
egressService as any,
logger
);
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger);
const notificationReactor = new NotificationReactor(notificationsService as any);
const incidentReactor = new IncidentReactor(incidentService as any);
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
// The harness drives stages 1 and 3 directly, so the stage 2 repositories are never reached.
const pipeline = new WorkerPipeline({
logger,
monitorsRepository: monitorsRepo as any,
maintenanceWindowsRepository: maintenanceWindowsRepo as any,
checksRepository: { findUnevaluatedByMonitorId: jest.fn() } as any,
jobsRepository: { upsertEvaluate: jest.fn(), pullEvaluated: jest.fn() } as any,
checkService: checkService as any,
networkService: networkService as any,
proxyResolver: proxyResolver as any,
bufferService: bufferStub as any,
dockerLogsService: dockerLogsService as any,
egressService: egressService as any,
statusService,
dispatcher: reactorDispatcher,
});
// Mirror the production check→evaluate flow: read the monitor fresh each cycle (so the
// accumulated statusWindow is visible), produce a check, then evaluate against that monitor.
const heartbeatJob = async (seedMonitor: Monitor) => {
const monitor = await monitorsRepo.findByIdLean(seedMonitor.id);
if (!monitor) return;
const result = await checkProducer.produce(monitor);
const result = await pipeline.produce(monitor);
if (!result) return; // skipped (e.g. maintenance window)
const evaluation = await checkEvaluator.evaluate(result.status, result.check, monitor);
const evaluation = await pipeline.evaluateCheck(result.status, result.check, monitor);
await reactorDispatcher.dispatch(evaluation);
};
server/test/integration/evaluateCursorMultiWorker.test.ts
+47 -29
View File
@@ -8,6 +8,7 @@ import JobModel from "../../src/domain/jobs/job.model.ts";
import { CheckService } from "../../src/domain/checks/check.service.ts";
import { BufferService } from "../../src/service/bufferService.ts";
import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts";
import { WorkerPipeline } from "../../src/worker/worker.pipeline.ts";
import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts";
import { createMockLogger } from "../helpers/createMockLogger.ts";
import type { Job } from "../../src/domain/jobs/job.type.ts";
@@ -29,8 +30,9 @@ import type { ILogger } from "../../src/utils/logger.ts";
//
// The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each
// is applied, so a late batch is always evaluated. This drives the real time-series query, the
// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an
// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's.
// real jobs collection, the real buffer flush, the real pipeline ingest and evaluate stages and the
// real DBQueueWorker job path against an in-process mongod, so the handoff semantics under test
// are MongoDB's, not a mock's.
let mongod: MongoMemoryServer;
@@ -105,16 +107,39 @@ describe("Evaluate cursor with more than one processing worker", () => {
let evaluator: DBQueueWorker;
let evaluatorJobsRepository: MongoJobsRepository;
// One BufferService per worker process, each flushing into the same collection and jobs table.
const createWorkerBuffer = (workerId: string) =>
new BufferService(
// One pipeline and BufferService per worker process, each flushing into the same collection and
// jobs table. Only stages 2 and 3 are exercised: the status service is stubbed to record which
// checks reached evaluation, and stage 1's network dependencies are never called.
const createWorkerPipeline = (workerId: string) => {
const jobsRepository = new MongoJobsRepository(workerId);
const bufferService = new BufferService(
logger,
checkService,
stubGeoChecksService as any,
stubDockerLogsService as any,
stubSettingsService as any,
new MongoJobsRepository(workerId)
(checks): Promise<void> => pipeline.ingestChecks(checks)
);
const pipeline = new WorkerPipeline({
logger,
monitorsRepository: monitorsRepository as any,
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([]) } as any,
checksRepository,
jobsRepository,
checkService,
networkService: { requestStatus: jest.fn<any>() } as any,
proxyResolver: { resolve: jest.fn<any>() } as any,
bufferService,
dockerLogsService: { buildDockerLogs: jest.fn<any>() } as any,
statusService: {
updateMonitorStatus: async (_status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
evaluatedCheckIds.push(check.id);
return { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 };
},
} as any,
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) } as any,
});
return { pipeline, bufferService, jobsRepository };
};
// Claim every due evaluate job and run it through the worker's real per-job path.
const runDueEvaluateJobs = async () => {
@@ -130,35 +155,28 @@ describe("Evaluate cursor with more than one processing worker", () => {
monitorsRepository.seed(makeMonitor());
checksRepository = new MongoChecksRepository(logger);
checkService = new CheckService(monitorsRepository, logger, checksRepository);
bufferA = createWorkerBuffer("worker-a");
bufferB = createWorkerBuffer("worker-b");
evaluatedCheckIds = [];
evaluatorJobsRepository = new MongoJobsRepository("worker-b");
const workerA = createWorkerPipeline("worker-a");
const workerB = createWorkerPipeline("worker-b");
bufferA = workerA.bufferService;
bufferB = workerB.bufferService;
const unused = jest.fn<any>().mockResolvedValue(undefined);
evaluatorJobsRepository = workerB.jobsRepository;
evaluator = new DBQueueWorker({
logger,
isDbConnected: () => true,
jobsRepository: evaluatorJobsRepository,
monitorsRepository,
checksRepository,
checkService,
bufferService: bufferB,
checkProducer: { produce: jest.fn<any>() },
checkEvaluator: {
evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
evaluatedCheckIds.push(check.id);
return {
monitor,
status,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 },
decision: {},
};
},
} as any,
geoCheckPipeline: { run: jest.fn<any>() },
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
helper: {} as any,
handlers: {
check: unused,
evaluate: workerB.pipeline.handleEvaluate,
"geo-check": unused,
"cleanup-orphaned": unused,
"cleanup-retention": unused,
egress: unused,
},
queueWorkersRepository: {} as any,
queueMode: "worker",
queuePrimaryProcesses: true,
server/test/integration/incidentLifecycle.test.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import type { IMonitorsRepository } from "../../src/domain/monitors/monitor.repo
import type { IUsersRepository } from "../../src/domain/users/user.repository.interface.ts";
import type { INotificationMessageBuilder } from "../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
import type { MonitorActionDecision } from "../../src/worker/worker.helper.ts";
import type { MonitorActionDecision } from "../../src/worker/worker.interface.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
server/test/unit/services/bufferService.test.ts
+35 -106
View File
@@ -1,10 +1,8 @@
import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals";
import { BufferService } from "../../../src/service/bufferService.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
import type { ICheckService } from "../../../src/domain/checks/check.service.ts";
import type { IGeoChecksService } from "../../../src/domain/geo-checks/geo-check.service.ts";
import type { ISettingsService } from "../../../src/domain/app-settings/app-settings.service.ts";
import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { GeoCheck } from "../../../src/domain/geo-checks/geo-check.type.ts";
import type { IDockerLogsService } from "../../../src/domain/docker/docker-log.service.ts";
@@ -12,11 +10,6 @@ import type { DockerLog } from "../../../src/domain/docker/docker-log.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
const createMockCheckService = () =>
({
createChecks: jest.fn().mockResolvedValue([]),
}) as unknown as jest.Mocked<ICheckService>;
const createMockGeoChecksService = () =>
({
createGeoChecks: jest.fn().mockResolvedValue([]),
@@ -32,11 +25,6 @@ const createMockSettingsService = (nodeEnv: string = "development") =>
getSettings: jest.fn().mockReturnValue({ nodeEnv }),
}) as unknown as jest.Mocked<ISettingsService>;
const createMockJobsRepository = () =>
({
upsertEvaluate: jest.fn().mockResolvedValue(true),
}) as unknown as jest.Mocked<IJobsRepository>;
const makeCheck = (overrides?: Partial<Check>): Check =>
({
id: "check-1",
@@ -72,13 +60,12 @@ const makeDockerLog = (overrides?: Partial<DockerLog>): DockerLog =>
const createService = (nodeEnv: string = "development") => {
const logger = createMockLogger();
const checkService = createMockCheckService();
const ingest = jest.fn<(checks: Check[]) => Promise<void>>().mockResolvedValue(undefined);
const geoChecksService = createMockGeoChecksService();
const dockerLogsService = createMockDockerLogsService();
const settingsService = createMockSettingsService(nodeEnv);
const jobsRepository = createMockJobsRepository();
const service = new BufferService(logger as any, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
return { service, logger, checkService, geoChecksService, dockerLogsService, jobsRepository };
const service = new BufferService(logger as any, geoChecksService, dockerLogsService, settingsService, ingest);
return { service, logger, ingest, geoChecksService, dockerLogsService };
};
// ── Tests ────────────────────────────────────────────────────────────────────
@@ -127,17 +114,17 @@ describe("BufferService", () => {
describe("addToBuffer", () => {
it("adds a check to the buffer", async () => {
const { service, checkService } = createService();
const { service, ingest } = createService();
const check = makeCheck();
service.addToBuffer(check);
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledWith([check]);
expect(ingest).toHaveBeenCalledWith([check]);
});
it("adds multiple checks to the buffer", async () => {
const { service, checkService } = createService();
const { service, ingest } = createService();
const check1 = makeCheck({ id: "c1" });
const check2 = makeCheck({ id: "c2" });
@@ -145,7 +132,7 @@ describe("BufferService", () => {
service.addToBuffer(check2);
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledWith([check1, check2]);
expect(ingest).toHaveBeenCalledWith([check1, check2]);
});
it("logs error if push throws", () => {
@@ -268,21 +255,20 @@ describe("BufferService", () => {
});
it("flushes all buffers when timer fires", async () => {
const { service, checkService, geoChecksService, dockerLogsService } = createService();
const { service, ingest, geoChecksService, dockerLogsService } = createService();
service.addToBuffer(makeCheck());
service.addGeoCheckToBuffer(makeGeoCheck());
service.addDockerLogToBuffer(makeDockerLog());
await jest.advanceTimersByTimeAsync(1000);
expect(checkService.createChecks).toHaveBeenCalled();
expect(ingest).toHaveBeenCalledWith([expect.objectContaining({ id: "check-1" })]);
expect(geoChecksService.createGeoChecks).toHaveBeenCalled();
expect(dockerLogsService.createDockerLogs).toHaveBeenCalled();
});
it("reschedules after flush completes", async () => {
const { service, checkService } = createService();
(checkService.createChecks as jest.Mock).mockResolvedValue([]);
const { service, ingest } = createService();
service.addToBuffer(makeCheck());
await jest.advanceTimersByTimeAsync(1000);
@@ -291,12 +277,12 @@ describe("BufferService", () => {
service.addToBuffer(makeCheck({ id: "c2" }));
await jest.advanceTimersByTimeAsync(1000);
expect(checkService.createChecks).toHaveBeenCalledTimes(2);
expect(ingest).toHaveBeenCalledTimes(2);
});
it("reschedules even when flush throws", async () => {
const { service, checkService, logger } = createService();
(checkService.createChecks as jest.Mock).mockRejectedValueOnce(new Error("DB down"));
const { service, ingest, logger } = createService();
ingest.mockRejectedValueOnce(new Error("DB down"));
service.addToBuffer(makeCheck());
await jest.advanceTimersByTimeAsync(1000);
@@ -309,11 +295,10 @@ describe("BufferService", () => {
);
// Should still reschedule — add another check and flush
(checkService.createChecks as jest.Mock).mockResolvedValue([]);
service.addToBuffer(makeCheck({ id: "c2" }));
await jest.advanceTimersByTimeAsync(1000);
expect(checkService.createChecks).toHaveBeenCalledTimes(2);
expect(ingest).toHaveBeenCalledTimes(2);
});
it("logs error and reschedules when flush throws past its own catch", async () => {
@@ -350,25 +335,26 @@ describe("BufferService", () => {
// ── flushBuffer ──────────────────────────────────────────────────────
describe("flushBuffer", () => {
it("does nothing when buffer is empty", async () => {
const { service, checkService } = createService();
it("hands an empty batch to ingest so the pipeline can retry anything it left unarmed", async () => {
const { service, ingest, logger } = createService();
await service.flushBuffer();
expect(checkService.createChecks).not.toHaveBeenCalled();
expect(ingest).toHaveBeenCalledWith([]);
expect(logger.debug).not.toHaveBeenCalled();
});
it("flushes checks to checksService and clears buffer", async () => {
const { service, checkService } = createService();
it("hands the batch to ingest and clears the buffer", async () => {
const { service, ingest } = createService();
const check = makeCheck();
service.addToBuffer(check);
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledWith([check]);
expect(ingest).toHaveBeenNthCalledWith(1, [check]);
// Buffer should be empty now
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(ingest).toHaveBeenNthCalledWith(2, []);
});
it("logs debug message before flushing", async () => {
@@ -386,68 +372,11 @@ describe("BufferService", () => {
);
});
it("arms the evaluate stage once per distinct monitor after the checks are stored", async () => {
const { service, jobsRepository } = createService();
service.addToBuffer(makeCheck({ id: "c1", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }));
service.addToBuffer(makeCheck({ id: "c2", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }));
service.addToBuffer(makeCheck({ id: "c3", metadata: { monitorId: "mon-2", teamId: "team-1", type: "http" } }));
await service.flushBuffer();
const createdAt = Date.parse("2026-01-01T00:00:00.000Z");
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith(
"mon-1",
[
{ checkId: "c1", createdAt },
{ checkId: "c2", createdAt },
],
expect.any(Number)
);
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
});
it("retries a failed arm on the next flush without re-storing the checks", async () => {
const { service, checkService, jobsRepository, logger } = createService();
(jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed"));
service.addToBuffer(makeCheck({ id: "c1" }));
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
await service.flushBuffer(); // buffer is empty, only the retry runs
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number));
await service.flushBuffer(); // nothing left to retry
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
});
it("ingestChecks with an empty batch neither writes nor arms", async () => {
const { service, checkService, jobsRepository } = createService();
await service.ingestChecks([]);
expect(checkService.createChecks).not.toHaveBeenCalled();
expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
});
it("does not arm the evaluate stage when the check write fails", async () => {
const { service, checkService, jobsRepository } = createService();
(checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed"));
service.addToBuffer(makeCheck());
await service.flushBuffer();
expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
});
it("clears buffer even on error to prevent infinite retries", async () => {
const { service, checkService, logger } = createService();
(checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed"));
service.addToBuffer(makeCheck());
const { service, ingest, logger } = createService();
ingest.mockRejectedValueOnce(new Error("DB write failed"));
const check = makeCheck();
service.addToBuffer(check);
await service.flushBuffer();
@@ -457,14 +386,14 @@ describe("BufferService", () => {
method: "flushBuffer",
})
);
// Buffer should be cleared
// Buffer should be cleared: the failed batch is not handed over again
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(ingest).toHaveBeenNthCalledWith(2, []);
});
it("logs 'Unknown error' for non-Error thrown values", async () => {
const { service, checkService, logger } = createService();
(checkService.createChecks as jest.Mock).mockRejectedValue(null);
const { service, ingest, logger } = createService();
ingest.mockRejectedValue(null);
service.addToBuffer(makeCheck());
await service.flushBuffer();
@@ -609,28 +538,28 @@ describe("BufferService", () => {
describe("shutdown", () => {
it("stops the flush timer and flushes all buffers", async () => {
const { service, checkService, geoChecksService, dockerLogsService } = createService();
const { service, ingest, geoChecksService, dockerLogsService } = createService();
service.addToBuffer(makeCheck());
service.addGeoCheckToBuffer(makeGeoCheck());
service.addDockerLogToBuffer(makeDockerLog());
await service.shutdown();
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(ingest).toHaveBeenCalledTimes(1);
expect(geoChecksService.createGeoChecks).toHaveBeenCalledTimes(1);
expect(dockerLogsService.createDockerLogs).toHaveBeenCalledTimes(1);
// Timer is cleared: advancing past the flush interval triggers no further flush.
await jest.advanceTimersByTimeAsync(60 * 1000);
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(ingest).toHaveBeenCalledTimes(1);
});
it("is safe to call with empty buffers", async () => {
const { service, checkService, geoChecksService, dockerLogsService } = createService();
const { service, ingest, geoChecksService, dockerLogsService } = createService();
await service.shutdown();
expect(checkService.createChecks).not.toHaveBeenCalled();
expect(ingest).toHaveBeenCalledWith([]);
expect(geoChecksService.createGeoChecks).not.toHaveBeenCalled();
expect(dockerLogsService.createDockerLogs).not.toHaveBeenCalled();
});
server/test/unit/services/checkEvaluator.test.ts
-123
View File
@@ -1,123 +0,0 @@
import { describe, expect, it, jest } from "@jest/globals";
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "m1",
teamId: "team",
type: "http",
interval: 60000,
status: "up",
...overrides,
}) as Monitor;
const createEvaluator = (overrides?: Record<string, any>) => {
const defaults = {
statusService: {
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
},
monitorStatusPolicy: new MonitorStatusPolicy(),
logger: createMockLogger(),
...overrides,
};
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
return { evaluator, defaults };
};
describe("CheckEvaluator", () => {
it("threads status, check, and monitor through to the status service", async () => {
const { evaluator, defaults } = createEvaluator();
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const check = { id: "check-1" } as any;
const monitor = makeMonitor({ status: "up" });
await evaluator.evaluate(status, check, monitor);
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor);
});
it("returns an evaluation that opens an incident on a down transition", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1" } as any;
const statusChange = { monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 };
const { evaluator } = createEvaluator({
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
});
const result = await evaluator.evaluate(status, check, makeMonitor());
expect(result).toMatchObject({
monitor: statusChange.monitor,
status,
check,
statusChange,
decision: expect.objectContaining({
shouldCreateIncident: true,
shouldSendNotification: true,
incidentReason: "status_down",
}),
});
});
it("returns an evaluation that resolves an incident on recovery", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const statusChange = { monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down", code: 200 };
const { evaluator } = createEvaluator({
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
});
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor({ status: "down" }));
expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
});
it("produces a no-op decision when status did not change", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const { evaluator } = createEvaluator();
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor());
expect(result.decision).toMatchObject({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
});
});
// ── degraded egress ───────────────────────────────────────────────────────
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "degraded" } as any;
const monitor = makeMonitor({ status: "up" });
const { evaluator, defaults } = createEvaluator();
const result = await evaluator.evaluate(status, check, monitor);
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
expect(result.monitor).toBe(monitor);
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
expect(result.decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
});
expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
});
it("evaluates a check flagged with egress ok as normal", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "ok" } as any;
const { evaluator, defaults } = createEvaluator();
await evaluator.evaluate(status, check, makeMonitor());
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
});
});
server/test/unit/services/checkProducer.test.ts
-461
View File
@@ -1,461 +0,0 @@
import { describe, expect, it, jest } from "@jest/globals";
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
import { NETWORK_ERROR } from "../../../src/types/network.ts";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "m1",
teamId: "team",
type: "http",
interval: 60000,
status: "up",
...overrides,
}) as Monitor;
// A maintenance window straddling "now" — active per isWindowActive.
const activeWindow = () => {
const now = Date.now();
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
};
const createProducer = (overrides?: Record<string, any>) => {
const defaults = {
logger: createMockLogger(),
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}), findDockerTlsKeyById: jest.fn().mockResolvedValue(null) },
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
buffer: { addToBuffer: jest.fn() },
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
...overrides,
};
const producer = new CheckProducer(
defaults.monitorsRepository as any,
defaults.maintenanceWindowsRepository as any,
defaults.checkService as any,
defaults.networkService as any,
defaults.proxyResolver as any,
defaults.buffer as any,
defaults.dockerLogsService as any,
defaults.egressService as any,
defaults.logger as any
);
return { producer, defaults };
};
describe("CheckProducer", () => {
it("throws when monitor id is missing", async () => {
const { producer } = createProducer();
await expect(producer.produce({} as Monitor)).rejects.toThrow("No monitor id");
});
// ── maintenance gate ──────────────────────────────────────────────────────
it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => {
const { producer, defaults } = createProducer({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
});
const result = await producer.produce(makeMonitor({ status: "up" }));
expect(result).toBeNull();
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
});
it("does not re-write status when already in maintenance", async () => {
const { producer, defaults } = createProducer({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
});
const result = await producer.produce(makeMonitor({ status: "maintenance" }));
expect(result).toBeNull();
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
const monitor = makeMonitor({ status: "maintenance" });
const { producer, defaults } = createProducer({
monitorsRepository: {
updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }),
findDockerTlsKeyById: jest.fn().mockResolvedValue(null),
},
});
await producer.produce(monitor);
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
});
it("does not touch the status of a monitor that was not in maintenance", async () => {
const { producer, defaults } = createProducer();
await producer.produce(makeMonitor({ status: "up" }));
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("proceeds with the check when the maintenance window is inactive", async () => {
const { producer, defaults } = createProducer({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
});
await producer.produce(makeMonitor());
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
});
// ── proxy resolution ──────────────────────────────────────────────────────
it("passes the resolved proxy url into requestStatus", async () => {
const { producer, defaults } = createProducer({
proxyResolver: { resolve: jest.fn().mockResolvedValue("http://proxy.example.com:8080") },
});
const monitor = makeMonitor();
await producer.produce(monitor);
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: "http://proxy.example.com:8080" });
});
it("still produces a check when the resolver returns undefined", async () => {
const { producer, defaults } = createProducer();
const monitor = makeMonitor();
const result = await producer.produce(monitor);
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined });
expect(result).toEqual({ status: expect.objectContaining({ monitorId: "m1" }), check: { id: "check-1" } });
});
// ── acquire / record ──────────────────────────────────────────────────────
// ── docker tls key resolution ─────────────────────────────────────────────
it("fetches the stored key for a docker monitor with a key set and passes it in the context", async () => {
const { producer, defaults } = createProducer({
monitorsRepository: { updateById: jest.fn(), findDockerTlsKeyById: jest.fn().mockResolvedValue("v1.abc123.iv.tag.data") },
});
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
await producer.produce(monitor);
expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: "v1.abc123.iv.tag.data" });
});
it("does not fetch a key for a docker monitor without one set", async () => {
const { producer, defaults } = createProducer();
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKeySet: false });
await producer.produce(monitor);
expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
});
it("does not fetch a key for non-docker monitors even if the flag is set", async () => {
const { producer, defaults } = createProducer();
await producer.produce(makeMonitor({ type: "http", dockerTlsKeySet: true }));
expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
});
it("passes undefined when the flag is set but the repository has no key", async () => {
const { producer, defaults } = createProducer();
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
await producer.produce(monitor);
expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
});
it("throws when the network response is null", async () => {
const { producer } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(null) },
});
await expect(producer.produce(makeMonitor())).rejects.toThrow("No network response");
});
it("returns null, warns, and does not buffer when toCheck yields nothing", async () => {
const { producer, defaults } = createProducer({
checkService: { toCheck: jest.fn().mockReturnValue(null) },
});
const result = await producer.produce(makeMonitor());
expect(result).toBeNull();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
});
it("buffers the built check and returns the status and check on success", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
const check = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
const result = await producer.produce(makeMonitor());
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
expect(result).toEqual({ status, check });
});
it("builds and buffers docker logs for a docker status", async () => {
const status = { type: "docker", monitorId: "m1", teamId: "team", status: true, code: 200, message: "OK", payload: {} };
const dockerLogs = [{ id: "docker-log-1" }, { id: "docker-log-2" }];
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
buffer: { addToBuffer: jest.fn(), addDockerLogToBuffer: jest.fn() },
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue(dockerLogs) },
});
await producer.produce(makeMonitor({ type: "docker" }));
expect(defaults.dockerLogsService.buildDockerLogs).toHaveBeenCalledWith(status);
expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(1, dockerLogs[0]);
expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(2, dockerLogs[1]);
});
it("does not build docker logs for a non-docker status", async () => {
const { producer, defaults } = createProducer();
await producer.produce(makeMonitor());
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
});
// ── egress self-check ─────────────────────────────────────────────────────
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
});
it("does not consult egress when an HTTP response was received", async () => {
for (const status of [
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
]) {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
}
});
it("consults egress for a timeout or network error", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
});
it("never consults egress for a docker daemon on a local socket", async () => {
for (const url of ["unix:///var/run/docker.sock", "/var/run/docker.sock"]) {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ type: "docker", url }));
// Local IPC never leaves the instance, so it cannot fail through egress.
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
}
});
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
for (const monitor of [
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
]) {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(monitor);
// Both requests leave the instance, so our own loss of egress can be the cause.
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
}
});
it("does not consult egress when the provider reports that the peer answered", async () => {
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
// them from a connection failure.
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ type: "dns" }));
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
});
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ type: "grpc" }));
// An explicit report from the provider outranks the HTTP-status heuristic.
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
});
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
const result = await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
expect(result?.check.egressStatus).toBe("degraded");
});
// ── status window across a spell ─────────────────────────────────────────
// Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either
// side of a spell are never adjacent.
it("clears the status window on the first degraded check of a spell", async () => {
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] });
});
it("does not re-write the status window for later degraded checks in the same spell", async () => {
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ statusWindow: [] }));
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("leaves the status window alone when the probe found egress fine", async () => {
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
});
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("flags a failing check as ok when the probe found egress fine", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
});
await producer.produce(makeMonitor());
expect(check.egressStatus).toBe("ok");
});
it("buffers the check when the egress assessment rejects", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) },
});
const result = await producer.produce(makeMonitor());
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
expect(check).not.toHaveProperty("egressStatus");
expect(result).toEqual({ status: failingStatus, check });
expect(defaults.logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" })
);
});
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check).not.toHaveProperty("egressStatus");
});
});
server/test/unit/services/dbQueueWorker.test.ts
+79 -197
View File
@@ -9,10 +9,11 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
// ── Notes ──────────────────────────────────────────────────────────────────────
//
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and
// renews their locks, and registers itself in the worker heartbeat. The repository
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
// test only the orchestration around it, with the repository fully mocked.
// sized to free capacity, hands each job to the handler registered for its type,
// renews locks while a job runs, and registers itself in the worker heartbeat. What a
// handler does is covered in workerPipeline.test.ts and geoChecksPipeline.test.ts; the
// repository claim/lock atomicity in integration/jobsRepository.test.ts. Here we test
// only the orchestration, with the repository and every handler mocked.
//
// Fake timers are mandatory: init() starts self-re-arming setTimeout loops and a
// heartbeat setInterval, so without them the loops would leak across tests. The
@@ -73,6 +74,17 @@ const makeEvaluateJob = (...checkIds: string[]): Job =>
pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })),
});
// One resolved mock per job type; a test overrides the one it drives
const makeHandlers = (overrides?: Record<string, any>) => ({
check: jest.fn<any>().mockResolvedValue(undefined),
evaluate: jest.fn<any>().mockResolvedValue(undefined),
"geo-check": jest.fn<any>().mockResolvedValue(undefined),
"cleanup-orphaned": jest.fn<any>().mockResolvedValue(undefined),
"cleanup-retention": jest.fn<any>().mockResolvedValue(undefined),
egress: jest.fn<any>().mockResolvedValue(undefined),
...overrides,
});
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
const jobsRepository = {
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
@@ -98,12 +110,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
findAllForScheduling: jest.fn<any>().mockResolvedValue([]),
updateById: jest.fn<any>().mockResolvedValue({}),
};
const checksRepository = {
findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([]),
};
const checkService = {
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
};
const bufferService = {
addToBuffer: jest.fn<any>(),
addGeoCheckToBuffer: jest.fn<any>(),
@@ -112,19 +118,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
flushGeoBuffer: jest.fn<any>().mockResolvedValue(undefined),
shutdown: jest.fn<any>().mockResolvedValue(undefined),
};
const checkProducer = {
produce: jest.fn<any>().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }),
};
const checkEvaluator = {
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), statusChange: { monitor: makeMonitor() }, decision: {} }),
};
const geoCheckPipeline = { run: jest.fn<any>().mockResolvedValue(null) };
const dispatcher = { dispatch: jest.fn<any>().mockResolvedValue(undefined) };
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
const handlers = makeHandlers();
const queueWorkersRepository = {
upsert: jest.fn<any>().mockResolvedValue(undefined),
deleteById: jest.fn<any>().mockResolvedValue(undefined),
@@ -137,14 +131,8 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
isDbConnected,
jobsRepository,
monitorsRepository,
checksRepository,
checkService,
bufferService,
checkProducer,
checkEvaluator,
geoCheckPipeline,
dispatcher,
helper,
handlers,
queueWorkersRepository,
logger,
...overrides?.mocks,
@@ -282,19 +270,18 @@ describe("DBQueueWorker", () => {
const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
expect(mocks.checkProducer.produce).toHaveBeenCalled();
// The check stage only produces; the buffer arms evaluate once the check is durably stored
expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
expect(mocks.handlers.check).toHaveBeenCalledWith(job);
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled();
});
it("records a failure (and not a success) when the job handler throws", async () => {
const job = makeJob({ type: "check" });
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: failingProducer },
mocks: {
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
handlers: makeHandlers({ check: jest.fn<any>().mockRejectedValue(new Error("network down")) }),
},
});
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
@@ -310,7 +297,6 @@ describe("DBQueueWorker", () => {
process.on("unhandledRejection", onUnhandled);
try {
const job = makeJob({ type: "check" });
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
const { mocks, worker } = await start({
mocks: {
jobsRepository: {
@@ -318,7 +304,7 @@ describe("DBQueueWorker", () => {
claimDueBatch: claimOnce(job),
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
},
checkProducer: failingProducer,
handlers: makeHandlers({ check: jest.fn<any>().mockRejectedValue(new Error("network down")) }),
},
});
await jest.advanceTimersByTimeAsync(1); // let the recordFailure rejection settle
@@ -336,7 +322,7 @@ describe("DBQueueWorker", () => {
});
it("swallows the failure when both the success and failure writes reject", async () => {
const job = makeJob({ type: "check", intervalMs: 60000 }); // produce succeeds → recordSuccess path
const job = makeJob({ type: "check", intervalMs: 60000 }); // handler succeeds → recordSuccess path
const { mocks, worker } = await start({
mocks: {
jobsRepository: {
@@ -367,7 +353,7 @@ describe("DBQueueWorker", () => {
claimDueBatch: claimOnce(job),
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
},
checkProducer: { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) },
handlers: makeHandlers({ check: jest.fn<any>().mockRejectedValue(new Error("network down")) }),
},
});
await jest.advanceTimersByTimeAsync(1);
@@ -386,173 +372,66 @@ describe("DBQueueWorker", () => {
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
});
it("evaluate job loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
const job = makeEvaluateJob("c1");
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
it("routes every job type to the handler registered for it, with the claimed job", async () => {
const jobs: Job[] = [
makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null, pendingChecks: [{ checkId: "c1", createdAt: 1000 }] }),
makeJob({ id: "geo-check:m1", type: "geo-check", intervalMs: 300000 }),
makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 }),
makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 }),
makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 }),
];
const handed = new Set<string>();
const claimDueBatch = jest.fn<any>(async (type: string) => {
const job = jobs.find((j) => j.type === type);
if (!job || handed.has(type)) return [];
handed.add(type);
return [job];
});
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch } } });
expect(checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks);
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
// The monitor cursor is gone: nothing writes lastEvaluatedAt
expect(mocks.monitorsRepository.updateById).not.toHaveBeenCalled();
for (const job of jobs) {
expect(mocks.handlers[job.type]).toHaveBeenCalledWith(job);
}
expect(mocks.handlers.check).not.toHaveBeenCalled();
});
it("evaluate job with no pending checks does nothing", async () => {
const job = makeEvaluateJob();
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
expect(mocks.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled();
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
});
it("evaluate job for a deleted monitor drops every pending id", async () => {
const job = makeEvaluateJob("c1", "c2");
const monitorsRepository = { findByIdLean: jest.fn<any>().mockResolvedValue(null), updateById: jest.fn<any>() };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, monitorsRepository },
});
expect(mocks.checkEvaluator.evaluate).not.toHaveBeenCalled();
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
});
it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
const job = makeEvaluateJob("c1", "c2", "c3");
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) };
const jobsRepository = {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
};
const { mocks } = await start({ mocks: { jobsRepository, checksRepository } });
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2);
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" }));
});
it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
const job = makeEvaluateJob("c1", "c-gone");
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
});
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(1);
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]);
});
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
const job = makeEvaluateJob("c1", "c2");
const initialMonitor = makeMonitor();
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
const monitorsRepository = {
findByIdLean: jest.fn<any>().mockResolvedValue(initialMonitor),
updateById: jest.fn<any>().mockResolvedValue({}),
};
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) };
const checkEvaluator = {
evaluate: jest.fn<any>().mockResolvedValue({ monitor: postWriteMonitor, statusChange: { monitor: postWriteMonitor }, decision: {} }),
};
const { mocks } = await start({
mocks: {
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
monitorsRepository,
checksRepository,
checkEvaluator,
},
});
// The monitor is read once for the whole backlog, not once per check
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1);
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
// First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first
expect(mocks.checkEvaluator.evaluate.mock.calls[0][2]).toBe(initialMonitor);
expect(mocks.checkEvaluator.evaluate.mock.calls[1][2]).toBe(postWriteMonitor);
});
it("cleanup-orphaned job runs the helper's cleanup function", async () => {
const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 });
const cleanupFn = jest.fn<any>().mockResolvedValue(undefined);
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(cleanupFn),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
expect(cleanupFn).toHaveBeenCalled();
});
it("cleanup-retention job runs the helper's retention function", async () => {
const job = makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 });
const retentionFn = jest.fn<any>().mockResolvedValue(undefined);
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(retentionFn),
};
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
expect(retentionFn).toHaveBeenCalled();
});
it("egress job runs the helper's egress recovery function with the claimed job", async () => {
it("reschedules an interval job after its handler succeeds", async () => {
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
const egressFn = jest.fn<any>().mockResolvedValue(undefined);
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
};
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
});
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.geoCheckPipeline.run).toHaveBeenCalled();
expect(mocks.dispatcher.dispatch).not.toHaveBeenCalled();
expect(mocks.handlers.egress).toHaveBeenCalledWith(job);
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
});
it("renews the lock while a slow job is still running", async () => {
const job = makeJob({ type: "check" });
const gate = deferred<{ status: unknown; check: unknown }>();
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
const gate = deferred<void>();
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer },
mocks: {
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
},
});
// Job is parked on produce(); push past one renewal interval.
// Job is parked in its handler; push past one renewal interval.
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledWith([job.id], expect.any(Number));
// Let it finish; success is still recorded.
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
gate.resolve();
await jest.advanceTimersByTimeAsync(1);
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled();
});
it("stops renewing once the lock is lost to another worker", async () => {
const job = makeJob({ type: "check" });
const gate = deferred<{ status: unknown; check: unknown }>();
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
const gate = deferred<void>();
const jobsRepository = {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
renewLocks: jest.fn<any>().mockResolvedValue(0),
};
const { mocks } = await start({ mocks: { jobsRepository, checkProducer: slowProducer } });
const { mocks } = await start({ mocks: { jobsRepository, handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }) } });
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); // lease lost → renewal timer stops
@@ -562,7 +441,7 @@ describe("DBQueueWorker", () => {
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1);
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
gate.resolve();
await jest.advanceTimersByTimeAsync(1);
});
});
@@ -683,11 +562,8 @@ describe("DBQueueWorker", () => {
}
return [];
});
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>(() => gate.promise)), // blocks → keeps the slot full
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, helper } });
const handlers = makeHandlers({ "cleanup-orphaned": jest.fn<any>(() => gate.promise) }); // blocks → keeps the slot full
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, handlers } });
const before = claims(mocks, "cleanup-orphaned"); // the blocking job is now in-flight → capacity 0
await jest.advanceTimersByTimeAsync(1000);
@@ -804,19 +680,21 @@ describe("DBQueueWorker", () => {
it("waits for an in-flight job to finish before flushing", async () => {
const job = makeJob({ type: "check" });
const gate = deferred<{ status: unknown; check: unknown }>();
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
const gate = deferred<void>();
const { worker, mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer },
mocks: {
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
},
});
// Job is parked on produce(); drain must not flush while it is still in flight.
// Job is parked in its handler; drain must not flush while it is still in flight.
const draining = worker.drain();
await jest.advanceTimersByTimeAsync(DRAIN_POLL_MS * 3);
expect(mocks.bufferService.shutdown).not.toHaveBeenCalled();
// Let the job finish; drain observes inFlight == 0, then flushes.
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
gate.resolve();
await jest.advanceTimersByTimeAsync(DRAIN_POLL_MS);
await draining;
expect(mocks.bufferService.shutdown).toHaveBeenCalledTimes(1);
@@ -824,10 +702,12 @@ describe("DBQueueWorker", () => {
it("times out after DRAIN_TIMEOUT_MS, warns, and still flushes", async () => {
const job = makeJob({ type: "check" });
const gate = deferred<{ status: unknown; check: unknown }>();
const stuckProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
const gate = deferred<void>();
const { worker, mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: stuckProducer },
mocks: {
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
},
});
const draining = worker.drain();
@@ -837,7 +717,7 @@ describe("DBQueueWorker", () => {
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("timed out") }));
expect(mocks.bufferService.shutdown).toHaveBeenCalledTimes(1);
gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); // release the stuck job for cleanup
gate.resolve(); // release the stuck job for cleanup
});
});
@@ -892,15 +772,17 @@ describe("DBQueueWorker", () => {
it("counts in-flight jobs while they run and releases on completion", async () => {
const job = makeJob({ type: "check" });
const gate = deferred<{ status: unknown; check: unknown }>();
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
const gate = deferred<void>();
const { worker } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOneCheck(job) }, checkProducer: slowProducer },
mocks: {
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOneCheck(job) },
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
},
});
expect(worker.getHealth().inFlight).toBe(1); // parked on produce()
expect(worker.getHealth().inFlight).toBe(1); // parked in its handler
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
gate.resolve();
await jest.advanceTimersByTimeAsync(1);
expect(worker.getHealth().inFlight).toBe(0);
});
server/test/unit/services/checkPipeline.test.ts → server/test/unit/services/geoChecksPipeline.test.ts
+53 -1
View File
@@ -1,5 +1,6 @@
import { describe, expect, it, jest } from "@jest/globals";
import { GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
import { GeoChecksPipeline } from "../../../src/worker/worker.geo-pipeline.ts";
import type { Job } from "../../../src/domain/jobs/job.type.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
@@ -25,12 +26,14 @@ const activeWindow = () => {
const createGeoPipeline = (overrides?: Record<string, any>) => {
const defaults = {
logger: createMockLogger(),
monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(null) },
geoChecksService: { buildGeoCheck: jest.fn().mockResolvedValue(null) },
buffer: { addGeoCheckToBuffer: jest.fn() },
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
...overrides,
};
const pipeline = new GeoChecksPipeline(
defaults.monitorsRepository as any,
defaults.maintenanceWindowsRepository as any,
defaults.geoChecksService as any,
defaults.buffer as any,
@@ -42,7 +45,56 @@ const createGeoPipeline = (overrides?: Record<string, any>) => {
const geoMonitor = (overrides?: Partial<Monitor>) =>
makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckLocations: ["us-east"], ...overrides });
const makeJob = (overrides?: Partial<Job>): Job => ({
id: "geo-check:m1",
type: "geo-check",
refId: "m1",
isActive: true,
nextScheduledAt: 1000,
intervalMs: 300000,
lockedBy: null,
lockedUntil: null,
pendingChecks: [],
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
...overrides,
});
describe("GeoChecksPipeline", () => {
describe("handle", () => {
it("loads the monitor by refId and runs the pipeline for it", async () => {
const geoCheck = { id: "gc-1", monitorId: "m1" };
const { pipeline, defaults } = createGeoPipeline({
monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(geoMonitor()) },
geoChecksService: { buildGeoCheck: jest.fn().mockResolvedValue(geoCheck) },
});
await pipeline.handle(makeJob());
expect(defaults.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
expect(defaults.buffer.addGeoCheckToBuffer).toHaveBeenCalledWith(geoCheck);
});
it("does nothing for a job without a refId", async () => {
const { pipeline, defaults } = createGeoPipeline();
await pipeline.handle(makeJob({ refId: null }));
expect(defaults.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
expect(defaults.geoChecksService.buildGeoCheck).not.toHaveBeenCalled();
});
it("does nothing when the monitor no longer exists", async () => {
const { pipeline, defaults } = createGeoPipeline();
await pipeline.handle(makeJob());
expect(defaults.geoChecksService.buildGeoCheck).not.toHaveBeenCalled();
});
});
it("throws when monitor id is missing", async () => {
const { pipeline } = createGeoPipeline();
await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id");
server/test/unit/services/incidentService.test.ts
+1 -1
View File
@@ -7,7 +7,7 @@ import type { IUsersRepository } from "../../../src/domain/users/user.repository
import type { INotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { Incident } from "../../../src/domain/incidents/incident.type.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
server/test/unit/services/monitorStatusPolicy.test.ts
-74
View File
@@ -1,74 +0,0 @@
import { describe, expect, it } from "@jest/globals";
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
import type { StatusChangeResult } from "../../../src/types/network.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
const makeStatusChange = (overrides: Partial<StatusChangeResult> & { status?: string }): StatusChangeResult => {
const { status = "up", ...rest } = overrides;
return {
monitor: { id: "m1", status } as any,
statusChanged: false,
prevStatus: "up",
code: 200,
timestamp: 0,
...rest,
} as StatusChangeResult;
};
// ── Tests ────────────────────────────────────────────────────────────────────
describe("MonitorStatusPolicy", () => {
const policy = new MonitorStatusPolicy();
it("does nothing when statusChanged is false", () => {
const decision = policy.evaluate(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "up" }));
expect(decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
});
});
it("creates incident and notifies when monitor goes down", () => {
const decision = policy.evaluate(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 }));
expect(decision).toMatchObject({
shouldCreateIncident: true,
shouldSendNotification: true,
incidentReason: "status_down",
notificationReason: "status_change",
});
});
it("creates incident and notifies when monitor is breached", () => {
const decision = policy.evaluate(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({
shouldCreateIncident: true,
shouldSendNotification: true,
incidentReason: "threshold_breach",
notificationReason: "threshold_breach",
});
});
it("resolves incident when monitor recovers from down", () => {
const decision = policy.evaluate(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "down" }));
expect(decision).toMatchObject({
shouldResolveIncident: true,
shouldSendNotification: true,
shouldCreateIncident: false,
notificationReason: "status_change",
});
});
it("resolves incident when monitor recovers from breached", () => {
const decision = policy.evaluate(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" }));
expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
});
it("does not create or resolve for unhandled status transitions", () => {
const decision = policy.evaluate(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false });
});
});
server/test/unit/services/notificationMessageBuilder.test.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { describe, expect, it, beforeEach } from "@jest/globals";
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
server/test/unit/services/notificationsService.test.ts
+1 -1
View File
@@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { Notification } from "../../../src/domain/notifications/notification.type.ts";
import type { MonitorStatusResponse } from "../../../src/types/network.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
server/test/unit/services/workerPipeline.test.ts
+874
View File
@@ -0,0 +1,874 @@
import { describe, expect, it, jest } from "@jest/globals";
import { WorkerPipeline } from "../../../src/worker/worker.pipeline.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { Job } from "../../../src/domain/jobs/job.type.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { StatusChangeResult } from "../../../src/types/network.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
import { NETWORK_ERROR } from "../../../src/types/network.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "m1",
teamId: "team",
type: "http",
interval: 60000,
status: "up",
...overrides,
}) as Monitor;
const makeJob = (overrides?: Partial<Job>): Job => ({
id: "check:m1",
type: "check",
refId: "m1",
isActive: true,
nextScheduledAt: 1000,
intervalMs: 60000,
lockedBy: null,
lockedUntil: null,
pendingChecks: [],
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
...overrides,
});
// An evaluate row as ingest leaves it: one pending entry per stored check
const makeEvaluateJob = (...checkIds: string[]): Job =>
makeJob({
id: "evaluate:m1",
type: "evaluate",
intervalMs: null,
pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })),
});
const makeCheck = (overrides?: Partial<Check>): Check =>
({
id: "check-1",
metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" },
status: true,
statusCode: 200,
responseTime: 100,
message: "OK",
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
...overrides,
}) as Check;
const makeStatusChange = (overrides: Partial<StatusChangeResult> & { status?: string }): StatusChangeResult => {
const { status = "up", ...rest } = overrides;
return {
monitor: { id: "m1", status } as any,
statusChanged: false,
prevStatus: "up",
code: 200,
timestamp: 0,
...rest,
} as StatusChangeResult;
};
// A maintenance window straddling "now" — active per isWindowActive.
const activeWindow = () => {
const now = Date.now();
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
};
const createPipeline = (overrides?: Record<string, any>) => {
const deps = {
logger: createMockLogger(),
monitorsRepository: {
findByIdLean: jest.fn<any>().mockResolvedValue(makeMonitor()),
updateById: jest.fn<any>().mockResolvedValue({}),
findDockerTlsKeyById: jest.fn<any>().mockResolvedValue(null),
},
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([]) },
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([]) },
jobsRepository: {
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
pullEvaluated: jest.fn<any>().mockResolvedValue(true),
},
checkService: {
toCheck: jest.fn<any>().mockReturnValue({ id: "check-1" }),
toStatusResponse: jest.fn<any>().mockReturnValue({ monitorId: "m1", status: true, code: 200, message: "OK" }),
createChecks: jest.fn<any>().mockResolvedValue([]),
},
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
proxyResolver: { resolve: jest.fn<any>().mockResolvedValue(undefined) },
bufferService: { addToBuffer: jest.fn<any>(), addDockerLogToBuffer: jest.fn<any>() },
dockerLogsService: { buildDockerLogs: jest.fn<any>().mockResolvedValue([]) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue(null) },
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(makeStatusChange({})) },
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
...overrides,
};
const pipeline = new WorkerPipeline(deps as any);
return { pipeline, deps };
};
// ── Stage 1: check job ───────────────────────────────────────────────────────
describe("WorkerPipeline", () => {
describe("handleCheck", () => {
it("loads the monitor by refId and produces a check for it", async () => {
const { pipeline, deps } = createPipeline();
await pipeline.handleCheck(makeJob());
expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
expect(deps.networkService.requestStatus).toHaveBeenCalled();
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith({ id: "check-1" });
});
it("does nothing for a job without a refId", async () => {
const { pipeline, deps } = createPipeline();
await pipeline.handleCheck(makeJob({ refId: null }));
expect(deps.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
expect(deps.networkService.requestStatus).not.toHaveBeenCalled();
});
it("does nothing when the monitor no longer exists", async () => {
const { pipeline, deps } = createPipeline({
monitorsRepository: { findByIdLean: jest.fn<any>().mockResolvedValue(null) },
});
await pipeline.handleCheck(makeJob());
expect(deps.networkService.requestStatus).not.toHaveBeenCalled();
});
});
describe("produce", () => {
it("throws when monitor id is missing", async () => {
const { pipeline } = createPipeline();
await expect(pipeline.produce({} as Monitor)).rejects.toThrow("No monitor id");
});
// ── maintenance gate ──────────────────────────────────────────────────
it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => {
const { pipeline, deps } = createPipeline({
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([activeWindow()]) },
});
const result = await pipeline.produce(makeMonitor({ status: "up" }));
expect(result).toBeNull();
expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
expect(deps.networkService.requestStatus).not.toHaveBeenCalled();
expect(deps.bufferService.addToBuffer).not.toHaveBeenCalled();
});
it("does not re-write status when already in maintenance", async () => {
const { pipeline, deps } = createPipeline({
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([activeWindow()]) },
});
const result = await pipeline.produce(makeMonitor({ status: "maintenance" }));
expect(result).toBeNull();
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
const monitor = makeMonitor({ status: "maintenance" });
const { pipeline, deps } = createPipeline({
monitorsRepository: {
updateById: jest.fn<any>().mockResolvedValue({ ...monitor, status: "initializing" }),
findDockerTlsKeyById: jest.fn<any>().mockResolvedValue(null),
},
});
await pipeline.produce(monitor);
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
expect(deps.networkService.requestStatus).toHaveBeenCalled();
});
it("does not touch the status of a monitor that was not in maintenance", async () => {
const { pipeline, deps } = createPipeline();
await pipeline.produce(makeMonitor({ status: "up" }));
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("proceeds with the check when the maintenance window is inactive", async () => {
const { pipeline, deps } = createPipeline({
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([{ ...activeWindow(), active: false }]) },
});
await pipeline.produce(makeMonitor());
expect(deps.networkService.requestStatus).toHaveBeenCalled();
});
// ── proxy resolution ──────────────────────────────────────────────────
it("passes the resolved proxy url into requestStatus", async () => {
const { pipeline, deps } = createPipeline({
proxyResolver: { resolve: jest.fn<any>().mockResolvedValue("http://proxy.example.com:8080") },
});
const monitor = makeMonitor();
await pipeline.produce(monitor);
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: "http://proxy.example.com:8080" });
});
it("still produces a check when the resolver returns undefined", async () => {
const { pipeline, deps } = createPipeline();
const monitor = makeMonitor();
const result = await pipeline.produce(monitor);
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined });
expect(result).toEqual({ status: expect.objectContaining({ monitorId: "m1" }), check: { id: "check-1" } });
});
// ── docker tls key resolution ─────────────────────────────────────────
it("fetches the stored key for a docker monitor with a key set and passes it in the context", async () => {
const { pipeline, deps } = createPipeline({
monitorsRepository: { updateById: jest.fn<any>(), findDockerTlsKeyById: jest.fn<any>().mockResolvedValue("v1.abc123.iv.tag.data") },
});
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
await pipeline.produce(monitor);
expect(deps.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: "v1.abc123.iv.tag.data" });
});
it("does not fetch a key for a docker monitor without one set", async () => {
const { pipeline, deps } = createPipeline();
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKeySet: false });
await pipeline.produce(monitor);
expect(deps.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
});
it("does not fetch a key for non-docker monitors even if the flag is set", async () => {
const { pipeline, deps } = createPipeline();
await pipeline.produce(makeMonitor({ type: "http", dockerTlsKeySet: true }));
expect(deps.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
});
it("passes undefined when the flag is set but the repository has no key", async () => {
const { pipeline, deps } = createPipeline();
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
await pipeline.produce(monitor);
expect(deps.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
});
// ── acquire / record ──────────────────────────────────────────────────
it("throws when the network response is null", async () => {
const { pipeline } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(null) },
});
await expect(pipeline.produce(makeMonitor())).rejects.toThrow("No network response");
});
it("returns null, warns, and does not buffer when toCheck yields nothing", async () => {
const { pipeline, deps } = createPipeline({
checkService: { toCheck: jest.fn<any>().mockReturnValue(null) },
});
const result = await pipeline.produce(makeMonitor());
expect(result).toBeNull();
expect(deps.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
expect(deps.bufferService.addToBuffer).not.toHaveBeenCalled();
});
it("buffers the built check and returns the status and check on success", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
const check = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(status) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
});
const result = await pipeline.produce(makeMonitor());
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
expect(result).toEqual({ status, check });
});
it("builds and buffers docker logs for a docker status", async () => {
const status = { type: "docker", monitorId: "m1", teamId: "team", status: true, code: 200, message: "OK", payload: {} };
const dockerLogs = [{ id: "docker-log-1" }, { id: "docker-log-2" }];
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(status) },
dockerLogsService: { buildDockerLogs: jest.fn<any>().mockResolvedValue(dockerLogs) },
});
await pipeline.produce(makeMonitor({ type: "docker" }));
expect(deps.dockerLogsService.buildDockerLogs).toHaveBeenCalledWith(status);
expect(deps.bufferService.addDockerLogToBuffer).toHaveBeenNthCalledWith(1, dockerLogs[0]);
expect(deps.bufferService.addDockerLogToBuffer).toHaveBeenNthCalledWith(2, dockerLogs[1]);
});
it("does not build docker logs for a non-docker status", async () => {
const { pipeline, deps } = createPipeline();
await pipeline.produce(makeMonitor());
expect(deps.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
});
// ── egress self-check ─────────────────────────────────────────────────
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
});
await pipeline.produce(makeMonitor());
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
});
it("does not consult egress when an HTTP response was received", async () => {
for (const status of [
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
]) {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(status) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(makeMonitor());
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
}
});
it("consults egress for a timeout or network error", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(makeMonitor());
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
});
it("never consults egress for a docker daemon on a local socket", async () => {
for (const url of ["unix:///var/run/docker.sock", "/var/run/docker.sock"]) {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, type: "docker" }) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(makeMonitor({ type: "docker", url }));
// Local IPC never leaves the instance, so it cannot fail through egress.
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
}
});
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
for (const monitor of [
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
]) {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(monitor);
// Both requests leave the instance, so our own loss of egress can be the cause.
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
}
});
it("does not consult egress when the provider reports that the peer answered", async () => {
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
// them from a connection failure.
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(makeMonitor({ type: "dns" }));
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
});
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(makeMonitor({ type: "grpc" }));
// An explicit report from the provider outranks the HTTP-status heuristic.
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
});
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
const result = await pipeline.produce(makeMonitor());
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
expect(result?.check.egressStatus).toBe("degraded");
});
// ── status window across a spell ─────────────────────────────────────
// Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either
// side of a spell are never adjacent.
it("clears the status window on the first degraded check of a spell", async () => {
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] });
});
it("does not re-write the status window for later degraded checks in the same spell", async () => {
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
});
await pipeline.produce(makeMonitor({ statusWindow: [] }));
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("leaves the status window alone when the probe found egress fine", async () => {
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("ok") },
});
await pipeline.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("flags a failing check as ok when the probe found egress fine", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("ok") },
});
await pipeline.produce(makeMonitor());
expect(check.egressStatus).toBe("ok");
});
it("buffers the check when the egress assessment rejects", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockRejectedValue(new Error("settings unavailable")) },
});
const result = await pipeline.produce(makeMonitor());
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
expect(check).not.toHaveProperty("egressStatus");
expect(result).toEqual({ status: failingStatus, check });
expect(deps.logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "WorkerPipeline", method: "assessEgress" })
);
});
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
});
await pipeline.produce(makeMonitor());
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check).not.toHaveProperty("egressStatus");
});
});
// ── Stage 2: ingestion ───────────────────────────────────────────────────
describe("ingestChecks", () => {
it("stores the batch, then arms the evaluate stage once per distinct monitor", async () => {
const { pipeline, deps } = createPipeline();
const batch = [
makeCheck({ id: "c1", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }),
makeCheck({ id: "c2", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }),
makeCheck({ id: "c3", metadata: { monitorId: "mon-2", teamId: "team-1", type: "http" } }),
];
await pipeline.ingestChecks(batch);
const createdAt = Date.parse("2026-01-01T00:00:00.000Z");
expect(deps.checkService.createChecks).toHaveBeenCalledWith(batch);
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledWith(
"mon-1",
[
{ checkId: "c1", createdAt },
{ checkId: "c2", createdAt },
],
expect.any(Number)
);
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
});
it("retries a failed arm on the next ingest without re-storing the checks", async () => {
const { pipeline, deps } = createPipeline();
deps.jobsRepository.upsertEvaluate.mockRejectedValueOnce(new Error("arm failed"));
await pipeline.ingestChecks([makeCheck({ id: "c1" })]);
expect(deps.checkService.createChecks).toHaveBeenCalledTimes(1);
expect(deps.logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
await pipeline.ingestChecks([]); // nothing new, only the retry runs
expect(deps.checkService.createChecks).toHaveBeenCalledTimes(1);
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith(
"mon-1",
[{ checkId: "c1", createdAt: expect.any(Number) }],
expect.any(Number)
);
await pipeline.ingestChecks([]); // nothing left to retry
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
});
it("neither writes nor arms for an empty batch", async () => {
const { pipeline, deps } = createPipeline();
await pipeline.ingestChecks([]);
expect(deps.checkService.createChecks).not.toHaveBeenCalled();
expect(deps.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
});
it("does not arm the evaluate stage when the check write fails", async () => {
const { pipeline, deps } = createPipeline();
deps.checkService.createChecks.mockRejectedValue(new Error("DB write failed"));
await expect(pipeline.ingestChecks([makeCheck()])).rejects.toThrow("DB write failed");
expect(deps.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
});
});
// ── Stage 3: evaluation ──────────────────────────────────────────────────
describe("handleEvaluate", () => {
it("loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
const job = makeEvaluateJob("c1");
const { pipeline, deps } = createPipeline({
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) },
});
await pipeline.handleEvaluate(job);
expect(deps.checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(1);
expect(deps.dispatcher.dispatch).toHaveBeenCalledTimes(1);
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
// The monitor cursor is gone: nothing writes lastEvaluatedAt
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("does nothing when the row has no pending checks", async () => {
const { pipeline, deps } = createPipeline();
await pipeline.handleEvaluate(makeEvaluateJob());
expect(deps.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
expect(deps.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled();
});
it("drops every pending id for a deleted monitor", async () => {
const job = makeEvaluateJob("c1", "c2");
const { pipeline, deps } = createPipeline({
monitorsRepository: { findByIdLean: jest.fn<any>().mockResolvedValue(null) },
});
await pipeline.handleEvaluate(job);
expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled();
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
});
it("stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
const job = makeEvaluateJob("c1", "c2", "c3");
const { pipeline, deps } = createPipeline({
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) },
jobsRepository: {
upsertEvaluate: jest.fn<any>(),
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
},
});
await pipeline.handleEvaluate(job);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2);
expect(deps.dispatcher.dispatch).toHaveBeenCalledTimes(2);
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
expect(deps.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Lost lease") }));
});
it("pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
const job = makeEvaluateJob("c1", "c-gone");
const { pipeline, deps } = createPipeline({
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }, // c-gone was cleaned up
});
await pipeline.handleEvaluate(job);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(1);
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]);
});
it("reads the monitor once and threads the post-write monitor across the backlog", async () => {
const job = makeEvaluateJob("c1", "c2");
const initialMonitor = makeMonitor();
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
const { pipeline, deps } = createPipeline({
monitorsRepository: { findByIdLean: jest.fn<any>().mockResolvedValue(initialMonitor), updateById: jest.fn<any>() },
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) },
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(makeStatusChange({ monitor: postWriteMonitor })) },
});
await pipeline.handleEvaluate(job);
// The monitor is read once for the whole backlog, not once per check
expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2);
// First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first
expect(deps.statusService.updateMonitorStatus.mock.calls[0][2]).toBe(initialMonitor);
expect(deps.statusService.updateMonitorStatus.mock.calls[1][2]).toBe(postWriteMonitor);
});
});
describe("evaluateCheck", () => {
it("threads status, check, and monitor through to the status service", async () => {
const { pipeline, deps } = createPipeline();
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const check = { id: "check-1" } as any;
const monitor = makeMonitor({ status: "up" });
await pipeline.evaluateCheck(status, check, monitor);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor);
});
it("returns an evaluation that opens an incident on a down transition", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1" } as any;
const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 });
const { pipeline } = createPipeline({
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
});
const result = await pipeline.evaluateCheck(status, check, makeMonitor());
expect(result).toMatchObject({
monitor: statusChange.monitor,
status,
check,
statusChange,
decision: expect.objectContaining({
shouldCreateIncident: true,
shouldSendNotification: true,
incidentReason: "status_down",
}),
});
});
it("returns an evaluation that resolves an incident on recovery", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down" });
const { pipeline } = createPipeline({
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
});
const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor({ status: "down" }));
expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
});
it("produces a no-op decision when status did not change", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const { pipeline } = createPipeline();
const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor());
expect(result.decision).toMatchObject({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
});
});
// ── degraded egress ───────────────────────────────────────────────────
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "degraded" } as any;
const monitor = makeMonitor({ status: "up" });
const { pipeline, deps } = createPipeline();
const result = await pipeline.evaluateCheck(status, check, monitor);
expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled();
expect(result.monitor).toBe(monitor);
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
expect(result.decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
});
expect(deps.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
});
it("evaluates a check flagged with egress ok as normal", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "ok" } as any;
const { pipeline, deps } = createPipeline();
await pipeline.evaluateCheck(status, check, makeMonitor());
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
});
});
// The decision table, driven through evaluateCheck with the status service stubbed to return each transition.
describe("decision", () => {
const decide = async (statusChange: StatusChangeResult) => {
const { pipeline } = createPipeline({
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
});
const result = await pipeline.evaluateCheck(
{ monitorId: "m1", status: true, code: 200, message: "OK" } as any,
{ id: "check-1" } as any,
makeMonitor()
);
return result.decision;
};
it("does nothing when statusChanged is false", async () => {
const decision = await decide(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "up" }));
expect(decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
});
});
it("creates incident and notifies when monitor goes down", async () => {
const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 }));
expect(decision).toMatchObject({
shouldCreateIncident: true,
shouldSendNotification: true,
incidentReason: "status_down",
notificationReason: "status_change",
});
});
it("creates incident and notifies when monitor is breached", async () => {
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({
shouldCreateIncident: true,
shouldSendNotification: true,
incidentReason: "threshold_breach",
notificationReason: "threshold_breach",
});
});
it("resolves incident when monitor recovers from down", async () => {
const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "down" }));
expect(decision).toMatchObject({
shouldResolveIncident: true,
shouldSendNotification: true,
shouldCreateIncident: false,
notificationReason: "status_change",
});
});
it("resolves incident when monitor recovers from breached", async () => {
const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" }));
expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
});
it("does not create or resolve for unhandled status transitions", async () => {
const decision = await decide(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false });
});
});
});