Merge pull request #3733 from bluewave-labs/fix/excess-polling

Fix/excess polling
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-06-22 09:26:36 -07:00
commit 09446c9fc1
19 files changed
+570 -438

No files matched your search

+4 -8
View File
@@ -103,7 +103,7 @@ import { ILogger } from "@/utils/logger.js";
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
import { CheckProducer } from "@/worker/worker.check-producer.js";
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
@@ -229,7 +229,7 @@ export const initializeServices = async ({
const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository);
const statusService = new StatusService(logger, bufferService, monitorsRepository, monitorStatsRepository, checksRepository);
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
// Notification providers
const webhookProvider = new WebhookProvider(logger);
@@ -276,8 +276,6 @@ export const initializeServices = async ({
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
// pipelines
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
const workerHelper = new WorkerHelper(
@@ -290,10 +288,7 @@ export const initializeServices = async ({
monitorStatsRepository,
checksRepository,
incidentsRepository,
geoChecksRepository,
reactorDispatcher,
checkPipeline,
geoCheckPipeline
geoChecksRepository
);
const worker = await DBQueueWorker.create(
@@ -339,6 +334,7 @@ export const initializeServices = async ({
monitorsRepository,
maintenanceWindowsRepository,
jobsRepository,
worker,
});
const monitorService = new MonitorService({
worker,
@@ -1,6 +1,7 @@
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { IWorker } from "@/worker/worker.interface.js";
import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js";
import { AppError } from "@/utils/AppError.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
@@ -42,19 +43,23 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
private monitorsRepository: IMonitorsRepository;
private maintenanceWindowsRepository: IMaintenanceWindowsRepository;
private jobsRepository: IJobsRepository;
private worker: IWorker;
constructor({
monitorsRepository,
maintenanceWindowsRepository,
jobsRepository,
worker,
}: {
monitorsRepository: IMonitorsRepository;
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
jobsRepository: IJobsRepository;
worker: IWorker;
}) {
this.monitorsRepository = monitorsRepository;
this.maintenanceWindowsRepository = maintenanceWindowsRepository;
this.jobsRepository = jobsRepository;
this.worker = worker;
}
get serviceName() {
@@ -78,6 +83,9 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]);
// Rearm jobs to run soon (now + jitter to avoid herding)
await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime());
// Wake the (possibly idle, backed-off) loops so the rearmed jobs run promptly instead of waiting up to POLL_MAX_MS
this.worker.wake("check");
this.worker.wake("geo-check");
}
};
@@ -22,6 +22,8 @@ export interface IMonitorsRepository {
createMonitors(monitors: Monitor[]): Promise<Monitor[]>;
// single fetch
findById(monitorId: string, teamId: string): Promise<Monitor>;
// for workers, don't need all details
findByIdLean(monitorId: string): Promise<Monitor | null>;
// collection fetch
findAll(): Promise<Monitor[] | null>;
@@ -38,6 +38,11 @@ class MongoMonitorsRepository implements IMonitorsRepository {
return this.toEntity(monitor);
};
findByIdLean = async (monitorId: string): Promise<Monitor | null> => {
const monitor = await MonitorModel.findOne({ _id: monitorId });
return monitor ? this.toEntity(monitor) : null;
};
findAll = async (): Promise<Monitor[]> => {
const monitors = await MonitorModel.find();
return this.mapDocuments(monitors);
+7 -18
View File
@@ -1,4 +1,3 @@
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import type { Check, CheckDiskInfo, CheckSnapshot } from "@/domain/checks/check.type.js";
@@ -17,7 +16,6 @@ import type {
} from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import { IBufferService } from "./bufferService.js";
import type { HardwareStatusMetrics } from "@/types/network.js";
import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.types.js";
@@ -42,30 +40,21 @@ export interface IStatusService {
| GrpcStatusPayload
| undefined
>,
check: Check
check: Check,
monitor: Monitor
): Promise<StatusChangeResult>;
}
export class StatusService implements IStatusService {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private buffer: IBufferService;
private monitorsRepository: IMonitorsRepository;
private monitorStatsRepository: IMonitorStatsRepository;
private checksRepository: IChecksRepository;
constructor(
logger: ILogger,
buffer: IBufferService,
monitorsRepository: IMonitorsRepository,
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository
) {
constructor(logger: ILogger, monitorsRepository: IMonitorsRepository, monitorStatsRepository: IMonitorStatsRepository) {
this.logger = logger;
this.buffer = buffer;
this.monitorsRepository = monitorsRepository;
this.monitorStatsRepository = monitorStatsRepository;
this.checksRepository = checksRepository;
}
get serviceName() {
@@ -212,11 +201,11 @@ export class StatusService implements IStatusService {
| GrpcStatusPayload
| undefined
>,
check: Check
check: Check,
monitor: Monitor
): Promise<StatusChangeResult> => {
try {
const { monitorId, teamId, status, code } = statusResponse;
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
const { status, code } = statusResponse;
// Update running stats
await this.tryUpdateRunningStats(monitor, statusResponse);
@@ -262,7 +251,7 @@ export class StatusService implements IStatusService {
};
}
// First evaluate reachability-based status changes, which apply to all monitor types
// First evaluate reachability status changes, which apply to all monitor types
// and take precedence over hardware breaches.
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
if (reachabilityResult.transitioned) {
+4 -3
View File
@@ -1,11 +1,12 @@
import { MonitorStatusResponse } from "@/types/network.js";
import { MonitorEvaluation } from "@/worker/worker.interface.js";
import { Check } from "@/domain/checks/check.type.js";
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IStatusService } from "@/service/statusService.js";
export interface ICheckEvaluator {
evaluate(status: MonitorStatusResponse, check: Check): Promise<MonitorEvaluation>;
evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
}
export class CheckEvaluator implements ICheckEvaluator {
@@ -13,11 +14,11 @@ export class CheckEvaluator implements ICheckEvaluator {
private statusService: IStatusService,
private monitorStatusPolicy: IMonitorStatusPolicy
) {}
evaluate = async (status: MonitorStatusResponse, check: Check) => {
evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
// ****************************
// Step 3: Evaluate and return result to reactors
// ****************************
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor);
// Step 5. Get decisions and create an evaluation obj
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
@@ -7,8 +7,6 @@ import { IBufferService } from "@/service/bufferService.js";
import { AppError } from "@/utils/AppError.js";
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
import { ICheckProducer } from "@/worker/worker.check-producer.js";
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
const SERVICE_NAME = "CheckPipeline";
@@ -16,22 +14,6 @@ export interface ICheckPipeline {
run(monitor: Monitor): Promise<MonitorEvaluation | null>; // null = skipped
}
export class CheckPipeline implements ICheckPipeline {
constructor(
private checkProducer: ICheckProducer,
private checkEvaluator: ICheckEvaluator
) {}
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
const checkResult = await this.checkProducer.produce(monitor);
if (!checkResult) {
return null; // skipped due to maintenance / geo guards / other guards
}
const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check);
return evaluation;
};
}
export class GeoChecksPipeline implements ICheckPipeline {
constructor(
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
+50 -9
View File
@@ -15,7 +15,8 @@ import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.rep
import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
const SERVICE_NAME = "JobQueue";
const POLL_MS = 250;
const POLL_MS = 250; // base poll interval while a loop is actively claiming work
const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
@@ -40,6 +41,17 @@ export class DBQueueWorker implements IWorker {
"cleanup-retention": 0,
};
// Per job type poll intervals. These back off on idle and reset on work or wake()
private pollMs: Record<JobType, number> = {
check: POLL_MS,
"geo-check": POLL_MS,
evaluate: POLL_MS,
"cleanup-orphaned": POLL_MS,
"cleanup-retention": POLL_MS,
};
private ticking: Partial<Record<JobType, boolean>> = {}; // Whether a loop is mid tick
private tickFns = new Map<JobType, () => void>(); // Store tick fns so we can run them immediately on wake
constructor(
private logger: ILogger,
private jobsRepository: IJobsRepository,
@@ -150,7 +162,7 @@ export class DBQueueWorker implements IWorker {
private runCheck = async (job: Job) => {
if (!job.refId) return;
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
if (!monitor) return;
await this.checkProducer.produce(monitor);
};
@@ -161,19 +173,22 @@ export class DBQueueWorker implements IWorker {
private runEvaluate = async (job: Job) => {
if (!job.refId) return;
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
if (!monitor) return;
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
let current = monitor;
for (const check of checks) {
const status = this.checkService.toStatusResponse(check);
const evaluation = await this.checkEvaluator.evaluate(status, check);
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
}
};
private runGeoCheck = async (job: Job) => {
const [monitor] = await this.monitorsRepository.findByIds([job.refId!]);
const monitor = await this.monitorsRepository.findByIdLean(job.refId!);
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
};
@@ -235,8 +250,10 @@ export class DBQueueWorker implements IWorker {
};
private startLoop = (type: JobType) => {
this.pollMs[type] = POLL_MS;
const tick = async () => {
if (this.stopped) return;
this.ticking[type] = true;
try {
// Claim a batch sized to the free capacity
const capacity = CONCURRENCY[type] - this.inFlight[type];
@@ -247,6 +264,8 @@ export class DBQueueWorker implements IWorker {
this.inFlight[type]--; // job done, free the slot
});
}
// Back off if no jobs found, reset to base if jobs found
this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS;
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : String(error),
@@ -254,11 +273,26 @@ export class DBQueueWorker implements IWorker {
method: `loop:${type}`,
});
}
if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down
// Clearing the flag and re-arming run synchronously (no await between), so wake() can never interleave here.
this.ticking[type] = false;
if (!this.stopped) this.timers.set(type, setTimeout(tick, this.pollMs[type])); // re-arm unless shutting down
};
this.tickFns.set(type, tick);
tick();
};
// Wake a worker loop after adding a job that's due immediately.
// Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off
wake = (type: JobType) => {
this.pollMs[type] = POLL_MS;
if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing
const tick = this.tickFns.get(type);
if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done
const pending = this.timers.get(type);
if (pending) clearTimeout(pending); // Clear out scheduled tick if any
this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick
};
// Seed queue
private reconcile = async () => {
const now = Date.now();
@@ -308,7 +342,11 @@ export class DBQueueWorker implements IWorker {
addJob = async (_monitorId: string, monitor: Monitor) => {
const now = Date.now();
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true));
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true));
this.wake("check");
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true));
this.wake("geo-check");
}
};
deleteJob = async (monitor: Monitor) => {
await this.jobsRepository.deleteById(monitor.id);
@@ -317,12 +355,15 @@ export class DBQueueWorker implements IWorker {
await this.jobsRepository.setActiveById(monitor.id, false);
};
resumeJob = async (monitor: Monitor) => {
await this.jobsRepository.setActiveById(monitor.id, true);
await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately
this.wake("check");
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check");
};
updateJob = async (monitor: Monitor) => {
await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval);
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away
this.wake("geo-check");
} else {
await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check");
}
+1 -48
View File
@@ -1,5 +1,4 @@
const SERVICE_NAME = "JobQueueHelper";
import type { Monitor } from "@/domain/monitors/monitor.types.js";
import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
@@ -11,13 +10,9 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
export interface IWorkerHelper {
readonly serviceName: string;
getHeartbeatJob(): (monitor: Monitor) => Promise<void>;
getHeartbeatGeoJob(): (monitor: Monitor) => Promise<void>;
getCleanupOrphanedJob(): () => Promise<void>;
getCleanupRetentionJob(): () => Promise<void>;
}
@@ -49,9 +44,6 @@ export class WorkerHelper implements IWorkerHelper {
private checksRepository: IChecksRepository;
private incidentsRepository: IIncidentsRepository;
private geoChecksRepository: IGeoChecksRepository;
private reactorDispatcher: IReactorDispatcher;
private checkPipeline: ICheckPipeline;
private geoCheckPipeline: ICheckPipeline;
constructor(
logger: ILogger,
@@ -63,10 +55,7 @@ export class WorkerHelper implements IWorkerHelper {
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository,
incidentsRepository: IIncidentsRepository,
geoChecksRepository: IGeoChecksRepository,
reactorDispatcher: IReactorDispatcher,
checkPipeline: ICheckPipeline,
geoCheckPipeline: ICheckPipeline
geoChecksRepository: IGeoChecksRepository
) {
this.logger = logger;
this.checkService = checkService;
@@ -78,48 +67,12 @@ export class WorkerHelper implements IWorkerHelper {
this.checksRepository = checksRepository;
this.incidentsRepository = incidentsRepository;
this.geoChecksRepository = geoChecksRepository;
this.reactorDispatcher = reactorDispatcher;
this.checkPipeline = checkPipeline;
this.geoCheckPipeline = geoCheckPipeline;
}
get serviceName() {
return WorkerHelper.SERVICE_NAME;
}
getHeartbeatJob = () => {
return async (monitor: Monitor) => {
try {
const evaluation = await this.checkPipeline.run(monitor);
if (evaluation) await this.reactorDispatcher.dispatch(evaluation);
} catch (error: unknown) {
this.logger.warn({
message: error instanceof Error ? error.message : "Unknown error",
service: SERVICE_NAME,
method: "getMonitorJob",
stack: error instanceof Error ? error.stack : undefined,
});
throw error;
}
};
};
getHeartbeatGeoJob = () => {
return async (monitor: Monitor) => {
try {
await this.geoCheckPipeline.run(monitor);
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : "Unknown error",
service: SERVICE_NAME,
method: "getHeartbeatGeoJob",
stack: error instanceof Error ? error.stack : undefined,
});
// Don't throw - geo check failures shouldn't crash the job scheduler
}
};
};
getCleanupOrphanedJob = () => {
return async () => {
try {
+2
View File
@@ -1,5 +1,6 @@
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { Check } from "@/domain/checks/check.type.js";
import { JobType } from "@/domain/jobs/job.type.js";
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
import { MonitorActionDecision } from "@/worker/worker.helper.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
@@ -70,6 +71,7 @@ export interface IWorker {
pauseJob(monitor: Monitor): Promise<void>;
resumeJob(monitor: Monitor): Promise<void>;
updateJob(monitor: Monitor): Promise<void>;
wake(type: JobType): void;
shutdown(): Promise<void>;
getMetrics(): Promise<WorkerMetrics>;
getJobs(pagination: WorkerJobsPagination): Promise<WorkerJobsPage>;
@@ -40,6 +40,11 @@ export class InMemoryMonitorsRepository implements IMonitorsRepository {
return this.monitors.filter((m) => monitorIds.includes(m.id)).map((m) => ({ ...m }));
}
async findByIdLean(monitorId: string): Promise<Monitor | null> {
const monitor = this.monitors.find((m) => m.id === monitorId);
return monitor ? { ...monitor } : null;
}
async findByIdsWithChecks(monitorIds: string[], _checksCount?: number): Promise<Monitor[]> {
return this.findByIds(monitorIds);
}
+12 -42
View File
@@ -1,7 +1,5 @@
import { jest } from "@jest/globals";
import { WorkerHelper } from "../../src/worker/worker.helper.ts";
import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts";
import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts";
import { CheckProducer } from "../../src/worker/worker.check-producer.ts";
import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts";
import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts";
@@ -71,20 +69,6 @@ const createStubMonitorStatsRepo = () => ({
deleteByMonitorIdsNotIn: jest.fn(),
});
const createStubChecksRepo = () => ({
create: jest.fn(),
createChecks: jest.fn(),
findByMonitorId: jest.fn(),
findByTeamId: jest.fn(),
findLatestByMonitorIds: jest.fn(),
findByDateRangeAndMonitorId: jest.fn(),
findSummaryByTeamId: jest.fn(),
deleteByMonitorId: jest.fn(),
deleteByTeamId: jest.fn(),
deleteByMonitorIdsNotIn: jest.fn(),
deleteOlderThan: jest.fn(),
});
export interface HeartbeatTestHarness {
monitorsRepo: InMemoryMonitorsRepository;
incidentsRepo: InMemoryIncidentsRepository;
@@ -108,13 +92,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
const logger = createMockLogger() as any;
const bufferStub = { addToBuffer: jest.fn(), addGeoCheckToBuffer: jest.fn(), scheduleNextFlush: jest.fn() };
const statusService = new StatusService(
logger,
bufferStub as any,
monitorsRepo as any,
createStubMonitorStatsRepo() as any,
createStubChecksRepo() as any
);
const statusService = new StatusService(logger, monitorsRepo as any, createStubMonitorStatsRepo() as any);
const messageBuilder = { extractThresholdBreaches: jest.fn().mockReturnValue([]) };
const incidentService = new IncidentService(logger, incidentsRepo, monitorsRepo as any, { findById: jest.fn() } as any, messageBuilder as any);
@@ -149,7 +127,6 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
};
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
const geoChecksService = { buildGeoCheck: jest.fn() };
const checkProducer = new CheckProducer(
monitorsRepo as any,
@@ -160,28 +137,21 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
logger
);
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger);
const notificationReactor = new NotificationReactor(notificationsService as any);
const incidentReactor = new IncidentReactor(incidentService as any);
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
const helper = new WorkerHelper(
logger,
checkService as any,
{ getDBSettings: jest.fn() } as any,
monitorsRepo as any,
{ deleteByMonitorIdsNotIn: jest.fn() } as any,
{ findAllTeamIds: jest.fn() } as any,
createStubMonitorStatsRepo() as any,
createStubChecksRepo() as any,
incidentsRepo as any,
{ deleteByMonitorIdsNotIn: jest.fn() } as any,
reactorDispatcher,
checkPipeline,
geoCheckPipeline
);
// Mirror the production check→evaluate flow: read the monitor fresh each cycle (so the
// accumulated statusWindow is visible), produce a check, then evaluate against that monitor.
const heartbeatJob = async (seedMonitor: Monitor) => {
const monitor = await monitorsRepo.findByIdLean(seedMonitor.id);
if (!monitor) return;
const result = await checkProducer.produce(monitor);
if (!result) return; // skipped (e.g. maintenance window)
const evaluation = await checkEvaluator.evaluate(result.status, result.check, monitor);
await reactorDispatcher.dispatch(evaluation);
};
return {
monitorsRepo,
@@ -193,7 +163,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
bufferStub,
maintenanceWindowsRepo,
messageBuilder,
heartbeatJob: helper.getHeartbeatJob(),
heartbeatJob,
setNextResponse,
setNextResponseFull,
};
@@ -0,0 +1,87 @@
import { describe, expect, it, jest } from "@jest/globals";
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "m1",
teamId: "team",
type: "http",
interval: 60000,
status: "up",
...overrides,
}) as Monitor;
const createEvaluator = (overrides?: Record<string, any>) => {
const defaults = {
statusService: {
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
},
monitorStatusPolicy: new MonitorStatusPolicy(),
...overrides,
};
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
return { evaluator, defaults };
};
describe("CheckEvaluator", () => {
it("threads status, check, and monitor through to the status service", async () => {
const { evaluator, defaults } = createEvaluator();
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const check = { id: "check-1" } as any;
const monitor = makeMonitor({ status: "up" });
await evaluator.evaluate(status, check, monitor);
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor);
});
it("returns an evaluation that opens an incident on a down transition", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1" } as any;
const statusChange = { monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 };
const { evaluator } = createEvaluator({
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
});
const result = await evaluator.evaluate(status, check, makeMonitor());
expect(result).toMatchObject({
monitor: statusChange.monitor,
status,
check,
statusChange,
decision: expect.objectContaining({
shouldCreateIncident: true,
shouldSendNotification: true,
incidentReason: "status_down",
}),
});
});
it("returns an evaluation that resolves an incident on recovery", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const statusChange = { monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down", code: 200 };
const { evaluator } = createEvaluator({
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
});
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor({ status: "down" }));
expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
});
it("produces a no-op decision when status did not change", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const { evaluator } = createEvaluator();
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor());
expect(result.decision).toMatchObject({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
});
});
});
+1 -112
View File
@@ -1,8 +1,5 @@
import { describe, expect, it, jest } from "@jest/globals";
import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
import { GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
@@ -23,114 +20,6 @@ const activeWindow = () => {
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
};
// ── CheckPipeline ────────────────────────────────────────────────────────────
const createCheckPipeline = (overrides?: Record<string, any>) => {
const defaults = {
logger: createMockLogger(),
networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
statusService: {
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }),
},
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
buffer: { addToBuffer: jest.fn() },
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) },
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
monitorStatusPolicy: new MonitorStatusPolicy(),
...overrides,
};
const checkProducer = new CheckProducer(
defaults.monitorsRepository as any,
defaults.maintenanceWindowsRepository as any,
defaults.checkService as any,
defaults.networkService as any,
defaults.buffer as any,
defaults.logger as any
);
const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
const pipeline = new CheckPipeline(checkProducer, checkEvaluator);
return { pipeline, defaults };
};
describe("CheckPipeline", () => {
it("throws when monitor id is missing", async () => {
const { pipeline } = createCheckPipeline();
await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id");
});
it("skips and sets maintenance status when in maintenance window", async () => {
const { pipeline, defaults } = createCheckPipeline({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
});
const result = await pipeline.run(makeMonitor({ status: "up" }));
expect(result).toBeNull();
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
});
it("does not re-write status when already in maintenance", async () => {
const { pipeline, defaults } = createCheckPipeline({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
});
await pipeline.run(makeMonitor({ status: "maintenance" }));
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("throws when network response is null", async () => {
const { pipeline } = createCheckPipeline({
networkService: { requestStatus: jest.fn().mockResolvedValue(null) },
});
await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response");
});
it("returns null and warns when toCheck returns null", async () => {
const { pipeline, defaults } = createCheckPipeline({
checkService: { toCheck: jest.fn().mockReturnValue(null) },
});
const result = await pipeline.run(makeMonitor());
expect(result).toBeNull();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
});
it("buffers the check, updates status, and returns an evaluation", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
const statusChange = { monitor: { id: "m1", status: "down" }, statusChanged: true, prevStatus: "up", code: 500 };
const { pipeline, defaults } = createCheckPipeline({
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
});
const result = await pipeline.run(makeMonitor());
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith({ id: "check-1" });
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalled();
expect(result).toMatchObject({
monitor: statusChange.monitor,
status,
check: { id: "check-1" },
statusChange,
decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, incidentReason: "status_down" }),
});
});
it("proceeds with the check when no maintenance window is active", async () => {
const { pipeline, defaults } = createCheckPipeline({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
});
await pipeline.run(makeMonitor());
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
});
});
// ── GeoChecksPipeline ──────────────────────────────────────────────────────────
const createGeoPipeline = (overrides?: Record<string, any>) => {
@@ -0,0 +1,120 @@
import { describe, expect, it, jest } from "@jest/globals";
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "m1",
teamId: "team",
type: "http",
interval: 60000,
status: "up",
...overrides,
}) as Monitor;
// A maintenance window straddling "now" — active per isWindowActive.
const activeWindow = () => {
const now = Date.now();
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
};
const createProducer = (overrides?: Record<string, any>) => {
const defaults = {
logger: createMockLogger(),
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) },
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
buffer: { addToBuffer: jest.fn() },
...overrides,
};
const producer = new CheckProducer(
defaults.monitorsRepository as any,
defaults.maintenanceWindowsRepository as any,
defaults.checkService as any,
defaults.networkService as any,
defaults.buffer as any,
defaults.logger as any
);
return { producer, defaults };
};
describe("CheckProducer", () => {
it("throws when monitor id is missing", async () => {
const { producer } = createProducer();
await expect(producer.produce({} as Monitor)).rejects.toThrow("No monitor id");
});
// ── maintenance gate ──────────────────────────────────────────────────────
it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => {
const { producer, defaults } = createProducer({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
});
const result = await producer.produce(makeMonitor({ status: "up" }));
expect(result).toBeNull();
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
});
it("does not re-write status when already in maintenance", async () => {
const { producer, defaults } = createProducer({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
});
const result = await producer.produce(makeMonitor({ status: "maintenance" }));
expect(result).toBeNull();
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("proceeds with the check when the maintenance window is inactive", async () => {
const { producer, defaults } = createProducer({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
});
await producer.produce(makeMonitor());
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
});
// ── acquire / record ──────────────────────────────────────────────────────
it("throws when the network response is null", async () => {
const { producer } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(null) },
});
await expect(producer.produce(makeMonitor())).rejects.toThrow("No network response");
});
it("returns null, warns, and does not buffer when toCheck yields nothing", async () => {
const { producer, defaults } = createProducer({
checkService: { toCheck: jest.fn().mockReturnValue(null) },
});
const result = await producer.produce(makeMonitor());
expect(result).toBeNull();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
});
it("buffers the built check and returns the status and check on success", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
const check = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
const result = await producer.produce(makeMonitor());
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
expect(result).toEqual({ status, check });
});
});
+152 -2
View File
@@ -80,6 +80,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
};
const monitorsRepository = {
findByIds: jest.fn<any>().mockResolvedValue([makeMonitor()]),
findByIdLean: jest.fn<any>().mockResolvedValue(makeMonitor()),
findAll: jest.fn<any>().mockResolvedValue([]),
updateById: jest.fn<any>().mockResolvedValue({}),
};
@@ -94,7 +95,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
produce: jest.fn<any>().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }),
};
const checkEvaluator = {
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), decision: {} }),
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), statusChange: { monitor: makeMonitor() }, decision: {} }),
};
const geoCheckPipeline = { run: jest.fn<any>().mockResolvedValue(null) };
const dispatcher = { dispatch: jest.fn<any>().mockResolvedValue(undefined) };
@@ -268,7 +269,7 @@ describe("DBQueueWorker", () => {
const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]);
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
expect(mocks.checkProducer.produce).toHaveBeenCalled();
// The check stage only produces; the buffer arms evaluate once the check is durably stored
expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
@@ -307,6 +308,35 @@ describe("DBQueueWorker", () => {
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
});
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
const initialMonitor = makeMonitor();
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
const monitorsRepository = {
findByIdLean: jest.fn<any>().mockResolvedValue(initialMonitor),
updateById: jest.fn<any>().mockResolvedValue({}),
};
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) };
const checkEvaluator = {
evaluate: jest.fn<any>().mockResolvedValue({ monitor: postWriteMonitor, statusChange: { monitor: postWriteMonitor }, decision: {} }),
};
const { mocks } = await start({
mocks: {
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
monitorsRepository,
checksRepository,
checkEvaluator,
},
});
// The monitor is read once for the whole backlog, not once per check
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1);
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
// First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first
expect(mocks.checkEvaluator.evaluate.mock.calls[0][2]).toBe(initialMonitor);
expect(mocks.checkEvaluator.evaluate.mock.calls[1][2]).toBe(postWriteMonitor);
});
it("cleanup-orphaned job runs the helper's cleanup function", async () => {
const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 });
const cleanupFn = jest.fn<any>().mockResolvedValue(undefined);
@@ -449,6 +479,126 @@ describe("DBQueueWorker", () => {
});
});
// ── poll backoff + wake() ───────────────────────────────────────────────────
//
// An idle loop grows its poll interval up to POLL_MAX_MS (5s). wake() fast-paths
// it back so a freshly-enqueued "run now" job doesn't wait out that backoff.
describe("poll backoff and wake", () => {
const POLL_MAX_MS = 5000;
const claims = (mocks: any, type: string) => mocks.jobsRepository.claimDueBatch.mock.calls.filter((c: any[]) => c[0] === type).length;
it("backs off the idle poll interval toward POLL_MAX_MS", async () => {
const { mocks } = await start({ queueMode: "worker" }); // claimDueBatch returns [] → always idle
await jest.advanceTimersByTimeAsync(15000); // let the check loop grow to the cap
const before = claims(mocks, "check");
await jest.advanceTimersByTimeAsync(POLL_MAX_MS * 2); // ~2 polls at a 5s cadence
const delta = claims(mocks, "check") - before;
// At the cap that's ~2 polls; at the un-backed-off 250ms cadence it would be ~40.
expect(delta).toBeGreaterThanOrEqual(1);
expect(delta).toBeLessThanOrEqual(4);
});
it("resets the poll interval to POLL_MS once a tick finds work after being idle", async () => {
let busy = false;
const claimDueBatch = jest.fn<any>(async (type: string) => (busy && type === "check" ? [makeJob()] : []));
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch } } });
await jest.advanceTimersByTimeAsync(15000); // idle → backed off to the 5s cap
busy = true;
await jest.advanceTimersByTimeAsync(POLL_MAX_MS); // the next (backed-off) tick fires, finds work → resets pollMs to POLL_MS
const before = claims(mocks, "check");
await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms
// Polling fast again (~2 in 500ms), not once per 5s — proves the interval reset on finding work.
expect(claims(mocks, "check") - before).toBeGreaterThanOrEqual(1);
});
it("does not back off a loop that is at capacity (capacity === 0)", async () => {
// cleanup-orphaned has concurrency 1; hold its one job in-flight so capacity stays 0.
// The backoff condition is `capacity > 0 && jobs.length === 0`, so an at-capacity loop
// must keep polling at POLL_MS (a freed slot should be picked up promptly, not after 5s).
const gate = deferred<void>();
let handed = false;
const claimDueBatch = jest.fn<any>(async (type: string) => {
if (type === "cleanup-orphaned" && !handed) {
handed = true;
return [makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 })];
}
return [];
});
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>(() => gate.promise)), // blocks → keeps the slot full
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, helper } });
const before = claims(mocks, "cleanup-orphaned"); // the blocking job is now in-flight → capacity 0
await jest.advanceTimersByTimeAsync(1000);
const delta = claims(mocks, "cleanup-orphaned") - before;
// ~4 polls at POLL_MS; if an at-capacity loop wrongly backed off it would be ~1.
expect(delta).toBeGreaterThanOrEqual(3);
gate.resolve();
await jest.advanceTimersByTimeAsync(1); // let the held job finish for a clean shutdown
});
it("wake() re-fires an idle, backed-off loop immediately", async () => {
const { worker, mocks } = await start({ queueMode: "worker" });
await jest.advanceTimersByTimeAsync(15000); // back off the check loop to the cap
const before = claims(mocks, "check");
worker.wake("check");
await jest.advanceTimersByTimeAsync(1);
expect(claims(mocks, "check")).toBe(before + 1); // fired now, not 5s later
});
it("wake() resets the cadence so the loop polls fast again, not at the 5s cap", async () => {
const { worker, mocks } = await start({ queueMode: "worker" });
await jest.advanceTimersByTimeAsync(15000); // backed off to the cap
worker.wake("check");
await jest.advanceTimersByTimeAsync(1); // immediate tick (still idle → next poll at ~POLL_MS*2)
const before = claims(mocks, "check");
await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms
// Polled again well within the old 5s cap, proving the interval was reset.
expect(claims(mocks, "check")).toBe(before + 1);
});
it("wake() is a no-op for a loop that isn't running in this process", async () => {
const { worker, mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false }); // no loops
expect(() => worker.wake("check")).not.toThrow();
await jest.advanceTimersByTimeAsync(1);
expect(claims(mocks, "check")).toBe(0);
});
it("addJob wakes the check loop so a new monitor's first check runs promptly", async () => {
const { worker, mocks } = await start({ queueMode: "worker" });
await jest.advanceTimersByTimeAsync(15000); // backed off to the cap
const before = claims(mocks, "check");
await worker.addJob("m2", makeMonitor({ id: "m2" }));
await jest.advanceTimersByTimeAsync(1);
expect(claims(mocks, "check")).toBe(before + 1);
});
it("resumeJob wakes the check loop so a resumed monitor runs promptly", async () => {
const { worker, mocks } = await start({ queueMode: "worker" });
await jest.advanceTimersByTimeAsync(15000); // backed off to the cap
const before = claims(mocks, "check");
await worker.resumeJob(makeMonitor());
await jest.advanceTimersByTimeAsync(1);
expect(claims(mocks, "check")).toBe(before + 1);
});
});
// ── shutdown ────────────────────────────────────────────────────────────────
describe("shutdown", () => {
@@ -3,6 +3,7 @@ import { MaintenanceWindowService } from "../../../src/domain/maintenance-window
import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts";
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts";
import type { IWorker } from "../../../src/worker/worker.interface.ts";
import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -33,16 +34,23 @@ const createJobsRepo = () =>
markMonitorsDue: jest.fn().mockResolvedValue(0),
}) as unknown as jest.Mocked<IJobsRepository>;
const createWorker = () =>
({
wake: jest.fn(),
}) as unknown as jest.Mocked<IWorker>;
const createService = (overrides?: {
monitorsRepository?: ReturnType<typeof createMonitorsRepo>;
maintenanceWindowsRepository?: ReturnType<typeof createMaintenanceWindowsRepo>;
jobsRepository?: ReturnType<typeof createJobsRepo>;
worker?: ReturnType<typeof createWorker>;
}) => {
const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo();
const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo();
const jobsRepository = overrides?.jobsRepository ?? createJobsRepo();
const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository });
return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository };
const worker = overrides?.worker ?? createWorker();
const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker });
return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker };
};
const makeWindow = (overrides?: Partial<MaintenanceWindow>): MaintenanceWindow => ({
@@ -275,7 +283,7 @@ describe("MaintenanceWindowService", () => {
it("flips covered monitors to initializing when deleting an active window", async () => {
const maintenanceWindowsRepository = createMaintenanceWindowsRepo();
(maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] }));
const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository });
const { service, monitorsRepository, jobsRepository, worker } = createService({ maintenanceWindowsRepository });
await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" });
@@ -283,6 +291,19 @@ describe("MaintenanceWindowService", () => {
expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]);
// Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval
expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number));
// ...and the idle loops are woken so the re-armed jobs run promptly
expect(worker.wake).toHaveBeenCalledWith("check");
expect(worker.wake).toHaveBeenCalledWith("geo-check");
});
it("does not wake the loops when no monitor leaves maintenance", async () => {
const maintenanceWindowsRepository = createMaintenanceWindowsRepo();
(maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeWindow({ active: false, monitorIds: ["mon-1"] }));
const { service, worker } = createService({ maintenanceWindowsRepository });
await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" });
expect(worker.wake).not.toHaveBeenCalled();
});
it("excludes the deleted window from the overlap check", async () => {
+83 -85
View File
@@ -7,21 +7,15 @@ import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts";
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
import type { IBufferService } from "../../../src/service/bufferService.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
const createBuffer = (): jest.Mocked<Pick<IBufferService, "addToBuffer">> => ({
addToBuffer: jest.fn(),
});
const createMonitorsRepo = () => {
const findById = jest.fn();
const updateStatusWindowAndChecks = jest
.fn()
.mockImplementation(
(
async (
_id: unknown,
_tid: unknown,
status: boolean,
@@ -30,27 +24,23 @@ const createMonitorsRepo = () => {
maxRecentChecks: number,
statusPatch?: Partial<Monitor>
) => {
// Grab the monitor that findById was configured to resolve with
const monitor = findById.mock.results.at(-1)?.value;
if (monitor && typeof monitor.then === "function") {
return monitor.then((m: any) => {
m.statusWindow = m.statusWindow || [];
m.statusWindow.push(status);
while (m.statusWindow.length > windowSize) {
m.statusWindow.shift();
}
m.recentChecks = m.recentChecks || [];
m.recentChecks.push(checkSnapshot);
while (m.recentChecks.length > maxRecentChecks) {
m.recentChecks.shift();
}
if (statusPatch) {
Object.assign(m, statusPatch);
}
return { ...m };
});
// updateMonitorStatus no longer reads the monitor itself — tests seed it via
// findById.mockResolvedValue, so the mock resolves it here as the post-write monitor.
const m: any = await findById();
m.statusWindow = m.statusWindow || [];
m.statusWindow.push(status);
while (m.statusWindow.length > windowSize) {
m.statusWindow.shift();
}
return Promise.reject(new Error("findById not mocked"));
m.recentChecks = m.recentChecks || [];
m.recentChecks.push(checkSnapshot);
while (m.recentChecks.length > maxRecentChecks) {
m.recentChecks.shift();
}
if (statusPatch) {
Object.assign(m, statusPatch);
}
return { ...m };
}
);
return {
@@ -67,23 +57,17 @@ const createMonitorStatsRepo = () =>
updateByMonitorId: jest.fn(),
}) as unknown as jest.Mocked<IMonitorStatsRepository>;
const createChecksRepo = () => ({}) as unknown as jest.Mocked<IChecksRepository>;
const createService = (overrides?: {
logger?: ReturnType<typeof createMockLogger>;
buffer?: ReturnType<typeof createBuffer>;
monitorsRepository?: ReturnType<typeof createMonitorsRepo>;
monitorStatsRepository?: ReturnType<typeof createMonitorStatsRepo>;
checksRepository?: ReturnType<typeof createChecksRepo>;
}) => {
const logger = overrides?.logger ?? createMockLogger();
const buffer = overrides?.buffer ?? createBuffer();
const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo();
const monitorStatsRepository = overrides?.monitorStatsRepository ?? createMonitorStatsRepo();
const checksRepository = overrides?.checksRepository ?? createChecksRepo();
const service = new StatusService(logger as any, buffer as any, monitorsRepository, monitorStatsRepository, checksRepository);
return { service, logger, buffer, monitorsRepository, monitorStatsRepository, checksRepository };
const service = new StatusService(logger as any, monitorsRepository, monitorStatsRepository);
return { service, logger, monitorsRepository, monitorStatsRepository };
};
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
@@ -264,19 +248,31 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck());
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
expect(result.statusChanged).toBe(false);
expect(result.prevStatus).toBe("up");
});
it("uses the prefetched monitor and skips findById when one is passed", async () => {
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" });
const { service, monitorsRepository } = createService();
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
expect(monitorsRepository.findById).not.toHaveBeenCalled();
expect(result.prevStatus).toBe("up");
expect(result.monitor).toBe(monitor);
});
it("pushes to statusWindow and trims to statusWindowSize", async () => {
const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5 });
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
// Atomic push should have been called with the correct status and window size
expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith(
@@ -297,7 +293,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }));
await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }), monitor);
expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS);
expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" }));
@@ -315,7 +311,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -333,7 +329,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("up");
@@ -350,7 +346,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck());
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -370,7 +366,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -387,7 +383,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
@@ -399,7 +395,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse(), makeCheck());
await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
expect(monitor.statusWindow).toEqual([true]);
});
@@ -410,23 +406,25 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse(), makeCheck());
await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
expect(monitor.recentChecks).toHaveLength(1);
});
it("throws AppError when repository throws", async () => {
const monitor = makeMonitor();
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockRejectedValue(new Error("db error"));
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error"));
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Failed to update monitor");
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Failed to update monitor");
});
it("throws AppError with 'Unknown error' for non-Error exceptions", async () => {
const monitor = makeMonitor();
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockRejectedValue("string error");
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error");
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Unknown error");
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Unknown error");
});
it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => {
@@ -444,7 +442,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
@@ -462,7 +460,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -484,7 +482,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
expect(result.monitor.status).toBe("up");
});
@@ -502,7 +500,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.monitor.status).toBe("down");
});
@@ -518,7 +516,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -535,7 +533,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -550,7 +548,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -566,7 +564,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -581,7 +579,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -601,7 +599,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
expect(result.monitor.status).toBe("up");
});
@@ -616,7 +614,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
@@ -629,7 +627,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down"));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck());
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
expect(result.monitor.status).toBe("up");
expect(logger.warn).toHaveBeenCalledWith(
@@ -647,7 +645,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck());
const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck(), monitor);
expect(result.code).toBe(201);
expect(result.timestamp).toBeGreaterThan(0);
@@ -688,7 +686,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.cpu).toBe(true);
expect(result.thresholdBreaches?.memory).toBe(false);
@@ -705,7 +703,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.memory).toBe(true);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
@@ -722,7 +720,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.disk).toBe(true);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
@@ -739,7 +737,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.temp).toBe(true);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
@@ -761,7 +759,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} },
} as any);
await service.updateMonitorStatus(response, makeCheck());
await service.updateMonitorStatus(response, makeCheck(), monitor);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
expect(patch.cpuAlertCounter).toBe(5);
@@ -780,7 +778,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.monitor.status).toBe("breached");
expect(result.statusChanged).toBe(false);
@@ -795,7 +793,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("up");
@@ -818,7 +816,7 @@ describe("StatusService", () => {
status: false,
data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }));
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor);
expect(result.monitor.status).toBe("down");
});
@@ -830,7 +828,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.cpu).toBe(false);
});
@@ -842,7 +840,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.memory).toBe(false);
});
@@ -856,7 +854,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.temp).toBe(false);
});
@@ -870,7 +868,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.temp).toBe(false);
});
@@ -882,7 +880,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse(undefined as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches).toBeUndefined();
});
@@ -894,7 +892,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches).toBeUndefined();
});
@@ -909,7 +907,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck());
const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
expect(result.thresholdBreaches).toBeUndefined();
});
@@ -923,7 +921,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.disk).toBe(false);
});
@@ -937,7 +935,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.disk).toBe(false);
});
@@ -951,7 +949,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.cpu).toBe(false);
expect(result.thresholdBreaches?.temp).toBe(false);
@@ -966,7 +964,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.memory).toBe(false);
});
@@ -980,7 +978,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches?.disk).toBe(false);
});
@@ -994,7 +992,7 @@ describe("StatusService", () => {
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck());
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
expect(patch.cpuAlertCounter).toBe(2);
@@ -1020,7 +1018,7 @@ describe("StatusService", () => {
status: false,
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }));
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -1040,7 +1038,7 @@ describe("StatusService", () => {
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {} },
} as any);
await service.updateMonitorStatus(response, makeCheck());
await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(monitor.cpuAlertCounter).toBe(0);
expect(monitor.memoryAlertCounter).toBe(0);
+3 -90
View File
@@ -1,14 +1,11 @@
import { describe, expect, it, jest } from "@jest/globals";
import { WorkerHelper } from "../../../src/worker/worker.helper.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
//
// WorkerHelper is now thin glue: it wires the scheduler's job functions to the
// CheckPipeline / GeoChecksPipeline (acquire→record→decide) and the
// ReactorDispatcher (react), and owns the cleanup jobs. The pipeline and
// dispatcher internals are tested in checkPipeline.test.ts / reactorDispatcher.test.ts;
// WorkerHelper is now thin glue: it wires the geo job to the GeoChecksPipeline and
// owns the cleanup jobs. The pipeline internals are tested in checkPipeline.test.ts;
// here we test only the glue + cleanup jobs.
const createHelper = (overrides?: Record<string, unknown>) => {
@@ -42,10 +39,6 @@ const createHelper = (overrides?: Record<string, unknown>) => {
deleteOlderThan: jest.fn().mockResolvedValue(0),
};
const checkPipeline = { run: jest.fn().mockResolvedValue(null) };
const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) };
const reactorDispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) };
const defaults = {
logger: createMockLogger(),
checkService,
@@ -57,9 +50,6 @@ const createHelper = (overrides?: Record<string, unknown>) => {
checksRepository,
incidentsRepository,
geoChecksRepository,
reactorDispatcher,
checkPipeline,
geoCheckPipeline,
...overrides,
};
@@ -73,26 +63,11 @@ const createHelper = (overrides?: Record<string, unknown>) => {
defaults.monitorStatsRepository as any,
defaults.checksRepository as any,
defaults.incidentsRepository as any,
defaults.geoChecksRepository as any,
defaults.reactorDispatcher as any,
defaults.checkPipeline as any,
defaults.geoCheckPipeline as any
defaults.geoChecksRepository as any
);
return { helper, defaults };
};
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "m1",
teamId: "team",
type: "http",
interval: 60000,
status: "up",
geoCheckEnabled: false,
geoCheckLocations: [],
...overrides,
}) as Monitor;
// ── Tests ────────────────────────────────────────────────────────────────────
describe("WorkerHelper", () => {
@@ -103,68 +78,6 @@ describe("WorkerHelper", () => {
});
});
// ── getHeartbeatJob (glue) ────────────────────────────────────────────────
describe("getHeartbeatJob", () => {
it("runs the pipeline and dispatches the evaluation to reactors", async () => {
const evaluation = { monitor: { id: "m1" }, decision: {} };
const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(evaluation) } });
await helper.getHeartbeatJob()(makeMonitor());
expect(defaults.checkPipeline.run).toHaveBeenCalled();
expect(defaults.reactorDispatcher.dispatch).toHaveBeenCalledWith(evaluation);
});
it("does not dispatch when the pipeline returns null (skipped)", async () => {
const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(null) } });
await helper.getHeartbeatJob()(makeMonitor());
expect(defaults.reactorDispatcher.dispatch).not.toHaveBeenCalled();
});
it("logs and rethrows when the pipeline throws", async () => {
const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue(new Error("boom")) } });
await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toThrow("boom");
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "boom" }));
});
it("rethrows non-Error failures", async () => {
const { helper } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue("unexpected") } });
await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toBe("unexpected");
});
});
// ── getHeartbeatGeoJob (glue) ─────────────────────────────────────────────
describe("getHeartbeatGeoJob", () => {
it("runs the geo pipeline", async () => {
const { helper, defaults } = createHelper();
await helper.getHeartbeatGeoJob()(makeMonitor());
expect(defaults.geoCheckPipeline.run).toHaveBeenCalled();
});
it("catches and logs a pipeline error without rethrowing", async () => {
const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue(new Error("api timeout")) } });
await expect(helper.getHeartbeatGeoJob()(makeMonitor())).resolves.toBeUndefined();
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "api timeout" }));
});
it("logs 'Unknown error' for non-Error failures", async () => {
const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue("string error") } });
await helper.getHeartbeatGeoJob()(makeMonitor());
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
});
});
// ── getCleanupOrphanedJob ────────────────────────────────────────────────
describe("getCleanupOrphanedJob", () => {