Merge pull request #3733 from bluewave-labs/fix/excess-polling
Fix/excess polling
This commit is contained in:
19 files changed
+570
-438
No files matched your search
@@ -103,7 +103,7 @@ import { ILogger } from "@/utils/logger.js";
|
||||
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
|
||||
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { CheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
|
||||
@@ -229,7 +229,7 @@ export const initializeServices = async ({
|
||||
|
||||
const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository);
|
||||
|
||||
const statusService = new StatusService(logger, bufferService, monitorsRepository, monitorStatsRepository, checksRepository);
|
||||
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
|
||||
|
||||
// Notification providers
|
||||
const webhookProvider = new WebhookProvider(logger);
|
||||
@@ -276,8 +276,6 @@ export const initializeServices = async ({
|
||||
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
|
||||
|
||||
// pipelines
|
||||
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
|
||||
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
|
||||
|
||||
const workerHelper = new WorkerHelper(
|
||||
@@ -290,10 +288,7 @@ export const initializeServices = async ({
|
||||
monitorStatsRepository,
|
||||
checksRepository,
|
||||
incidentsRepository,
|
||||
geoChecksRepository,
|
||||
reactorDispatcher,
|
||||
checkPipeline,
|
||||
geoCheckPipeline
|
||||
geoChecksRepository
|
||||
);
|
||||
|
||||
const worker = await DBQueueWorker.create(
|
||||
@@ -339,6 +334,7 @@ export const initializeServices = async ({
|
||||
monitorsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
jobsRepository,
|
||||
worker,
|
||||
});
|
||||
const monitorService = new MonitorService({
|
||||
worker,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { IWorker } from "@/worker/worker.interface.js";
|
||||
import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
@@ -42,19 +43,23 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
private jobsRepository: IJobsRepository;
|
||||
private worker: IWorker;
|
||||
|
||||
constructor({
|
||||
monitorsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
jobsRepository,
|
||||
worker,
|
||||
}: {
|
||||
monitorsRepository: IMonitorsRepository;
|
||||
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
jobsRepository: IJobsRepository;
|
||||
worker: IWorker;
|
||||
}) {
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.maintenanceWindowsRepository = maintenanceWindowsRepository;
|
||||
this.jobsRepository = jobsRepository;
|
||||
this.worker = worker;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
@@ -78,6 +83,9 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
|
||||
await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]);
|
||||
// Rearm jobs to run soon (now + jitter to avoid herding)
|
||||
await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime());
|
||||
// Wake the (possibly idle, backed-off) loops so the rearmed jobs run promptly instead of waiting up to POLL_MAX_MS
|
||||
this.worker.wake("check");
|
||||
this.worker.wake("geo-check");
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@ export interface IMonitorsRepository {
|
||||
createMonitors(monitors: Monitor[]): Promise<Monitor[]>;
|
||||
// single fetch
|
||||
findById(monitorId: string, teamId: string): Promise<Monitor>;
|
||||
// for workers, don't need all details
|
||||
findByIdLean(monitorId: string): Promise<Monitor | null>;
|
||||
|
||||
// collection fetch
|
||||
findAll(): Promise<Monitor[] | null>;
|
||||
|
||||
@@ -38,6 +38,11 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
return this.toEntity(monitor);
|
||||
};
|
||||
|
||||
findByIdLean = async (monitorId: string): Promise<Monitor | null> => {
|
||||
const monitor = await MonitorModel.findOne({ _id: monitorId });
|
||||
return monitor ? this.toEntity(monitor) : null;
|
||||
};
|
||||
|
||||
findAll = async (): Promise<Monitor[]> => {
|
||||
const monitors = await MonitorModel.find();
|
||||
return this.mapDocuments(monitors);
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import type { Check, CheckDiskInfo, CheckSnapshot } from "@/domain/checks/check.type.js";
|
||||
@@ -17,7 +16,6 @@ import type {
|
||||
} from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IBufferService } from "./bufferService.js";
|
||||
import type { HardwareStatusMetrics } from "@/types/network.js";
|
||||
import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.types.js";
|
||||
|
||||
@@ -42,30 +40,21 @@ export interface IStatusService {
|
||||
| GrpcStatusPayload
|
||||
| undefined
|
||||
>,
|
||||
check: Check
|
||||
check: Check,
|
||||
monitor: Monitor
|
||||
): Promise<StatusChangeResult>;
|
||||
}
|
||||
|
||||
export class StatusService implements IStatusService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
private logger: ILogger;
|
||||
private buffer: IBufferService;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private monitorStatsRepository: IMonitorStatsRepository;
|
||||
private checksRepository: IChecksRepository;
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
buffer: IBufferService,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
monitorStatsRepository: IMonitorStatsRepository,
|
||||
checksRepository: IChecksRepository
|
||||
) {
|
||||
constructor(logger: ILogger, monitorsRepository: IMonitorsRepository, monitorStatsRepository: IMonitorStatsRepository) {
|
||||
this.logger = logger;
|
||||
this.buffer = buffer;
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.monitorStatsRepository = monitorStatsRepository;
|
||||
this.checksRepository = checksRepository;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
@@ -212,11 +201,11 @@ export class StatusService implements IStatusService {
|
||||
| GrpcStatusPayload
|
||||
| undefined
|
||||
>,
|
||||
check: Check
|
||||
check: Check,
|
||||
monitor: Monitor
|
||||
): Promise<StatusChangeResult> => {
|
||||
try {
|
||||
const { monitorId, teamId, status, code } = statusResponse;
|
||||
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
|
||||
const { status, code } = statusResponse;
|
||||
|
||||
// Update running stats
|
||||
await this.tryUpdateRunningStats(monitor, statusResponse);
|
||||
@@ -262,7 +251,7 @@ export class StatusService implements IStatusService {
|
||||
};
|
||||
}
|
||||
|
||||
// First evaluate reachability-based status changes, which apply to all monitor types
|
||||
// First evaluate reachability status changes, which apply to all monitor types
|
||||
// and take precedence over hardware breaches.
|
||||
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
|
||||
if (reachabilityResult.transitioned) {
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
|
||||
export interface ICheckEvaluator {
|
||||
evaluate(status: MonitorStatusResponse, check: Check): Promise<MonitorEvaluation>;
|
||||
evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
|
||||
}
|
||||
|
||||
export class CheckEvaluator implements ICheckEvaluator {
|
||||
@@ -13,11 +14,11 @@ export class CheckEvaluator implements ICheckEvaluator {
|
||||
private statusService: IStatusService,
|
||||
private monitorStatusPolicy: IMonitorStatusPolicy
|
||||
) {}
|
||||
evaluate = async (status: MonitorStatusResponse, check: Check) => {
|
||||
evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
|
||||
// ****************************
|
||||
// Step 3: Evaluate and return result to reactors
|
||||
// ****************************
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor);
|
||||
|
||||
// Step 5. Get decisions and create an evaluation obj
|
||||
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
|
||||
|
||||
@@ -7,8 +7,6 @@ import { IBufferService } from "@/service/bufferService.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
|
||||
import { ICheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
|
||||
const SERVICE_NAME = "CheckPipeline";
|
||||
|
||||
@@ -16,22 +14,6 @@ export interface ICheckPipeline {
|
||||
run(monitor: Monitor): Promise<MonitorEvaluation | null>; // null = skipped
|
||||
}
|
||||
|
||||
export class CheckPipeline implements ICheckPipeline {
|
||||
constructor(
|
||||
private checkProducer: ICheckProducer,
|
||||
private checkEvaluator: ICheckEvaluator
|
||||
) {}
|
||||
|
||||
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
|
||||
const checkResult = await this.checkProducer.produce(monitor);
|
||||
if (!checkResult) {
|
||||
return null; // skipped due to maintenance / geo guards / other guards
|
||||
}
|
||||
const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check);
|
||||
return evaluation;
|
||||
};
|
||||
}
|
||||
|
||||
export class GeoChecksPipeline implements ICheckPipeline {
|
||||
constructor(
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
|
||||
|
||||
@@ -15,7 +15,8 @@ import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.rep
|
||||
import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
const POLL_MS = 250;
|
||||
const POLL_MS = 250; // base poll interval while a loop is actively claiming work
|
||||
const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs
|
||||
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
|
||||
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
|
||||
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
|
||||
@@ -40,6 +41,17 @@ export class DBQueueWorker implements IWorker {
|
||||
"cleanup-retention": 0,
|
||||
};
|
||||
|
||||
// Per job type poll intervals. These back off on idle and reset on work or wake()
|
||||
private pollMs: Record<JobType, number> = {
|
||||
check: POLL_MS,
|
||||
"geo-check": POLL_MS,
|
||||
evaluate: POLL_MS,
|
||||
"cleanup-orphaned": POLL_MS,
|
||||
"cleanup-retention": POLL_MS,
|
||||
};
|
||||
private ticking: Partial<Record<JobType, boolean>> = {}; // Whether a loop is mid tick
|
||||
private tickFns = new Map<JobType, () => void>(); // Store tick fns so we can run them immediately on wake
|
||||
|
||||
constructor(
|
||||
private logger: ILogger,
|
||||
private jobsRepository: IJobsRepository,
|
||||
@@ -150,7 +162,7 @@ export class DBQueueWorker implements IWorker {
|
||||
|
||||
private runCheck = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
await this.checkProducer.produce(monitor);
|
||||
};
|
||||
@@ -161,19 +173,22 @@ export class DBQueueWorker implements IWorker {
|
||||
|
||||
private runEvaluate = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
|
||||
|
||||
let current = monitor;
|
||||
for (const check of checks) {
|
||||
const status = this.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.checkEvaluator.evaluate(status, check);
|
||||
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
|
||||
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
|
||||
await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
|
||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||
}
|
||||
};
|
||||
|
||||
private runGeoCheck = async (job: Job) => {
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId!]);
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId!);
|
||||
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
|
||||
};
|
||||
|
||||
@@ -235,8 +250,10 @@ export class DBQueueWorker implements IWorker {
|
||||
};
|
||||
|
||||
private startLoop = (type: JobType) => {
|
||||
this.pollMs[type] = POLL_MS;
|
||||
const tick = async () => {
|
||||
if (this.stopped) return;
|
||||
this.ticking[type] = true;
|
||||
try {
|
||||
// Claim a batch sized to the free capacity
|
||||
const capacity = CONCURRENCY[type] - this.inFlight[type];
|
||||
@@ -247,6 +264,8 @@ export class DBQueueWorker implements IWorker {
|
||||
this.inFlight[type]--; // job done, free the slot
|
||||
});
|
||||
}
|
||||
// Back off if no jobs found, reset to base if jobs found
|
||||
this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS;
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
@@ -254,11 +273,26 @@ export class DBQueueWorker implements IWorker {
|
||||
method: `loop:${type}`,
|
||||
});
|
||||
}
|
||||
if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down
|
||||
// Clearing the flag and re-arming run synchronously (no await between), so wake() can never interleave here.
|
||||
this.ticking[type] = false;
|
||||
if (!this.stopped) this.timers.set(type, setTimeout(tick, this.pollMs[type])); // re-arm unless shutting down
|
||||
};
|
||||
this.tickFns.set(type, tick);
|
||||
tick();
|
||||
};
|
||||
|
||||
// Wake a worker loop after adding a job that's due immediately.
|
||||
// Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off
|
||||
wake = (type: JobType) => {
|
||||
this.pollMs[type] = POLL_MS;
|
||||
if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing
|
||||
const tick = this.tickFns.get(type);
|
||||
if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done
|
||||
const pending = this.timers.get(type);
|
||||
if (pending) clearTimeout(pending); // Clear out scheduled tick if any
|
||||
this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick
|
||||
};
|
||||
|
||||
// Seed queue
|
||||
private reconcile = async () => {
|
||||
const now = Date.now();
|
||||
@@ -308,7 +342,11 @@ export class DBQueueWorker implements IWorker {
|
||||
addJob = async (_monitorId: string, monitor: Monitor) => {
|
||||
const now = Date.now();
|
||||
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true));
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true));
|
||||
this.wake("check");
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true));
|
||||
this.wake("geo-check");
|
||||
}
|
||||
};
|
||||
deleteJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.deleteById(monitor.id);
|
||||
@@ -317,12 +355,15 @@ export class DBQueueWorker implements IWorker {
|
||||
await this.jobsRepository.setActiveById(monitor.id, false);
|
||||
};
|
||||
resumeJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.setActiveById(monitor.id, true);
|
||||
await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately
|
||||
this.wake("check");
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check");
|
||||
};
|
||||
updateJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval);
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away
|
||||
this.wake("geo-check");
|
||||
} else {
|
||||
await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
const SERVICE_NAME = "JobQueueHelper";
|
||||
import type { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
|
||||
@@ -11,13 +10,9 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
|
||||
export interface IWorkerHelper {
|
||||
readonly serviceName: string;
|
||||
getHeartbeatJob(): (monitor: Monitor) => Promise<void>;
|
||||
getHeartbeatGeoJob(): (monitor: Monitor) => Promise<void>;
|
||||
getCleanupOrphanedJob(): () => Promise<void>;
|
||||
getCleanupRetentionJob(): () => Promise<void>;
|
||||
}
|
||||
@@ -49,9 +44,6 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
private checksRepository: IChecksRepository;
|
||||
private incidentsRepository: IIncidentsRepository;
|
||||
private geoChecksRepository: IGeoChecksRepository;
|
||||
private reactorDispatcher: IReactorDispatcher;
|
||||
private checkPipeline: ICheckPipeline;
|
||||
private geoCheckPipeline: ICheckPipeline;
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
@@ -63,10 +55,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
monitorStatsRepository: IMonitorStatsRepository,
|
||||
checksRepository: IChecksRepository,
|
||||
incidentsRepository: IIncidentsRepository,
|
||||
geoChecksRepository: IGeoChecksRepository,
|
||||
reactorDispatcher: IReactorDispatcher,
|
||||
checkPipeline: ICheckPipeline,
|
||||
geoCheckPipeline: ICheckPipeline
|
||||
geoChecksRepository: IGeoChecksRepository
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.checkService = checkService;
|
||||
@@ -78,48 +67,12 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
this.checksRepository = checksRepository;
|
||||
this.incidentsRepository = incidentsRepository;
|
||||
this.geoChecksRepository = geoChecksRepository;
|
||||
this.reactorDispatcher = reactorDispatcher;
|
||||
this.checkPipeline = checkPipeline;
|
||||
this.geoCheckPipeline = geoCheckPipeline;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return WorkerHelper.SERVICE_NAME;
|
||||
}
|
||||
|
||||
getHeartbeatJob = () => {
|
||||
return async (monitor: Monitor) => {
|
||||
try {
|
||||
const evaluation = await this.checkPipeline.run(monitor);
|
||||
if (evaluation) await this.reactorDispatcher.dispatch(evaluation);
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: error instanceof Error ? error.message : "Unknown error",
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
getHeartbeatGeoJob = () => {
|
||||
return async (monitor: Monitor) => {
|
||||
try {
|
||||
await this.geoCheckPipeline.run(monitor);
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : "Unknown error",
|
||||
service: SERVICE_NAME,
|
||||
method: "getHeartbeatGeoJob",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
// Don't throw - geo check failures shouldn't crash the job scheduler
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
getCleanupOrphanedJob = () => {
|
||||
return async () => {
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { JobType } from "@/domain/jobs/job.type.js";
|
||||
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
@@ -70,6 +71,7 @@ export interface IWorker {
|
||||
pauseJob(monitor: Monitor): Promise<void>;
|
||||
resumeJob(monitor: Monitor): Promise<void>;
|
||||
updateJob(monitor: Monitor): Promise<void>;
|
||||
wake(type: JobType): void;
|
||||
shutdown(): Promise<void>;
|
||||
getMetrics(): Promise<WorkerMetrics>;
|
||||
getJobs(pagination: WorkerJobsPagination): Promise<WorkerJobsPage>;
|
||||
|
||||
@@ -40,6 +40,11 @@ export class InMemoryMonitorsRepository implements IMonitorsRepository {
|
||||
return this.monitors.filter((m) => monitorIds.includes(m.id)).map((m) => ({ ...m }));
|
||||
}
|
||||
|
||||
async findByIdLean(monitorId: string): Promise<Monitor | null> {
|
||||
const monitor = this.monitors.find((m) => m.id === monitorId);
|
||||
return monitor ? { ...monitor } : null;
|
||||
}
|
||||
|
||||
async findByIdsWithChecks(monitorIds: string[], _checksCount?: number): Promise<Monitor[]> {
|
||||
return this.findByIds(monitorIds);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import { jest } from "@jest/globals";
|
||||
import { WorkerHelper } from "../../src/worker/worker.helper.ts";
|
||||
import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts";
|
||||
import { CheckProducer } from "../../src/worker/worker.check-producer.ts";
|
||||
import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts";
|
||||
import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts";
|
||||
@@ -71,20 +69,6 @@ const createStubMonitorStatsRepo = () => ({
|
||||
deleteByMonitorIdsNotIn: jest.fn(),
|
||||
});
|
||||
|
||||
const createStubChecksRepo = () => ({
|
||||
create: jest.fn(),
|
||||
createChecks: jest.fn(),
|
||||
findByMonitorId: jest.fn(),
|
||||
findByTeamId: jest.fn(),
|
||||
findLatestByMonitorIds: jest.fn(),
|
||||
findByDateRangeAndMonitorId: jest.fn(),
|
||||
findSummaryByTeamId: jest.fn(),
|
||||
deleteByMonitorId: jest.fn(),
|
||||
deleteByTeamId: jest.fn(),
|
||||
deleteByMonitorIdsNotIn: jest.fn(),
|
||||
deleteOlderThan: jest.fn(),
|
||||
});
|
||||
|
||||
export interface HeartbeatTestHarness {
|
||||
monitorsRepo: InMemoryMonitorsRepository;
|
||||
incidentsRepo: InMemoryIncidentsRepository;
|
||||
@@ -108,13 +92,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
const logger = createMockLogger() as any;
|
||||
const bufferStub = { addToBuffer: jest.fn(), addGeoCheckToBuffer: jest.fn(), scheduleNextFlush: jest.fn() };
|
||||
|
||||
const statusService = new StatusService(
|
||||
logger,
|
||||
bufferStub as any,
|
||||
monitorsRepo as any,
|
||||
createStubMonitorStatsRepo() as any,
|
||||
createStubChecksRepo() as any
|
||||
);
|
||||
const statusService = new StatusService(logger, monitorsRepo as any, createStubMonitorStatsRepo() as any);
|
||||
|
||||
const messageBuilder = { extractThresholdBreaches: jest.fn().mockReturnValue([]) };
|
||||
const incidentService = new IncidentService(logger, incidentsRepo, monitorsRepo as any, { findById: jest.fn() } as any, messageBuilder as any);
|
||||
@@ -149,7 +127,6 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
};
|
||||
|
||||
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
|
||||
const geoChecksService = { buildGeoCheck: jest.fn() };
|
||||
|
||||
const checkProducer = new CheckProducer(
|
||||
monitorsRepo as any,
|
||||
@@ -160,28 +137,21 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
logger
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
|
||||
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger);
|
||||
|
||||
const notificationReactor = new NotificationReactor(notificationsService as any);
|
||||
const incidentReactor = new IncidentReactor(incidentService as any);
|
||||
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
|
||||
|
||||
const helper = new WorkerHelper(
|
||||
logger,
|
||||
checkService as any,
|
||||
{ getDBSettings: jest.fn() } as any,
|
||||
monitorsRepo as any,
|
||||
{ deleteByMonitorIdsNotIn: jest.fn() } as any,
|
||||
{ findAllTeamIds: jest.fn() } as any,
|
||||
createStubMonitorStatsRepo() as any,
|
||||
createStubChecksRepo() as any,
|
||||
incidentsRepo as any,
|
||||
{ deleteByMonitorIdsNotIn: jest.fn() } as any,
|
||||
reactorDispatcher,
|
||||
checkPipeline,
|
||||
geoCheckPipeline
|
||||
);
|
||||
// Mirror the production check→evaluate flow: read the monitor fresh each cycle (so the
|
||||
// accumulated statusWindow is visible), produce a check, then evaluate against that monitor.
|
||||
const heartbeatJob = async (seedMonitor: Monitor) => {
|
||||
const monitor = await monitorsRepo.findByIdLean(seedMonitor.id);
|
||||
if (!monitor) return;
|
||||
const result = await checkProducer.produce(monitor);
|
||||
if (!result) return; // skipped (e.g. maintenance window)
|
||||
const evaluation = await checkEvaluator.evaluate(result.status, result.check, monitor);
|
||||
await reactorDispatcher.dispatch(evaluation);
|
||||
};
|
||||
|
||||
return {
|
||||
monitorsRepo,
|
||||
@@ -193,7 +163,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
bufferStub,
|
||||
maintenanceWindowsRepo,
|
||||
messageBuilder,
|
||||
heartbeatJob: helper.getHeartbeatJob(),
|
||||
heartbeatJob,
|
||||
setNextResponse,
|
||||
setNextResponseFull,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
const createEvaluator = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
statusService: {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
...overrides,
|
||||
};
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
|
||||
return { evaluator, defaults };
|
||||
};
|
||||
|
||||
describe("CheckEvaluator", () => {
|
||||
it("threads status, check, and monitor through to the status service", async () => {
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const check = { id: "check-1" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
|
||||
await evaluator.evaluate(status, check, monitor);
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor);
|
||||
});
|
||||
|
||||
it("returns an evaluation that opens an incident on a down transition", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1" } as any;
|
||||
const statusChange = { monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 };
|
||||
const { evaluator } = createEvaluator({
|
||||
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
|
||||
});
|
||||
|
||||
const result = await evaluator.evaluate(status, check, makeMonitor());
|
||||
|
||||
expect(result).toMatchObject({
|
||||
monitor: statusChange.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange,
|
||||
decision: expect.objectContaining({
|
||||
shouldCreateIncident: true,
|
||||
shouldSendNotification: true,
|
||||
incidentReason: "status_down",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("returns an evaluation that resolves an incident on recovery", async () => {
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const statusChange = { monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down", code: 200 };
|
||||
const { evaluator } = createEvaluator({
|
||||
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
|
||||
});
|
||||
|
||||
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor({ status: "down" }));
|
||||
|
||||
expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
|
||||
});
|
||||
|
||||
it("produces a no-op decision when status did not change", async () => {
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const { evaluator } = createEvaluator();
|
||||
|
||||
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor());
|
||||
|
||||
expect(result.decision).toMatchObject({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,5 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import { GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
@@ -23,114 +20,6 @@ const activeWindow = () => {
|
||||
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
|
||||
};
|
||||
|
||||
// ── CheckPipeline ────────────────────────────────────────────────────────────
|
||||
|
||||
const createCheckPipeline = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
|
||||
statusService: {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) },
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
...overrides,
|
||||
};
|
||||
const checkProducer = new CheckProducer(
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.maintenanceWindowsRepository as any,
|
||||
defaults.checkService as any,
|
||||
defaults.networkService as any,
|
||||
defaults.buffer as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
|
||||
const pipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
return { pipeline, defaults };
|
||||
};
|
||||
|
||||
describe("CheckPipeline", () => {
|
||||
it("throws when monitor id is missing", async () => {
|
||||
const { pipeline } = createCheckPipeline();
|
||||
await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id");
|
||||
});
|
||||
|
||||
it("skips and sets maintenance status when in maintenance window", async () => {
|
||||
const { pipeline, defaults } = createCheckPipeline({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
const result = await pipeline.run(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
|
||||
});
|
||||
|
||||
it("does not re-write status when already in maintenance", async () => {
|
||||
const { pipeline, defaults } = createCheckPipeline({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
await pipeline.run(makeMonitor({ status: "maintenance" }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("throws when network response is null", async () => {
|
||||
const { pipeline } = createCheckPipeline({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(null) },
|
||||
});
|
||||
await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response");
|
||||
});
|
||||
|
||||
it("returns null and warns when toCheck returns null", async () => {
|
||||
const { pipeline, defaults } = createCheckPipeline({
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(null) },
|
||||
});
|
||||
|
||||
const result = await pipeline.run(makeMonitor());
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
|
||||
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("buffers the check, updates status, and returns an evaluation", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
|
||||
const statusChange = { monitor: { id: "m1", status: "down" }, statusChanged: true, prevStatus: "up", code: 500 };
|
||||
const { pipeline, defaults } = createCheckPipeline({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
|
||||
});
|
||||
|
||||
const result = await pipeline.run(makeMonitor());
|
||||
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith({ id: "check-1" });
|
||||
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
monitor: statusChange.monitor,
|
||||
status,
|
||||
check: { id: "check-1" },
|
||||
statusChange,
|
||||
decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, incidentReason: "status_down" }),
|
||||
});
|
||||
});
|
||||
|
||||
it("proceeds with the check when no maintenance window is active", async () => {
|
||||
const { pipeline, defaults } = createCheckPipeline({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
|
||||
});
|
||||
|
||||
await pipeline.run(makeMonitor());
|
||||
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── GeoChecksPipeline ──────────────────────────────────────────────────────────
|
||||
|
||||
const createGeoPipeline = (overrides?: Record<string, any>) => {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
// A maintenance window straddling "now" — active per isWindowActive.
|
||||
const activeWindow = () => {
|
||||
const now = Date.now();
|
||||
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
|
||||
};
|
||||
|
||||
const createProducer = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) },
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
...overrides,
|
||||
};
|
||||
const producer = new CheckProducer(
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.maintenanceWindowsRepository as any,
|
||||
defaults.checkService as any,
|
||||
defaults.networkService as any,
|
||||
defaults.buffer as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { producer, defaults };
|
||||
};
|
||||
|
||||
describe("CheckProducer", () => {
|
||||
it("throws when monitor id is missing", async () => {
|
||||
const { producer } = createProducer();
|
||||
await expect(producer.produce({} as Monitor)).rejects.toThrow("No monitor id");
|
||||
});
|
||||
|
||||
// ── maintenance gate ──────────────────────────────────────────────────────
|
||||
|
||||
it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-write status when already in maintenance", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor({ status: "maintenance" }));
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("proceeds with the check when the maintenance window is inactive", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── acquire / record ──────────────────────────────────────────────────────
|
||||
|
||||
it("throws when the network response is null", async () => {
|
||||
const { producer } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(null) },
|
||||
});
|
||||
|
||||
await expect(producer.produce(makeMonitor())).rejects.toThrow("No network response");
|
||||
});
|
||||
|
||||
it("returns null, warns, and does not buffer when toCheck yields nothing", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(null) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
|
||||
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("buffers the built check and returns the status and check on success", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
|
||||
const check = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(result).toEqual({ status, check });
|
||||
});
|
||||
});
|
||||
@@ -80,6 +80,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
};
|
||||
const monitorsRepository = {
|
||||
findByIds: jest.fn<any>().mockResolvedValue([makeMonitor()]),
|
||||
findByIdLean: jest.fn<any>().mockResolvedValue(makeMonitor()),
|
||||
findAll: jest.fn<any>().mockResolvedValue([]),
|
||||
updateById: jest.fn<any>().mockResolvedValue({}),
|
||||
};
|
||||
@@ -94,7 +95,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
produce: jest.fn<any>().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }),
|
||||
};
|
||||
const checkEvaluator = {
|
||||
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), decision: {} }),
|
||||
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), statusChange: { monitor: makeMonitor() }, decision: {} }),
|
||||
};
|
||||
const geoCheckPipeline = { run: jest.fn<any>().mockResolvedValue(null) };
|
||||
const dispatcher = { dispatch: jest.fn<any>().mockResolvedValue(undefined) };
|
||||
@@ -268,7 +269,7 @@ describe("DBQueueWorker", () => {
|
||||
const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]);
|
||||
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
|
||||
expect(mocks.checkProducer.produce).toHaveBeenCalled();
|
||||
// The check stage only produces; the buffer arms evaluate once the check is durably stored
|
||||
expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||
@@ -307,6 +308,35 @@ describe("DBQueueWorker", () => {
|
||||
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
|
||||
});
|
||||
|
||||
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const initialMonitor = makeMonitor();
|
||||
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
|
||||
const monitorsRepository = {
|
||||
findByIdLean: jest.fn<any>().mockResolvedValue(initialMonitor),
|
||||
updateById: jest.fn<any>().mockResolvedValue({}),
|
||||
};
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) };
|
||||
const checkEvaluator = {
|
||||
evaluate: jest.fn<any>().mockResolvedValue({ monitor: postWriteMonitor, statusChange: { monitor: postWriteMonitor }, decision: {} }),
|
||||
};
|
||||
const { mocks } = await start({
|
||||
mocks: {
|
||||
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkEvaluator,
|
||||
},
|
||||
});
|
||||
|
||||
// The monitor is read once for the whole backlog, not once per check
|
||||
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
|
||||
// First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first
|
||||
expect(mocks.checkEvaluator.evaluate.mock.calls[0][2]).toBe(initialMonitor);
|
||||
expect(mocks.checkEvaluator.evaluate.mock.calls[1][2]).toBe(postWriteMonitor);
|
||||
});
|
||||
|
||||
it("cleanup-orphaned job runs the helper's cleanup function", async () => {
|
||||
const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 });
|
||||
const cleanupFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
@@ -449,6 +479,126 @@ describe("DBQueueWorker", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── poll backoff + wake() ───────────────────────────────────────────────────
|
||||
//
|
||||
// An idle loop grows its poll interval up to POLL_MAX_MS (5s). wake() fast-paths
|
||||
// it back so a freshly-enqueued "run now" job doesn't wait out that backoff.
|
||||
|
||||
describe("poll backoff and wake", () => {
|
||||
const POLL_MAX_MS = 5000;
|
||||
const claims = (mocks: any, type: string) => mocks.jobsRepository.claimDueBatch.mock.calls.filter((c: any[]) => c[0] === type).length;
|
||||
|
||||
it("backs off the idle poll interval toward POLL_MAX_MS", async () => {
|
||||
const { mocks } = await start({ queueMode: "worker" }); // claimDueBatch returns [] → always idle
|
||||
await jest.advanceTimersByTimeAsync(15000); // let the check loop grow to the cap
|
||||
const before = claims(mocks, "check");
|
||||
await jest.advanceTimersByTimeAsync(POLL_MAX_MS * 2); // ~2 polls at a 5s cadence
|
||||
const delta = claims(mocks, "check") - before;
|
||||
// At the cap that's ~2 polls; at the un-backed-off 250ms cadence it would be ~40.
|
||||
expect(delta).toBeGreaterThanOrEqual(1);
|
||||
expect(delta).toBeLessThanOrEqual(4);
|
||||
});
|
||||
|
||||
it("resets the poll interval to POLL_MS once a tick finds work after being idle", async () => {
|
||||
let busy = false;
|
||||
const claimDueBatch = jest.fn<any>(async (type: string) => (busy && type === "check" ? [makeJob()] : []));
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch } } });
|
||||
|
||||
await jest.advanceTimersByTimeAsync(15000); // idle → backed off to the 5s cap
|
||||
busy = true;
|
||||
await jest.advanceTimersByTimeAsync(POLL_MAX_MS); // the next (backed-off) tick fires, finds work → resets pollMs to POLL_MS
|
||||
const before = claims(mocks, "check");
|
||||
|
||||
await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms
|
||||
|
||||
// Polling fast again (~2 in 500ms), not once per 5s — proves the interval reset on finding work.
|
||||
expect(claims(mocks, "check") - before).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("does not back off a loop that is at capacity (capacity === 0)", async () => {
|
||||
// cleanup-orphaned has concurrency 1; hold its one job in-flight so capacity stays 0.
|
||||
// The backoff condition is `capacity > 0 && jobs.length === 0`, so an at-capacity loop
|
||||
// must keep polling at POLL_MS (a freed slot should be picked up promptly, not after 5s).
|
||||
const gate = deferred<void>();
|
||||
let handed = false;
|
||||
const claimDueBatch = jest.fn<any>(async (type: string) => {
|
||||
if (type === "cleanup-orphaned" && !handed) {
|
||||
handed = true;
|
||||
return [makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 })];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>(() => gate.promise)), // blocks → keeps the slot full
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, helper } });
|
||||
|
||||
const before = claims(mocks, "cleanup-orphaned"); // the blocking job is now in-flight → capacity 0
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
const delta = claims(mocks, "cleanup-orphaned") - before;
|
||||
|
||||
// ~4 polls at POLL_MS; if an at-capacity loop wrongly backed off it would be ~1.
|
||||
expect(delta).toBeGreaterThanOrEqual(3);
|
||||
|
||||
gate.resolve();
|
||||
await jest.advanceTimersByTimeAsync(1); // let the held job finish for a clean shutdown
|
||||
});
|
||||
|
||||
it("wake() re-fires an idle, backed-off loop immediately", async () => {
|
||||
const { worker, mocks } = await start({ queueMode: "worker" });
|
||||
await jest.advanceTimersByTimeAsync(15000); // back off the check loop to the cap
|
||||
const before = claims(mocks, "check");
|
||||
|
||||
worker.wake("check");
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(claims(mocks, "check")).toBe(before + 1); // fired now, not 5s later
|
||||
});
|
||||
|
||||
it("wake() resets the cadence so the loop polls fast again, not at the 5s cap", async () => {
|
||||
const { worker, mocks } = await start({ queueMode: "worker" });
|
||||
await jest.advanceTimersByTimeAsync(15000); // backed off to the cap
|
||||
worker.wake("check");
|
||||
await jest.advanceTimersByTimeAsync(1); // immediate tick (still idle → next poll at ~POLL_MS*2)
|
||||
const before = claims(mocks, "check");
|
||||
|
||||
await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms
|
||||
|
||||
// Polled again well within the old 5s cap, proving the interval was reset.
|
||||
expect(claims(mocks, "check")).toBe(before + 1);
|
||||
});
|
||||
|
||||
it("wake() is a no-op for a loop that isn't running in this process", async () => {
|
||||
const { worker, mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false }); // no loops
|
||||
expect(() => worker.wake("check")).not.toThrow();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(claims(mocks, "check")).toBe(0);
|
||||
});
|
||||
|
||||
it("addJob wakes the check loop so a new monitor's first check runs promptly", async () => {
|
||||
const { worker, mocks } = await start({ queueMode: "worker" });
|
||||
await jest.advanceTimersByTimeAsync(15000); // backed off to the cap
|
||||
const before = claims(mocks, "check");
|
||||
|
||||
await worker.addJob("m2", makeMonitor({ id: "m2" }));
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(claims(mocks, "check")).toBe(before + 1);
|
||||
});
|
||||
|
||||
it("resumeJob wakes the check loop so a resumed monitor runs promptly", async () => {
|
||||
const { worker, mocks } = await start({ queueMode: "worker" });
|
||||
await jest.advanceTimersByTimeAsync(15000); // backed off to the cap
|
||||
const before = claims(mocks, "check");
|
||||
|
||||
await worker.resumeJob(makeMonitor());
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(claims(mocks, "check")).toBe(before + 1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── shutdown ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("shutdown", () => {
|
||||
|
||||
@@ -3,6 +3,7 @@ import { MaintenanceWindowService } from "../../../src/domain/maintenance-window
|
||||
import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts";
|
||||
import type { IWorker } from "../../../src/worker/worker.interface.ts";
|
||||
import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
@@ -33,16 +34,23 @@ const createJobsRepo = () =>
|
||||
markMonitorsDue: jest.fn().mockResolvedValue(0),
|
||||
}) as unknown as jest.Mocked<IJobsRepository>;
|
||||
|
||||
const createWorker = () =>
|
||||
({
|
||||
wake: jest.fn(),
|
||||
}) as unknown as jest.Mocked<IWorker>;
|
||||
|
||||
const createService = (overrides?: {
|
||||
monitorsRepository?: ReturnType<typeof createMonitorsRepo>;
|
||||
maintenanceWindowsRepository?: ReturnType<typeof createMaintenanceWindowsRepo>;
|
||||
jobsRepository?: ReturnType<typeof createJobsRepo>;
|
||||
worker?: ReturnType<typeof createWorker>;
|
||||
}) => {
|
||||
const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo();
|
||||
const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo();
|
||||
const jobsRepository = overrides?.jobsRepository ?? createJobsRepo();
|
||||
const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository });
|
||||
return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository };
|
||||
const worker = overrides?.worker ?? createWorker();
|
||||
const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker });
|
||||
return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker };
|
||||
};
|
||||
|
||||
const makeWindow = (overrides?: Partial<MaintenanceWindow>): MaintenanceWindow => ({
|
||||
@@ -275,7 +283,7 @@ describe("MaintenanceWindowService", () => {
|
||||
it("flips covered monitors to initializing when deleting an active window", async () => {
|
||||
const maintenanceWindowsRepository = createMaintenanceWindowsRepo();
|
||||
(maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] }));
|
||||
const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository });
|
||||
const { service, monitorsRepository, jobsRepository, worker } = createService({ maintenanceWindowsRepository });
|
||||
|
||||
await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" });
|
||||
|
||||
@@ -283,6 +291,19 @@ describe("MaintenanceWindowService", () => {
|
||||
expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]);
|
||||
// Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval
|
||||
expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number));
|
||||
// ...and the idle loops are woken so the re-armed jobs run promptly
|
||||
expect(worker.wake).toHaveBeenCalledWith("check");
|
||||
expect(worker.wake).toHaveBeenCalledWith("geo-check");
|
||||
});
|
||||
|
||||
it("does not wake the loops when no monitor leaves maintenance", async () => {
|
||||
const maintenanceWindowsRepository = createMaintenanceWindowsRepo();
|
||||
(maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeWindow({ active: false, monitorIds: ["mon-1"] }));
|
||||
const { service, worker } = createService({ maintenanceWindowsRepository });
|
||||
|
||||
await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" });
|
||||
|
||||
expect(worker.wake).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("excludes the deleted window from the overlap check", async () => {
|
||||
|
||||
@@ -7,21 +7,15 @@ import type { Check } from "../../../src/domain/checks/check.type.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts";
|
||||
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
|
||||
import type { IBufferService } from "../../../src/service/bufferService.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const createBuffer = (): jest.Mocked<Pick<IBufferService, "addToBuffer">> => ({
|
||||
addToBuffer: jest.fn(),
|
||||
});
|
||||
|
||||
const createMonitorsRepo = () => {
|
||||
const findById = jest.fn();
|
||||
const updateStatusWindowAndChecks = jest
|
||||
.fn()
|
||||
.mockImplementation(
|
||||
(
|
||||
async (
|
||||
_id: unknown,
|
||||
_tid: unknown,
|
||||
status: boolean,
|
||||
@@ -30,27 +24,23 @@ const createMonitorsRepo = () => {
|
||||
maxRecentChecks: number,
|
||||
statusPatch?: Partial<Monitor>
|
||||
) => {
|
||||
// Grab the monitor that findById was configured to resolve with
|
||||
const monitor = findById.mock.results.at(-1)?.value;
|
||||
if (monitor && typeof monitor.then === "function") {
|
||||
return monitor.then((m: any) => {
|
||||
m.statusWindow = m.statusWindow || [];
|
||||
m.statusWindow.push(status);
|
||||
while (m.statusWindow.length > windowSize) {
|
||||
m.statusWindow.shift();
|
||||
}
|
||||
m.recentChecks = m.recentChecks || [];
|
||||
m.recentChecks.push(checkSnapshot);
|
||||
while (m.recentChecks.length > maxRecentChecks) {
|
||||
m.recentChecks.shift();
|
||||
}
|
||||
if (statusPatch) {
|
||||
Object.assign(m, statusPatch);
|
||||
}
|
||||
return { ...m };
|
||||
});
|
||||
// updateMonitorStatus no longer reads the monitor itself — tests seed it via
|
||||
// findById.mockResolvedValue, so the mock resolves it here as the post-write monitor.
|
||||
const m: any = await findById();
|
||||
m.statusWindow = m.statusWindow || [];
|
||||
m.statusWindow.push(status);
|
||||
while (m.statusWindow.length > windowSize) {
|
||||
m.statusWindow.shift();
|
||||
}
|
||||
return Promise.reject(new Error("findById not mocked"));
|
||||
m.recentChecks = m.recentChecks || [];
|
||||
m.recentChecks.push(checkSnapshot);
|
||||
while (m.recentChecks.length > maxRecentChecks) {
|
||||
m.recentChecks.shift();
|
||||
}
|
||||
if (statusPatch) {
|
||||
Object.assign(m, statusPatch);
|
||||
}
|
||||
return { ...m };
|
||||
}
|
||||
);
|
||||
return {
|
||||
@@ -67,23 +57,17 @@ const createMonitorStatsRepo = () =>
|
||||
updateByMonitorId: jest.fn(),
|
||||
}) as unknown as jest.Mocked<IMonitorStatsRepository>;
|
||||
|
||||
const createChecksRepo = () => ({}) as unknown as jest.Mocked<IChecksRepository>;
|
||||
|
||||
const createService = (overrides?: {
|
||||
logger?: ReturnType<typeof createMockLogger>;
|
||||
buffer?: ReturnType<typeof createBuffer>;
|
||||
monitorsRepository?: ReturnType<typeof createMonitorsRepo>;
|
||||
monitorStatsRepository?: ReturnType<typeof createMonitorStatsRepo>;
|
||||
checksRepository?: ReturnType<typeof createChecksRepo>;
|
||||
}) => {
|
||||
const logger = overrides?.logger ?? createMockLogger();
|
||||
const buffer = overrides?.buffer ?? createBuffer();
|
||||
const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo();
|
||||
const monitorStatsRepository = overrides?.monitorStatsRepository ?? createMonitorStatsRepo();
|
||||
const checksRepository = overrides?.checksRepository ?? createChecksRepo();
|
||||
|
||||
const service = new StatusService(logger as any, buffer as any, monitorsRepository, monitorStatsRepository, checksRepository);
|
||||
return { service, logger, buffer, monitorsRepository, monitorStatsRepository, checksRepository };
|
||||
const service = new StatusService(logger as any, monitorsRepository, monitorStatsRepository);
|
||||
return { service, logger, monitorsRepository, monitorStatsRepository };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
@@ -264,19 +248,31 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck());
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.prevStatus).toBe("up");
|
||||
});
|
||||
|
||||
it("uses the prefetched monitor and skips findById when one is passed", async () => {
|
||||
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" });
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor);
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
|
||||
|
||||
expect(monitorsRepository.findById).not.toHaveBeenCalled();
|
||||
expect(result.prevStatus).toBe("up");
|
||||
expect(result.monitor).toBe(monitor);
|
||||
});
|
||||
|
||||
it("pushes to statusWindow and trims to statusWindowSize", async () => {
|
||||
const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5 });
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
// Atomic push should have been called with the correct status and window size
|
||||
expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith(
|
||||
@@ -297,7 +293,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }));
|
||||
await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }), monitor);
|
||||
|
||||
expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS);
|
||||
expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" }));
|
||||
@@ -315,7 +311,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -333,7 +329,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
@@ -350,7 +346,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck());
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
@@ -370,7 +366,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
@@ -387,7 +383,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -399,7 +395,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
await service.updateMonitorStatus(makeStatusResponse(), makeCheck());
|
||||
await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
|
||||
|
||||
expect(monitor.statusWindow).toEqual([true]);
|
||||
});
|
||||
@@ -410,23 +406,25 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
await service.updateMonitorStatus(makeStatusResponse(), makeCheck());
|
||||
await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
|
||||
|
||||
expect(monitor.recentChecks).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("throws AppError when repository throws", async () => {
|
||||
const monitor = makeMonitor();
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockRejectedValue(new Error("db error"));
|
||||
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error"));
|
||||
|
||||
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Failed to update monitor");
|
||||
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Failed to update monitor");
|
||||
});
|
||||
|
||||
it("throws AppError with 'Unknown error' for non-Error exceptions", async () => {
|
||||
const monitor = makeMonitor();
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockRejectedValue("string error");
|
||||
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error");
|
||||
|
||||
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Unknown error");
|
||||
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Unknown error");
|
||||
});
|
||||
|
||||
it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => {
|
||||
@@ -444,7 +442,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -462,7 +460,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -484,7 +482,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
|
||||
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
@@ -502,7 +500,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
@@ -518,7 +516,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
@@ -535,7 +533,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -550,7 +548,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -566,7 +564,7 @@ describe("StatusService", () => {
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
@@ -581,7 +579,7 @@ describe("StatusService", () => {
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -601,7 +599,7 @@ describe("StatusService", () => {
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
|
||||
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
@@ -616,7 +614,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -629,7 +627,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down"));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck());
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
|
||||
|
||||
expect(result.monitor.status).toBe("up");
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
@@ -647,7 +645,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck());
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck(), monitor);
|
||||
|
||||
expect(result.code).toBe(201);
|
||||
expect(result.timestamp).toBeGreaterThan(0);
|
||||
@@ -688,7 +686,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} } } as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.cpu).toBe(true);
|
||||
expect(result.thresholdBreaches?.memory).toBe(false);
|
||||
@@ -705,7 +703,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} } } as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.memory).toBe(true);
|
||||
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
||||
@@ -722,7 +720,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.disk).toBe(true);
|
||||
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
||||
@@ -739,7 +737,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.temp).toBe(true);
|
||||
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
||||
@@ -761,7 +759,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} },
|
||||
} as any);
|
||||
await service.updateMonitorStatus(response, makeCheck());
|
||||
await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
||||
expect(patch.cpuAlertCounter).toBe(5);
|
||||
@@ -780,7 +778,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
expect(result.statusChanged).toBe(false);
|
||||
@@ -795,7 +793,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
@@ -818,7 +816,7 @@ describe("StatusService", () => {
|
||||
status: false,
|
||||
data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
@@ -830,7 +828,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse({ data: { cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.cpu).toBe(false);
|
||||
});
|
||||
@@ -842,7 +840,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} } } as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.memory).toBe(false);
|
||||
});
|
||||
@@ -856,7 +854,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.temp).toBe(false);
|
||||
});
|
||||
@@ -870,7 +868,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.temp).toBe(false);
|
||||
});
|
||||
@@ -882,7 +880,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse(undefined as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches).toBeUndefined();
|
||||
});
|
||||
@@ -894,7 +892,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse({} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches).toBeUndefined();
|
||||
});
|
||||
@@ -909,7 +907,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck());
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches).toBeUndefined();
|
||||
});
|
||||
@@ -923,7 +921,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.disk).toBe(false);
|
||||
});
|
||||
@@ -937,7 +935,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.disk).toBe(false);
|
||||
});
|
||||
@@ -951,7 +949,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.cpu).toBe(false);
|
||||
expect(result.thresholdBreaches?.temp).toBe(false);
|
||||
@@ -966,7 +964,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.memory).toBe(false);
|
||||
});
|
||||
@@ -980,7 +978,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(result.thresholdBreaches?.disk).toBe(false);
|
||||
});
|
||||
@@ -994,7 +992,7 @@ describe("StatusService", () => {
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck());
|
||||
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
||||
expect(patch.cpuAlertCounter).toBe(2);
|
||||
@@ -1020,7 +1018,7 @@ describe("StatusService", () => {
|
||||
status: false,
|
||||
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
|
||||
} as any);
|
||||
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }));
|
||||
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor);
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
@@ -1040,7 +1038,7 @@ describe("StatusService", () => {
|
||||
const response = makeHardwareResponse({
|
||||
data: { cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {} },
|
||||
} as any);
|
||||
await service.updateMonitorStatus(response, makeCheck());
|
||||
await service.updateMonitorStatus(response, makeCheck(), monitor);
|
||||
|
||||
expect(monitor.cpuAlertCounter).toBe(0);
|
||||
expect(monitor.memoryAlertCounter).toBe(0);
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { WorkerHelper } from "../../../src/worker/worker.helper.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// WorkerHelper is now thin glue: it wires the scheduler's job functions to the
|
||||
// CheckPipeline / GeoChecksPipeline (acquire→record→decide) and the
|
||||
// ReactorDispatcher (react), and owns the cleanup jobs. The pipeline and
|
||||
// dispatcher internals are tested in checkPipeline.test.ts / reactorDispatcher.test.ts;
|
||||
// WorkerHelper is now thin glue: it wires the geo job to the GeoChecksPipeline and
|
||||
// owns the cleanup jobs. The pipeline internals are tested in checkPipeline.test.ts;
|
||||
// here we test only the glue + cleanup jobs.
|
||||
|
||||
const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
@@ -42,10 +39,6 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
deleteOlderThan: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
|
||||
const checkPipeline = { run: jest.fn().mockResolvedValue(null) };
|
||||
const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) };
|
||||
const reactorDispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) };
|
||||
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
checkService,
|
||||
@@ -57,9 +50,6 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
checksRepository,
|
||||
incidentsRepository,
|
||||
geoChecksRepository,
|
||||
reactorDispatcher,
|
||||
checkPipeline,
|
||||
geoCheckPipeline,
|
||||
...overrides,
|
||||
};
|
||||
|
||||
@@ -73,26 +63,11 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
defaults.monitorStatsRepository as any,
|
||||
defaults.checksRepository as any,
|
||||
defaults.incidentsRepository as any,
|
||||
defaults.geoChecksRepository as any,
|
||||
defaults.reactorDispatcher as any,
|
||||
defaults.checkPipeline as any,
|
||||
defaults.geoCheckPipeline as any
|
||||
defaults.geoChecksRepository as any
|
||||
);
|
||||
return { helper, defaults };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
geoCheckEnabled: false,
|
||||
geoCheckLocations: [],
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("WorkerHelper", () => {
|
||||
@@ -103,68 +78,6 @@ describe("WorkerHelper", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── getHeartbeatJob (glue) ────────────────────────────────────────────────
|
||||
|
||||
describe("getHeartbeatJob", () => {
|
||||
it("runs the pipeline and dispatches the evaluation to reactors", async () => {
|
||||
const evaluation = { monitor: { id: "m1" }, decision: {} };
|
||||
const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(evaluation) } });
|
||||
|
||||
await helper.getHeartbeatJob()(makeMonitor());
|
||||
|
||||
expect(defaults.checkPipeline.run).toHaveBeenCalled();
|
||||
expect(defaults.reactorDispatcher.dispatch).toHaveBeenCalledWith(evaluation);
|
||||
});
|
||||
|
||||
it("does not dispatch when the pipeline returns null (skipped)", async () => {
|
||||
const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(null) } });
|
||||
|
||||
await helper.getHeartbeatJob()(makeMonitor());
|
||||
|
||||
expect(defaults.reactorDispatcher.dispatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs and rethrows when the pipeline throws", async () => {
|
||||
const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue(new Error("boom")) } });
|
||||
|
||||
await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toThrow("boom");
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "boom" }));
|
||||
});
|
||||
|
||||
it("rethrows non-Error failures", async () => {
|
||||
const { helper } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue("unexpected") } });
|
||||
|
||||
await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toBe("unexpected");
|
||||
});
|
||||
});
|
||||
|
||||
// ── getHeartbeatGeoJob (glue) ─────────────────────────────────────────────
|
||||
|
||||
describe("getHeartbeatGeoJob", () => {
|
||||
it("runs the geo pipeline", async () => {
|
||||
const { helper, defaults } = createHelper();
|
||||
|
||||
await helper.getHeartbeatGeoJob()(makeMonitor());
|
||||
|
||||
expect(defaults.geoCheckPipeline.run).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("catches and logs a pipeline error without rethrowing", async () => {
|
||||
const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue(new Error("api timeout")) } });
|
||||
|
||||
await expect(helper.getHeartbeatGeoJob()(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "api timeout" }));
|
||||
});
|
||||
|
||||
it("logs 'Unknown error' for non-Error failures", async () => {
|
||||
const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue("string error") } });
|
||||
|
||||
await helper.getHeartbeatGeoJob()(makeMonitor());
|
||||
|
||||
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── getCleanupOrphanedJob ────────────────────────────────────────────────
|
||||
|
||||
describe("getCleanupOrphanedJob", () => {
|
||||
|
||||
Reference in new issue
Block a user