Merge pull request #3728 from bluewave-labs/feat/db-queue

feat: db queue
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-06-17 09:48:40 -07:00
commit 8277c4ed13
42 files changed
+3499 -3514

No files matched your search

+2 -1
View File
@@ -11,7 +11,8 @@ LOG_LEVEL=debug
# Database
# --------
DB_CONNECTION_STRING=mongodb://localhost:27017/uptime_db
QUEUE_TYPE=superSimpleQueue
QUEUE_MODE="primary"
QUEUE_PRIMARY_PROCESSES="true"
# JWT Authentication
# ------------------
+1 -1
View File
@@ -1,5 +1,5 @@
{
"ignore": ["node_modules/*"],
"watch": ["src/**/*.ts", "src/**/*.js", "*.json"],
"watch": ["src/**/*.ts", "src/**/*.js", "*.json", ".env"],
"ext": "ts,js,json"
}
+1181 -988
View File
File diff suppressed because it is too large. Load diff
+1 -4
View File
@@ -33,7 +33,6 @@
"dependencies": {
"@grpc/grpc-js": "^1.14.3",
"@grpc/proto-loader": "^0.8.0",
"@pulsecron/pulse": "1.6.8",
"axios": "^1.15.2",
"bcryptjs": "3.0.2",
"cacheable-lookup": "7.0.0",
@@ -48,12 +47,10 @@
"got": "14.4.7",
"handlebars": "^4.7.8",
"helmet": "^8.0.0",
"ioredis": "^5.4.2",
"isomorphic-dompurify": "^2.26.0",
"jmespath": "^0.16.0",
"jsdom": "^26.1.0",
"jsonwebtoken": "9.0.2",
"less-simple-scheduler": "2.6.0",
"mailersend": "^2.2.0",
"mjml": "^5.0.0-alpha.4",
"mongoose": "^8.3.3",
@@ -62,7 +59,6 @@
"ping": "0.4.4",
"sharp": "0.33.5",
"ssl-checker": "2.0.10",
"super-simple-scheduler": "1.9.2",
"swagger-ui-express": "5.0.1",
"winston": "^3.13.0",
"ws": "^8.19.0",
@@ -96,6 +92,7 @@
"globals": "^15.14.0",
"jest": "^30.2.0",
"lint-staged": "^16.2.7",
"mongodb-memory-server": "11.2.0",
"nodemon": "^3.1.11",
"prettier": "^3.3.3",
"prettier-plugin-ini": "1.3.0",
+3 -2
View File
@@ -1,6 +1,6 @@
import mongoose from "mongoose";
import { MonitorModel } from "../dist/db/models/Monitor.js";
import { UserModel } from "../dist/db/models/User.js";
import { MonitorModel } from "../dist/domain/monitors/monitor.model.js";
import { UserModel } from "../dist/domain/users/user.model.js";
// Seeds a large number of HTTP monitors against real, resolving domains so the
// scheduler can be exercised under load. Run after `npm run build` (imports the
@@ -241,6 +241,7 @@ async function run() {
url: `https://${domain}`,
isActive: true,
interval,
lastEvaluatedAt: 0, // evaluator watermark; without it findUnevaluatedByMonitorId gets new Date(undefined)
});
if (docs.length === batchSize) {
@@ -272,5 +272,6 @@ export const monitorResponseSchema = z
userId: z.string(),
createdAt: z.string(),
updatedAt: z.string(),
lastEvaluatedAt: z.number(),
})
.passthrough();
+2 -2
View File
@@ -1,5 +1,5 @@
import { z } from "zod";
import { DbTypes, QueueTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
import { booleanCoercion } from "@/api/validation/shared.js";
const envSchema = z.object({
@@ -12,8 +12,8 @@ const envSchema = z.object({
DB_CONNECTION_STRING: z.string().min(1, "Database connection string is required"),
DB_TYPE: z.enum(DbTypes).default("mongodb"),
QUEUE_TYPE: z.enum(QueueTypes).default("superSimpleQueue"),
QUEUE_MODE: z.enum(QueueModes).default("primary"),
QUEUE_PRIMARY_PROCESSES: booleanCoercion.default(true),
// JWT Authentication
JWT_SECRET: z.string().min(1, "JWT_SECRET is required"),
+32 -86
View File
@@ -1,4 +1,6 @@
import { Mongoose } from "mongoose";
import { hostname } from "node:os";
import { randomUUID } from "node:crypto";
import MongoDB from "../db/db.mongo.js";
import { IDb } from "@/db/db.interface.js";
import { EnvConfig, ISettingsService, SettingsService } from "@/domain/app-settings/app-settings.service.js";
@@ -30,8 +32,6 @@ import { EmailService, IEmailService } from "@/service/emailService.js";
import { GlobalPingService } from "@/service/globalPingService.js";
import { WorkerHelper } from "@/worker/worker.helper.js";
import { IWorker } from "@/worker/worker.interface.js";
import { LessSimpleWorker } from "@/worker/worker.less-simple.js";
import { SuperSimpleQueue } from "@/worker/worker.super-simple.js";
import { INetworkService, NetworkService } from "@/service/networkService.js";
import { IStatusService, StatusService } from "@/service/statusService.js";
import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
@@ -89,7 +89,6 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js";
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js";
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js";
import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js";
import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js";
@@ -101,12 +100,15 @@ import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
import { ILogger } from "@/utils/logger.js";
import { AppError } from "@/utils/AppError.js";
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
import { CheckProducer } from "@/worker/worker.check-producer.js";
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js";
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
export type InitializedServices = {
settingsService: ISettingsService;
@@ -151,12 +153,10 @@ export const initializeServices = async ({
logger,
envSettings,
settingsService,
queueMode = "primary",
}: {
logger: ILogger;
envSettings: EnvConfig;
settingsService: ISettingsService;
queueMode?: QueueMode;
}): Promise<InitializedServices> => {
// Create DB
@@ -164,58 +164,6 @@ export const initializeServices = async ({
db = new MongoDB(logger, envSettings);
await db.connect();
// ** NOTE **
// const dbType = envSettings.dbType;
// DB type has been fixed to MongoDB for now
// if (dbType === "mongodb") {
// db = new MongoDB(logger, envSettings);
// }
// if (!db) {
// throw new AppError({ message: "Unsupported database type", status: 500 });
// }
// await db.connect();
// Repositories
// ** NOTE **
// DB Type fixed to MongoDB for now
// let monitorsRepository: IMonitorsRepository;
// let checksRepository: IChecksRepository;
// let geoChecksRepository: IGeoChecksRepository;
// let monitorStatsRepository: IMonitorStatsRepository;
// let statusPagesRepository: IStatusPagesRepository;
// let usersRepository: IUsersRepository;
// let invitesRepository: IInvitesRepository;
// let recoveryTokensRepository: IRecoveryTokensRepository;
// let settingsRepository: ISettingsRepository;
// let notificationsRepository: INotificationsRepository;
// let tagsRepository: ITagsRepository;
// let incidentsRepository: IIncidentsRepository;
// let teamsRepository: ITeamsRepository;
// let maintenanceWindowsRepository: IMaintenanceWindowsRepository;
// if (dbType === "mongodb") {
// monitorsRepository = new MongoMonitorsRepository();
// checksRepository = new MongoChecksRepository(logger);
// geoChecksRepository = new MongoGeoChecksRepository(logger);
// monitorStatsRepository = new MongoMonitorStatsRepository();
// statusPagesRepository = new MongoStatusPagesRepository();
// usersRepository = new MongoUsersRepository();
// invitesRepository = new MongoInvitesRepository();
// recoveryTokensRepository = new MongoRecoveryTokensRepository();
// settingsRepository = new MongoSettingsRepository();
// notificationsRepository = new MongoNotificationsRepository();
// tagsRepository = new MongoTagsRepository();
// incidentsRepository = new MongoIncidentsRepository();
// teamsRepository = new MongoTeamsRepository();
// maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
// } else {
// throw new AppError({ message: "Unsupported database type", status: 500 });
// }
const monitorsRepository = new MongoMonitorsRepository();
const checksRepository = new MongoChecksRepository(logger);
const geoChecksRepository = new MongoGeoChecksRepository(logger);
@@ -230,6 +178,9 @@ export const initializeServices = async ({
const incidentsRepository = new MongoIncidentsRepository();
const teamsRepository = new MongoTeamsRepository();
const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
// One identity per process
const workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
const jobsRepository = new MongoJobsRepository(workerId);
const queueWorkersRepository = new MongoQueueWorkersRepository();
// Inject settings repository into settings service (now that DB is connected)
@@ -320,17 +271,12 @@ export const initializeServices = async ({
const incidentReactor = new IncidentReactor(incidentService);
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
// Check producer/evaluator
const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger);
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
// pipelines
const checkPipeline = new CheckPipeline(
monitorsRepository,
maintenanceWindowsRepository,
checkService,
networkService,
bufferService,
monitorStatusPolicy,
statusService,
logger
);
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
@@ -339,6 +285,7 @@ export const initializeServices = async ({
checkService,
settingsService,
monitorsRepository,
jobsRepository,
teamsRepository,
monitorStatsRepository,
checksRepository,
@@ -349,23 +296,22 @@ export const initializeServices = async ({
geoCheckPipeline
);
if (queueMode === "worker" && envSettings.queueType !== "lessSimpleQueue") {
throw new AppError({
message: `Worker mode requires QUEUE_TYPE="lessSimpleQueue" got ${envSettings.queueType}`,
status: 500,
service: "config services",
method: "initializeServices",
});
}
let worker: IWorker;
switch (envSettings.queueType) {
case "lessSimpleQueue":
worker = await LessSimpleWorker.create(logger, workerHelper, monitorsRepository, queueWorkersRepository, envSettings, queueMode);
break;
default:
worker = await SuperSimpleQueue.create(logger, workerHelper, monitorsRepository);
}
const worker = await DBQueueWorker.create(
logger,
jobsRepository,
monitorsRepository,
checksRepository,
checkService,
checkProducer,
checkEvaluator,
geoCheckPipeline,
reactorDispatcher,
workerHelper,
queueWorkersRepository,
envSettings.queueMode,
envSettings.queuePrimaryProcesses,
workerId
);
// Business services
const userService = new UserService({
@@ -0,0 +1,41 @@
import mongoose from "mongoose";
import { MonitorModel } from "../../domain/monitors/monitor.model.js";
import { logger } from "@/utils/logger.js";
const JOBS_COLLECTION = "jobs";
/**
* Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed.
*
* Stamp the upgrade moment, not 0. If set to 0, all checks will be replayed
*/
export async function backfillMonitorLastEvaluatedAt(): Promise<void> {
const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt";
try {
// Drop the legacy jobs collection first. The old super-simple-scheduler persisted jobs to the
// same "jobs" collection the new db-queue uses, so any leftover rows carry an incompatible schema.
// Migrations run before syncIndexes(), so the new Job indexes are rebuilt right after this drop.
const db = mongoose.connection.db;
if (db) {
const existing = await db.listCollections({ name: JOBS_COLLECTION }).toArray();
if (existing.length > 0) {
await db.collection(JOBS_COLLECTION).drop();
logger.info({ service: SERVICE_NAME, message: `Dropped legacy "${JOBS_COLLECTION}" collection` });
}
}
logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" });
const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: Date.now() } });
logger.info({
service: SERVICE_NAME,
message: `Backfilled lastEvaluatedAt on ${result.modifiedCount} monitors`,
});
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.error({ service: SERVICE_NAME, message: `Error backfilling lastEvaluatedAt: ${errorMessage}` });
throw error;
}
}
+2
View File
@@ -6,6 +6,7 @@ import MigrationModel from "../../domain/migrations/migration.model.js";
import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js";
import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js";
import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js";
import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js";
import type { ILogger } from "@/utils/logger.js";
type MigrationEntry = {
@@ -21,6 +22,7 @@ const migrations: MigrationEntry[] = [
{ name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray },
{ name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex },
{ name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray },
{ name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt },
];
const runMigrations = async (logger?: ILogger) => {
@@ -1,5 +1,5 @@
import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js";
import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { AppError } from "@/utils/AppError.js";
import { ValidatedEnv } from "@/config/envValidation.js";
import type { StringValue } from "ms";
@@ -13,8 +13,8 @@ export type EnvConfig = {
clientHost: string;
dbConnectionString: string;
dbType: DbType;
queueType: QueueType;
queueMode: QueueMode;
queuePrimaryProcesses: boolean;
statusPageThemesEnabled: boolean;
};
@@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService {
clientHost: env.CLIENT_HOST,
dbConnectionString: env.DB_CONNECTION_STRING,
dbType: env.DB_TYPE,
queueType: env.QUEUE_TYPE,
queueMode: env.QUEUE_MODE,
queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES,
statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED,
};
}
@@ -1,9 +1,6 @@
export const DbTypes = ["mongodb"] as const;
export type DbType = (typeof DbTypes)[number];
export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const;
export type QueueType = (typeof QueueTypes)[number];
export const QueueModes = ["primary", "worker"] as const;
export type QueueMode = (typeof QueueModes)[number];
@@ -42,6 +42,7 @@ export interface IChecksRepository {
options?: { type?: MonitorType }
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult>;
findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
// update
//delete
deleteByMonitorId(monitorId: string): Promise<number>;
@@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository {
};
};
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
const docs = await CheckModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
createdAt: { $gt: new Date(since) },
})
.sort({ createdAt: 1 })
.lean<CheckDocument[]>();
return docs.map(this.toEntity);
};
deleteByMonitorId = async (monitorId: string): Promise<number> => {
const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
return result.deletedCount;
+27 -3
View File
@@ -11,7 +11,9 @@ const SERVICE_NAME = "checkService";
export interface ICheckService {
createChecks(checks: Check[]): Promise<Check[]>;
buildCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toLastEvaluatedAt(check: Check): number;
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
getChecksByMonitor(params: {
monitorId: string;
teamId: string;
@@ -56,7 +58,11 @@ export class CheckService implements ICheckService {
return this.checksRepository.createChecks(checks);
};
buildCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
toLastEvaluatedAt = (check: Check) => {
return new Date(check.createdAt).getTime();
};
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
const now = new Date().toISOString();
@@ -82,7 +88,7 @@ export class CheckService implements ICheckService {
this.logger.warn({
message: "Failed to build check",
service: SERVICE_NAME,
method: "buildCheck",
method: "toCheck",
});
return undefined;
}
@@ -131,6 +137,24 @@ export class CheckService implements ICheckService {
return check;
};
toStatusResponse = (check: Check) => {
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
monitorId: check.metadata.monitorId,
teamId: check.metadata.teamId,
type: check.metadata.type,
status: check.status,
code: check.statusCode,
message: check.message,
responseTime: check.responseTime,
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
payload:
check.metadata.type === "hardware"
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
: undefined,
};
return statusResponse;
};
getChecksByMonitor = async ({
monitorId,
teamId,
+79
View File
@@ -0,0 +1,79 @@
import { Schema, model } from "mongoose";
import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js";
export interface JobDocument extends Omit<Job, "id"> {
_id: string;
}
const JobSchema = new Schema<JobDocument>(
{
_id: {
type: String,
required: true,
},
type: {
type: String,
enum: JOB_TYPES,
required: true,
},
// monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate).
refId: {
type: String,
default: null,
},
// paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob)
isActive: {
type: Boolean,
default: true,
},
// Scheduling — epoch ms, so the claim/$min queries operate on plain numbers
nextScheduledAt: {
type: Number,
required: true,
},
intervalMs: {
type: Number,
default: null,
},
// Lock / lease
lockedBy: {
type: String,
default: null,
},
lockedUntil: {
type: Number,
default: null,
},
// Observability
runCount: {
type: Number,
default: 0,
},
failCount: {
type: Number,
default: 0,
},
lastFinishedAt: {
type: Number,
default: null,
},
lastFailReason: {
type: String,
default: null,
},
},
{ timestamps: true }
);
// Used for scanning for claims
JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 });
// for findById, upsertEvaluate, deleteById
JobSchema.index({ refId: 1, type: 1 });
const JobModel = model<JobDocument>("Job", JobSchema);
export { JobModel };
export default JobModel;
@@ -0,0 +1,72 @@
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
export type JobPageQuery = {
page?: number;
rowsPerPage?: number;
};
export type JobPage = {
jobs: Job[];
count: number;
};
export interface IJobsRepository {
// ********************
// Claims
// ********************
// Atomically claim a job
claimDue(type: JobType, now: number): Promise<Job | null>;
// locks up to `limit` due rows in one batch
claimDueBatch(type: JobType, limit: number, now: number): Promise<Job[]>;
// Extends the locks we still hold. Returns how many were renewed
renewLocks(ids: string[], now: number): Promise<number>;
// Completes a repeating job
recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise<boolean>;
// Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry
recordFailure(id: string, error: unknown, now: number): Promise<boolean>;
recordOneShot(id: string, now: number): Promise<boolean>;
// ********************
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
// ********************
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
// ********************
// Job CRUD
// ********************
// Create jobs. Takes a JobSeed
upsertJob(job: JobSeed): Promise<boolean>;
// Creates cleanup jobs, these are executed immediately so need their own method
upsertCleanupJob(job: JobSeed): Promise<boolean>;
// Pause/Resume
setActiveById(refId: string, isActive: boolean): Promise<boolean>;
// Edit job schedule
updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
// Delete, drop all rows
deleteById(refId: string): Promise<boolean>;
// Delete a single job of type
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
// Delete jobs that reference monitors that no longer exist
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
// Get job
findById(refId: string): Promise<Job[]>;
// ********************
// Observability
// ********************
findPage(pagination: JobPageQuery): Promise<JobPage>;
findAll(): Promise<Job[]>;
}
@@ -0,0 +1,287 @@
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
class MongoJobsRepository implements IJobsRepository {
constructor(private readonly workerId: string) {}
private toEntity(doc: JobDocument): Job {
return {
id: doc._id,
type: doc.type,
refId: doc.refId,
isActive: doc.isActive,
nextScheduledAt: doc.nextScheduledAt,
intervalMs: doc.intervalMs,
lockedBy: doc.lockedBy,
lockedUntil: doc.lockedUntil,
runCount: doc.runCount,
failCount: doc.failCount,
lastFinishedAt: doc.lastFinishedAt,
lastFailReason: doc.lastFailReason,
};
}
claimDue = async (type: JobType, now: number) => {
const doc = await JobModel.findOneAndUpdate(
{
type,
isActive: true,
nextScheduledAt: { $lte: now }, // Find due jobs
$or: [
{ lockedUntil: null }, // Not locked
{ lockedUntil: { $lt: now } }, // Free or expired lock
],
},
{
$set: {
lockedBy: this.workerId,
lockedUntil: now + LOCK_MS,
},
},
{
new: true,
sort: { nextScheduledAt: 1 }, // Get the oldest due job
}
).lean<JobDocument>();
return doc ? this.toEntity(doc) : null;
};
claimDueBatch = async (type: JobType, limit: number, now: number) => {
if (limit <= 0) return [];
const dueAndFree = {
type,
isActive: true,
nextScheduledAt: { $lte: now }, // due
$or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock
};
// 1. Pick the oldest-due candidates
const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>();
if (candidates.length === 0) return [];
const ids = candidates.map((d) => d._id);
// 2. lock them. Re-check free/expired so we never steal another worker's live lock
// only the rows this updateMany actually flips end up with claimed
const lockedUntil = now + LOCK_MS;
await JobModel.updateMany(
{ _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] },
{ $set: { lockedBy: this.workerId, lockedUntil } }
);
// 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time
const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean<JobDocument[]>();
return docs.map(this.toEntity);
};
renewLocks = async (ids: string[], now: number) => {
if (ids.length === 0) return 0;
const res = await JobModel.updateMany(
{ _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed
{ $set: { lockedUntil: now + LOCK_MS } }
);
return res.modifiedCount;
};
recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => {
// Fixed rate job scheduling
let nextScheduledAt = scheduledRunAt + intervalMs;
// If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd
if (nextScheduledAt <= now) {
nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs);
}
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
},
$inc: {
runCount: 1, // Increment run count
},
}
);
return res.modifiedCount === 1;
};
recordFailure = async (id: string, error: unknown, now: number) => {
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt: now + BACKOFF_MS,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFailReason: error instanceof Error ? error.message : String(error),
},
$inc: {
failCount: 1, // Increment fail count
},
}
);
return res.modifiedCount === 1;
};
recordOneShot = async (id: string, now: number) => {
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt: PARKED,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
},
$inc: {
runCount: 1, // Increment run count
},
}
);
return res.modifiedCount === 1;
};
upsertEvaluate = async (monitorId: string, now: number) => {
const filter = { type: "evaluate", refId: monitorId };
const update = {
$setOnInsert: {
_id: jobId("evaluate", monitorId),
// refId and Type inferred from filter
isActive: true,
intervalMs: null,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
$min: {
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
},
};
try {
const res = await JobModel.updateOne(filter, update, { upsert: true });
return res.acknowledged;
} catch (error) {
// Two callers can race the first insert for the same monitor, so the
// loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min.
if (error && typeof error === "object" && (error as { code?: number }).code === 11000) {
const res = await JobModel.updateOne(filter, update, { upsert: true });
return res.acknowledged;
}
throw error;
}
};
upsertJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
{
$set: {
// These fields are owned by the monitor, always safe to set
type: job.type,
refId: job.refId,
isActive: job.isActive,
intervalMs: job.intervalMs,
},
$setOnInsert: {
// These fields are scheduling related, only set on insert so they aren't clobbered
nextScheduledAt: job.nextScheduledAt,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
},
{ upsert: true }
);
return res.acknowledged;
};
upsertCleanupJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
{
$set: {
type: job.type,
refId: job.refId,
isActive: job.isActive,
intervalMs: job.intervalMs,
nextScheduledAt: job.nextScheduledAt,
},
$setOnInsert: {
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
},
{ upsert: true }
);
return res.acknowledged;
};
setActiveById = async (refId: string, isActive: boolean) => {
const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together
return res.modifiedCount > 0;
};
updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => {
const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } });
return res.modifiedCount === 1;
};
deleteById = async (refId: string) => {
const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
return res.deletedCount > 0;
};
deleteByIdAndType = async (refId: string, type: JobType) => {
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
return res.deletedCount > 0;
};
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
return res.deletedCount ?? 0;
};
findById = async (refId: string) => {
const docs = await JobModel.find({ refId }).lean<JobDocument[]>();
return docs.map(this.toEntity);
};
findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => {
const count = await JobModel.countDocuments();
const query = JobModel.find().sort({ nextScheduledAt: 1 });
if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage);
const docs = await query.lean<JobDocument[]>();
return { jobs: docs.map(this.toEntity), count };
};
findAll = async () => {
const docs = await JobModel.find().lean<JobDocument[]>();
return docs.map(this.toEntity);
};
}
export default MongoJobsRepository;
+32
View File
@@ -0,0 +1,32 @@
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
export type JobType = (typeof JOB_TYPES)[number];
export const LOCK_MS = 60_000;
export const BACKOFF_MS = 5_000;
export const PARKED = Number.MAX_SAFE_INTEGER;
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
export type Job = {
id: string;
type: JobType;
refId: string | null;
isActive: boolean;
// Scheduling fields
nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now
intervalMs: number | null; // repeat period in ms; null = one time job
// Lock fields
lockedBy: string | null; // workerId holding the lease
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
// observability (replaces what getMetrics/getJobs)
runCount: number;
failCount: number;
lastFinishedAt: number | null;
lastFailReason: string | null;
};
// These are the only values to set when creating a job. Repo owns the rest
export type JobSeed = Pick<Job, "id" | "type" | "refId" | "isActive" | "intervalMs" | "nextScheduledAt">;
@@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase {
teamId: Types.ObjectId;
createdAt: Date;
updatedAt: Date;
lastEvaluatedAt: number; // epoch ms
}
const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
@@ -377,6 +378,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
type: [checkSnapshotSchema],
default: [],
},
lastEvaluatedAt: {
type: Number,
default: 0,
},
},
{
timestamps: true,
@@ -571,6 +571,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
dnsRecordType: doc.dnsRecordType ?? undefined,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
lastEvaluatedAt: doc.lastEvaluatedAt,
};
};
@@ -636,6 +637,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
dnsRecordType: doc.dnsRecordType ?? undefined,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
lastEvaluatedAt: doc.lastEvaluatedAt,
};
};
@@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService {
recentChecks: [],
createdAt: "",
updatedAt: "",
lastEvaluatedAt: 0,
}));
const createdMonitors = await this.createMonitors(cleanedMonitors);
@@ -88,6 +88,7 @@ export interface Monitor {
recentChecks: CheckSnapshot[];
createdAt: string;
updatedAt: string;
lastEvaluatedAt: number; // epoch ms
}
export interface MonitorsSummary {
@@ -2,11 +2,15 @@ import { Schema, model } from "mongoose";
import { QueueModes } from "@/domain/app-settings/app-settings.type.js";
import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js";
// Single source of truth for worker liveness: the TTL that GCs stale records and
// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it.
export const WORKER_TTL_SECONDS = 180;
const QueueWorkerSchema = new Schema<QueueWorkerDocument>(
{
_id: { type: String, required: true },
mode: { type: String, enum: QueueModes, required: true },
lastSeenAt: { type: Date, default: Date.now, expires: 30 },
lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS },
},
{ timestamps: true }
);
+1 -1
View File
@@ -24,7 +24,7 @@ const startApp = async () => {
logger = new Logger({ envSettings });
// Initialize services (connects DB, creates repositories, injects settingsRepository)
const services = await initializeServices({ logger, envSettings, settingsService, queueMode: envSettings.queueMode });
const services = await initializeServices({ logger, envSettings, settingsService });
// If this is a worker instance, we're done. No need for express
if (envSettings.queueMode === "worker") {
@@ -0,0 +1,33 @@
import { MonitorStatusResponse } from "@/types/network.js";
import { MonitorEvaluation } from "@/worker/worker.interface.js";
import { Check } from "@/domain/checks/check.type.js";
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IStatusService } from "@/service/statusService.js";
export interface ICheckEvaluator {
evaluate(status: MonitorStatusResponse, check: Check): Promise<MonitorEvaluation>;
}
export class CheckEvaluator implements ICheckEvaluator {
constructor(
private statusService: IStatusService,
private monitorStatusPolicy: IMonitorStatusPolicy
) {}
evaluate = async (status: MonitorStatusResponse, check: Check) => {
// ****************************
// Step 3: Evaluate and return result to reactors
// ****************************
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
// Step 5. Get decisions and create an evaluation obj
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
const evaluation: MonitorEvaluation = {
monitor: statusChangeResult.monitor,
status,
check,
statusChange: statusChangeResult,
decision,
};
return evaluation;
};
}
+8 -78
View File
@@ -1,17 +1,14 @@
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { MonitorEvaluation } from "@/worker/worker.interface.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { ILogger } from "@/utils/logger.js";
import { INetworkService } from "@/service/networkService.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { IBufferService } from "@/service/bufferService.js";
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IStatusService } from "@/service/statusService.js";
import { AppError } from "@/utils/AppError.js";
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
import { ICheckProducer } from "@/worker/worker.check-producer.js";
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
const SERVICE_NAME = "CheckPipeline";
@@ -21,83 +18,16 @@ export interface ICheckPipeline {
export class CheckPipeline implements ICheckPipeline {
constructor(
private monitorsRepository: IMonitorsRepository,
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
private checkService: ICheckService,
private networkService: INetworkService,
private bufferService: IBufferService,
private monitorStatusPolicy: IMonitorStatusPolicy,
private statusService: IStatusService,
private logger: ILogger
private checkProducer: ICheckProducer,
private checkEvaluator: ICheckEvaluator
) {}
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
const now = new Date();
return maintenanceWindows.some((window) => isWindowActive(window, now));
}
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
const monitorId = monitor.id;
if (!monitorId) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getMonitorJob" });
const checkResult = await this.checkProducer.produce(monitor);
if (!checkResult) {
return null; // skipped due to maintenance / geo guards / other guards
}
// ****************************
// Step 1: Acquire
// ****************************
// Step 1a: Maintenance window gate - skip if in maintenance
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
if (maintenanceWindowActive) {
this.logger.debug({
message: `Monitor ${monitor.id} is in maintenance window`,
service: SERVICE_NAME,
method: "runCheckPipeline",
});
if (monitor.status !== "maintenance") {
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
}
return null;
}
// Step 1b: Acquire status
const status = await this.networkService.requestStatus(monitor);
if (!status) {
throw new Error("No network response");
}
// ****************************
// Step 2: Record
// ****************************
// Step 2a: Create & record a check, return null if fail
const check = this.checkService.buildCheck(status);
if (!check) {
this.logger.warn({
message: `No check could be built for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "runCHeckPipeline",
details: { code: status.code, message: status.message },
});
return null;
}
// Step 2b: Add to buffer
this.bufferService.addToBuffer(check);
// ****************************
// Step 3: Evaluate and return result to reactors
// ****************************
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
// Step 5. Get decisions and create an evaluation obj
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
const evaluation: MonitorEvaluation = {
monitor: statusChangeResult.monitor,
status,
check,
statusChange: statusChangeResult,
decision,
};
const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check);
return evaluation;
};
}
@@ -0,0 +1,82 @@
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { Check } from "@/domain/checks/check.type.js";
import { MonitorStatusResponse } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import { IBufferService } from "@/service/bufferService.js";
import { INetworkService } from "@/service/networkService.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
export interface ICheckProducer {
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
}
const SERVICE_NAME = "CheckProducer";
export class CheckProducer implements ICheckProducer {
constructor(
private monitorsRepository: IMonitorsRepository,
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
private checkService: ICheckService,
private networkService: INetworkService,
private bufferService: IBufferService,
private logger: ILogger
) {}
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
const now = new Date();
return windows.some((w) => isWindowActive(w, now));
}
produce = async (monitor: Monitor) => {
if (!monitor.id) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
}
// ****************************
// Step 1: Acquire
// ****************************
// Step 1a: Maintenance window gate - skip if in maintenance
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
if (maintenanceWindowActive) {
this.logger.debug({
message: `Monitor ${monitor.id} is in maintenance window`,
service: SERVICE_NAME,
method: "produce",
});
if (monitor.status !== "maintenance") {
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
}
return null;
}
// Step 1b: Acquire status
const status = await this.networkService.requestStatus(monitor);
if (!status) {
throw new Error("No network response");
}
// ****************************
// Step 2: Record
// ****************************
// Step 2a: Create & record a check, return null if fail
const check = this.checkService.toCheck(status);
if (!check) {
this.logger.warn({
message: `No check could be built for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "produce",
details: { code: status.code, message: status.message },
});
return null;
}
// Step 2b: Add to buffer
this.bufferService.addToBuffer(check);
return { status, check };
};
}
+389
View File
@@ -0,0 +1,389 @@
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js";
import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js";
import { ILogger } from "@/utils/logger.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { ICheckProducer } from "@/worker/worker.check-producer.js";
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
const SERVICE_NAME = "JobQueue";
const POLL_MS = 250;
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
const CONCURRENCY: Record<JobType, number> = {
check: 500,
"geo-check": 20,
evaluate: 20,
"cleanup-orphaned": 1,
"cleanup-retention": 1,
};
export class DBQueueWorker implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private stopped = false;
private timers = new Map<JobType | "heartbeat", NodeJS.Timeout>();
private inFlight: Record<JobType, number> = {
check: 0,
"geo-check": 0,
evaluate: 0,
"cleanup-orphaned": 0,
"cleanup-retention": 0,
};
constructor(
private logger: ILogger,
private jobsRepository: IJobsRepository,
private monitorsRepository: IMonitorsRepository,
private checksRepository: IChecksRepository,
private checkService: ICheckService, // toStatusResponse + toLastEvaluatedAt
private checkProducer: ICheckProducer,
private checkEvaluator: ICheckEvaluator,
private geoCheckPipeline: ICheckPipeline,
private dispatcher: IReactorDispatcher,
private helper: IWorkerHelper,
private queueWorkersRepository: IQueueWorkersRepository,
private queueMode: QueueMode,
private queuePrimaryProcesses: boolean,
private readonly workerId: string // same id MongoJobsRepository stamps into lockedBy
) {}
get serviceName() {
return DBQueueWorker.SERVICE_NAME;
}
static async create(
logger: ILogger,
jobsRepository: IJobsRepository,
monitorsRepository: IMonitorsRepository,
checksRepository: IChecksRepository,
checkService: ICheckService,
checkProducer: ICheckProducer,
checkEvaluator: ICheckEvaluator,
geoCheckPipeline: ICheckPipeline,
dispatcher: IReactorDispatcher,
helper: IWorkerHelper,
queueWorkersRepository: IQueueWorkersRepository,
queueMode: QueueMode,
queuePrimaryProcesses: boolean,
workerId: string
): Promise<DBQueueWorker> {
const instance = new DBQueueWorker(
logger,
jobsRepository,
monitorsRepository,
checksRepository,
checkService,
checkProducer,
checkEvaluator,
geoCheckPipeline,
dispatcher,
helper,
queueWorkersRepository,
queueMode,
queuePrimaryProcesses,
workerId
);
await instance.init();
return instance;
}
// Helpers, builds job seeds
private toCheckJob = (monitor: Monitor, now: number): JobSeed => ({
id: jobId("check", monitor.id),
type: "check",
refId: monitor.id,
isActive: monitor.isActive,
nextScheduledAt: now + Math.floor(Math.random() * monitor.interval), // Avoid herd
intervalMs: monitor.interval,
});
private toGeoCheckJob = (monitor: Monitor, now: number): JobSeed => ({
...this.toCheckJob(monitor, now),
id: jobId("geo-check", monitor.id),
type: "geo-check",
intervalMs: monitor.geoCheckInterval!,
});
private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({
id: jobId(type, null),
type,
refId: null,
isActive: true,
nextScheduledAt: now,
intervalMs: CLEANUP_INTERVAL_MS,
});
private toSummary = (job: Job): WorkerJobSummary => ({
monitorId: job.refId ?? job.id,
monitorUrl: null, // Jobs don't have URL
monitorType: job.type,
monitorInterval: job.intervalMs,
monitorGeoInterval: null,
monitorActive: job.isActive,
active: job.isActive,
lockedBy: job.lockedBy,
lockedUntil: job.lockedUntil,
lockedAt: null, // no lockedAt
runCount: job.runCount,
failCount: job.failCount,
failReason: job.lastFailReason,
lastRunAt: null, // not tracked
lastFinishedAt: job.lastFinishedAt,
lastRunTook: null, // not tracked
lastFailedAt: null, // not tracked
repeat: job.intervalMs,
});
// ********************
// Stage 1: This can eventually be offloaded to a separate service
// ********************
private runCheck = async (job: Job) => {
if (!job.refId) return;
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
if (!monitor) return;
const produced = await this.checkProducer.produce(monitor);
if (produced) {
await this.jobsRepository.upsertEvaluate(monitor.id, Date.now());
}
};
// ********************
// Stage 2: Evaluator loop stays here
// ********************
private runEvaluate = async (job: Job) => {
if (!job.refId) return;
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
if (!monitor) return;
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
for (const check of checks) {
const status = this.checkService.toStatusResponse(check);
const evaluation = await this.checkEvaluator.evaluate(status, check);
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
}
};
private runGeoCheck = async (job: Job) => {
const [monitor] = await this.monitorsRepository.findByIds([job.refId!]);
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
};
// Extend the lock while a job runs so a slow-but-alive job is never reclaimed.
// Returns false once we no longer hold the lock, so the renewal timer can stop.
private renewLock = async (job: Job): Promise<boolean> => {
try {
const held = await this.jobsRepository.renewLocks([job.id], Date.now());
if (held === 0) {
this.logger.warn({ message: `Lost lock on job ${job.id}, stopping renewal`, service: SERVICE_NAME, method: "renewLock" });
return false;
}
return true;
} catch (error: unknown) {
// Transient failure, keep renewing; a missed renewal is absorbed by the 1/3 margin
this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "renewLock" });
return true;
}
};
private runJob = async (job: Job) => {
const renewTimer = setInterval(async () => {
const renewLock = await this.renewLock(job);
if (!renewLock) clearInterval(renewTimer);
}, LOCK_RENEW_MS);
try {
switch (job.type) {
case "check":
await this.runCheck(job);
break;
case "evaluate":
await this.runEvaluate(job);
break;
case "geo-check":
await this.runGeoCheck(job);
break;
case "cleanup-orphaned":
await this.helper.getCleanupOrphanedJob()(); // Get job and execute
break;
case "cleanup-retention":
await this.helper.getCleanupRetentionJob()(); // Get job and execute
break;
}
if (job.intervalMs === null) {
// One-shot job
await this.jobsRepository.recordOneShot(job.id, Date.now());
} else {
await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now());
}
} catch (error: unknown) {
await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock
this.logger.warn({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: `runJob:${job.type}`,
});
} finally {
clearInterval(renewTimer);
}
};
private startLoop = (type: JobType) => {
const tick = async () => {
if (this.stopped) return;
try {
// Claim a batch sized to the free capacity
const capacity = CONCURRENCY[type] - this.inFlight[type];
const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now());
for (const job of jobs) {
this.inFlight[type]++; // stay within concurrency limits
this.runJob(job).finally(() => {
this.inFlight[type]--; // job done, free the slot
});
}
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: `loop:${type}`,
});
}
if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down
};
tick();
};
// Seed queue
private reconcile = async () => {
const now = Date.now();
const monitors = (await this.monitorsRepository.findAll()) ?? [];
for (const monitor of monitors) {
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
}
}
// Both cleanup jobs run immediately on every startup, then once a day
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now));
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now));
};
// Register worker
private heartbeat = async () => {
try {
await this.queueWorkersRepository.upsert(this.workerId, this.queueMode);
} catch (error: unknown) {
this.logger.warn({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: "heartbeat",
});
}
};
init = async () => {
this.stopped = false;
await this.heartbeat();
this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS));
if (this.queueMode === "primary") {
await this.reconcile();
}
if (this.queuePrimaryProcesses === true || this.queueMode === "worker") {
for (const type of Object.keys(this.inFlight) as JobType[]) {
this.startLoop(type);
}
}
return true;
};
addJob = async (_monitorId: string, monitor: Monitor) => {
const now = Date.now();
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
};
deleteJob = async (monitor: Monitor) => {
await this.jobsRepository.deleteById(monitor.id);
};
pauseJob = async (monitor: Monitor) => {
await this.jobsRepository.setActiveById(monitor.id, false);
};
resumeJob = async (monitor: Monitor) => {
await this.jobsRepository.setActiveById(monitor.id, true);
};
updateJob = async (monitor: Monitor) => {
await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval);
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now()));
} else {
await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check");
}
};
shutdown = async () => {
this.stopped = true;
// Stop all timers
for (const timer of this.timers.values()) {
clearTimeout(timer);
}
this.timers.clear(); // Clear the map out
// Remove workers from registry
await this.queueWorkersRepository.deleteById(this.workerId);
};
getMetrics = async (): Promise<WorkerMetrics> => {
const rows = await this.jobsRepository.findAll();
const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS);
const now = Date.now();
return rows.reduce<WorkerMetrics>(
(acc, job) => {
acc.jobs++;
acc.totalRuns += job.runCount;
acc.totalFailures += job.failCount;
if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++;
if (job.failCount > 0) {
acc.failingJobs++;
acc.jobsWithFailures.push({
monitorId: job.refId ?? job.id,
monitorUrl: null,
monitorType: job.type,
failedAt: job.lastFinishedAt,
failCount: job.failCount,
failReason: job.lastFailReason,
});
}
return acc;
},
{
jobs: 0,
activeJobs: 0,
failingJobs: 0,
jobsWithFailures: [],
totalRuns: 0,
totalFailures: 0,
workers,
}
);
};
getJobs = async (pagination: WorkerJobsPagination) => {
const { jobs, count } = await this.jobsRepository.findPage(pagination);
return { jobs: jobs.map(this.toSummary), count };
};
flushQueues = async () => {
await this.shutdown();
const ok = await this.init();
return { success: ok };
};
}
+14
View File
@@ -8,6 +8,7 @@ import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.i
import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
@@ -42,6 +43,7 @@ export class WorkerHelper implements IWorkerHelper {
private checkService: ICheckService;
private settingsService: ISettingsService;
private monitorsRepository: IMonitorsRepository;
private jobsRepository: IJobsRepository;
private teamsRepository: ITeamsRepository;
private monitorStatsRepository: IMonitorStatsRepository;
private checksRepository: IChecksRepository;
@@ -56,6 +58,7 @@ export class WorkerHelper implements IWorkerHelper {
checkService: ICheckService,
settingsService: ISettingsService,
monitorsRepository: IMonitorsRepository,
jobsRepository: IJobsRepository,
teamsRepository: ITeamsRepository,
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository,
@@ -69,6 +72,7 @@ export class WorkerHelper implements IWorkerHelper {
this.checkService = checkService;
this.settingsService = settingsService;
this.monitorsRepository = monitorsRepository;
this.jobsRepository = jobsRepository;
this.teamsRepository = teamsRepository;
this.monitorStatsRepository = monitorStatsRepository;
this.checksRepository = checksRepository;
@@ -190,6 +194,16 @@ export class WorkerHelper implements IWorkerHelper {
});
}
// Remove orphaned jobs
const deletedJobsCount = await this.jobsRepository.deleteByMonitorIdsNotIn(allMonitorIds);
if (deletedJobsCount > 0) {
this.logger.info({
message: `Deleted ${deletedJobsCount} orphaned jobs`,
service: SERVICE_NAME,
method: "getCleanupOrphanedJob",
});
}
this.logger.info({
message: "Cleanup of orphaned data completed",
service: SERVICE_NAME,
+1 -1
View File
@@ -1,8 +1,8 @@
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
import { Check } from "@/domain/checks/check.type.js";
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
import { MonitorActionDecision } from "@/worker/worker.helper.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
export type MonitorEvaluation = {
monitor: Monitor;
-427
View File
@@ -1,427 +0,0 @@
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { MongoStore, Scheduler, IJob, AddJobInput } from "less-simple-scheduler";
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { type WorkerMetrics, type WorkerJobsPagination, type WorkerJobsPage, IWorker } from "@/worker/worker.interface.js";
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
const SERVICE_NAME = "JobQueue";
// How long after its last heartbeat a worker is still considered alive. Must be longer than the scheduler's heartbeat
const WORKER_TTL_MS = 30_000;
// For discriminating job types
type MonitorJob = Omit<IJob, "data"> & { data: Monitor };
const MONITOR_TEMPLATES = new Set<string>(["monitor-job", "geo-check-job"]);
const isMonitorJob = (job: IJob): job is MonitorJob => MONITOR_TEMPLATES.has(job.template);
export class LessSimpleWorker implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private helper: IWorkerHelper;
private monitorsRepository: IMonitorsRepository;
private workersRepository: IQueueWorkersRepository;
private readonly scheduler: Scheduler;
private queueMode: QueueMode;
private heartbeatListener: ((workerId: string) => void) | null = null;
constructor(
logger: ILogger,
helper: IWorkerHelper,
monitorsRepository: IMonitorsRepository,
workersRepository: IQueueWorkersRepository,
scheduler: Scheduler,
queueMode: QueueMode
) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
this.workersRepository = workersRepository;
this.scheduler = scheduler;
this.queueMode = queueMode;
this.registerListeners();
}
get serviceName() {
return LessSimpleWorker.SERVICE_NAME;
}
private registerListeners = () => {
this.scheduler.on("scheduler:start", () => {
this.logger.info({
message: "Scheduler started",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:stop", () => {
this.logger.info({
message: "Scheduler stopped",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:error", (workerId, error) => {
this.logger.error({
message: `${workerId} error: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:locked", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} locked`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:abort", (workerId, job, reason) => {
this.logger.warn({
message: `${workerId} ${job.id} aborted: ${reason}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:attempt", (workerId, job, attempt) => {
this.logger.debug({
message: `${workerId} ${job.id} attempt ${attempt}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:complete", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} completed successfully`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:exhausted", (workerId, job, error) => {
this.logger.error({
message: `${workerId} ${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:fail", (workerId, job, error, attempt) => {
this.logger.warn({
message: `${workerId} ${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:start", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} started`,
service: SERVICE_NAME,
});
});
};
static async create(
logger: ILogger,
helper: IWorkerHelper,
monitorsRepository: IMonitorsRepository,
workersRepository: IQueueWorkersRepository,
envSettings: EnvConfig,
queueMode: QueueMode
) {
const store = new MongoStore({
url: envSettings.dbConnectionString ?? "mongodb://localhost:27017/uptime_db",
});
const scheduler = new Scheduler(store, {
concurrency: envSettings.queueMode === "primary" ? 0 : 50,
processEvery: 100,
});
const instance = new LessSimpleWorker(logger, helper, monitorsRepository, workersRepository, scheduler, queueMode);
await instance.init(queueMode);
return instance;
}
private startHeartbeat = (queueMode: QueueMode) => {
const beat = (
workerId: string // This is the heartbeat function that registers workers
) =>
this.workersRepository.upsert(workerId, queueMode).catch((error: unknown) => {
this.logger.warn({
message: `Worker heartbeat failed: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "startHeartbeat",
});
});
beat(this.scheduler.workerId); // Beat immediately to register
if (this.heartbeatListener) {
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); // remove old listeners
}
this.heartbeatListener = beat; /// Keep a reference to the listener
this.scheduler.on("scheduler:heartbeat", beat); // beat on every heartbeat
};
init = async (queueMode: QueueMode = "primary") => {
try {
await this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
// Register in the worker registry
this.startHeartbeat(queueMode);
// If this is a worker, return early - primary will populate queue
if (queueMode === "worker") {
this.logger.info({
message: `Worker queue: ${this.scheduler.workerId} initialized`,
service: SERVICE_NAME,
method: "init",
});
return true;
}
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
return true;
}
const jobInputs = monitors.flatMap((monitor) => this.buildJobInputs(monitor));
if (jobInputs.length > 0) {
await this.scheduler.addJobs(jobInputs);
}
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, upsert: true });
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000, upsert: true });
return true;
} catch (error: unknown) {
this.logger.error({
message: `Failed to initialize LessSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "init",
});
return false;
}
};
// Builds the scheduler inputs for a monitor: always a monitor-job, plus a
// geo-check job when the monitor supports and enables geo checks.
private buildJobInputs = (monitor: Monitor): AddJobInput[] => {
const inputs: AddJobInput[] = [
{
id: monitor.id,
template: "monitor-job",
repeat: monitor.interval,
active: monitor.isActive,
data: monitor,
upsert: true,
jitter: true,
},
];
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
inputs.push({
id: `${monitor.id}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
upsert: true,
jitter: true,
});
}
return inputs;
};
addJob = async (_monitorId: string, monitor: Monitor) => {
await this.scheduler.addJobs(this.buildJobInputs(monitor));
};
deleteJob = async (monitor: Monitor) => {
this.scheduler.removeJob(monitor.id);
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: Monitor) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to pause monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to pause geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor: Monitor) => {
const result = await this.scheduler.resumeJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to resume geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
private syncGeoJob = async (monitor: Monitor) => {
const geoJobId = `${monitor.id}-geo`;
const existingGeoJob = await this.scheduler.getJob(geoJobId);
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
return;
}
// If the job exists, update it
if (existingGeoJob) {
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
return;
}
// Otherwise, create it
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
upsert: true,
});
};
updateJob = async (monitor: Monitor) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
await this.syncGeoJob(monitor);
};
shutdown = async () => {
// Stop beating before we remove this worker, so a heartbeat firing between
// the remove and the scheduler stopping can't re-insert it.
if (this.heartbeatListener) {
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener);
this.heartbeatListener = null;
}
await this.workersRepository.deleteById(this.scheduler.workerId).catch((error: unknown) => {
this.logger.warn({
message: `Failed to deregister worker on shutdown: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "shutdown",
});
});
this.scheduler.stop();
};
getMetrics = async (): Promise<WorkerMetrics> => {
const stats = await this.scheduler.getStats();
const jobsWithFailures = stats.jobsWithFailures.map((failure) => {
// `data` is untyped at the store level: for monitor jobs it's the Monitor,
// for system jobs (e.g. cleanup) it has no url/type => null
const monitor = failure.data as Monitor | undefined;
return {
monitorId: failure.id,
monitorUrl: monitor?.url ?? null,
monitorType: monitor?.type ?? null,
failedAt: failure.failedAt,
failCount: failure.failCount,
failReason: failure.failReason,
};
});
const workers = await this.workersRepository.findRecent(WORKER_TTL_MS);
return {
jobs: stats.jobs,
activeJobs: stats.activeJobs,
failingJobs: stats.failingJobs,
totalRuns: stats.totalRuns,
totalFailures: stats.totalFailures,
jobsWithFailures,
workers,
};
};
getJobs = async ({ page = 0, rowsPerPage = 0 }: WorkerJobsPagination): Promise<WorkerJobsPage> => {
const listOptions = rowsPerPage > 0 ? { skip: Math.max(page, 0) * rowsPerPage, limit: rowsPerPage } : undefined;
const [jobs, count] = await Promise.all([this.scheduler.getJobs(listOptions), this.scheduler.countJobs()]);
const mapped = jobs.map((job) => {
const monitor = isMonitorJob(job) ? job.data : undefined;
return {
monitorId: job.id,
monitorUrl: monitor?.url || null,
monitorType: monitor?.type || null,
monitorInterval: monitor?.interval || null,
monitorGeoInterval: monitor?.geoCheckInterval || null,
monitorActive: monitor?.isActive ?? null,
active: job.active,
repeat: job.repeat ?? null,
lockedBy: job.lockedBy ?? null,
lockedUntil: job.lockedUntil ?? null,
lockedAt: job.lockedAt ?? null,
runCount: job.runCount ?? 0,
failCount: job.failCount ?? 0,
failReason: job.lastError ?? null,
lastRunAt: job.lastStartedAt ?? null,
lastFinishedAt: job.lastFinishedAt ?? null,
lastRunTook: job.lockedAt || !job.lastFinishedAt || !job.lastStartedAt ? null : job.lastFinishedAt - job.lastStartedAt,
lastFailedAt: job.lastFailedAt ?? null,
};
});
return { jobs: mapped, count };
};
flushQueues = async () => {
if (this.queueMode !== "primary") {
// Don't allow workers to flush, though this isn't currently reachable since workers have no API endpoints
this.logger.warn({ message: "Ignoring flush on non-primary worker", service: SERVICE_NAME, method: "flushQueues" });
return { success: false };
}
const flushRes = await this.scheduler.flushJobs(); // Less Simple Queue must flush before stopping as it needs DB
const stopRes = await this.scheduler.stop(); // Destructive, other workers will abort their jobs
const initRes = await this.init();
return {
success: Boolean(stopRes && flushRes && initRes),
};
};
}
-384
View File
@@ -1,384 +0,0 @@
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import Scheduler from "super-simple-scheduler";
import { Monitor, MonitorType, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { hostname } from "node:os";
import { randomUUID } from "node:crypto";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { IWorker, WorkerJobSummary } from "@/worker/worker.interface.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
const SERVICE_NAME = "JobQueue";
export class SuperSimpleQueue implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private helper: IWorkerHelper;
private workerId: string;
private monitorsRepository: IMonitorsRepository;
private readonly scheduler: Scheduler;
constructor(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
this.scheduler = scheduler;
this.registerListeners();
this.workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
}
get serviceName() {
return SuperSimpleQueue.SERVICE_NAME;
}
private registerListeners = () => {
this.scheduler.on("scheduler:start", () => {
this.logger.info({
message: "Scheduler started",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:stop", () => {
this.logger.info({
message: "Scheduler stopped",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:error", (error) => {
this.logger.error({
message: `Scheduler error: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:abort", (job, reason) => {
this.logger.warn({
message: `${job.id} aborted: ${reason}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:attempt", (job, attempt) => {
this.logger.debug({
message: `${job.id} attempt ${attempt}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:complete", (job) => {
this.logger.debug({
message: `${job.id} completed successfully`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:exhausted", (job, error) => {
this.logger.error({
message: `${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:fail", (job, error, attempt) => {
this.logger.warn({
message: `${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:start", (job) => {
this.logger.debug({
message: `${job.id} started`,
service: SERVICE_NAME,
});
});
};
static async create(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository) {
const scheduler = new Scheduler({
concurrency: 50,
// storeType: "mongo",
// storeType: "redis",
// dbUri: envSettings.dbConnectionString,
});
const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
await instance.init();
return instance;
}
init = async () => {
try {
this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
return true;
}
for (const monitor of monitors) {
const randomOffset = Math.floor(Math.random() * 100);
setTimeout(() => {
this.addJob(monitor.id, monitor);
}, randomOffset);
}
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 });
return true;
} catch (error: unknown) {
this.logger.error({
message: `Failed to initialize SuperSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "init",
});
return false;
}
};
addJob = async (monitorId: string, monitor: Monitor) => {
this.scheduler.addJob({
id: monitorId,
template: "monitor-job",
repeat: monitor.interval,
active: monitor.isActive,
data: monitor,
});
// Return early if we don't need geo checks
if (!supportsGeoCheck(monitor.type)) {
return;
}
// Add geo check job if enabled for HTTP monitors
if (monitor.geoCheckEnabled) {
this.scheduler.addJob({
id: `${monitorId}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
}
};
deleteJob = async (monitor: Monitor) => {
this.scheduler.removeJob(monitor.id);
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: Monitor) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to pause monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to pause geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor: Monitor) => {
const result = await this.scheduler.resumeJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to resume geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
private syncGeoJob = async (monitor: Monitor) => {
const geoJobId = `${monitor.id}-geo`;
const existingGeoJob = await this.scheduler.getJob(geoJobId);
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
return;
}
// If the job exists, update it
if (existingGeoJob) {
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
return;
}
// Otherwise, create it
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
};
updateJob = async (monitor: Monitor) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
await this.syncGeoJob(monitor);
};
shutdown = async () => {
this.scheduler.stop();
};
getMetrics = async () => {
const jobs = await this.scheduler.getJobs();
const metrics: {
jobs: number;
activeJobs: number;
failingJobs: number;
jobsWithFailures: Array<{
monitorId: string | number;
monitorUrl: string;
monitorType: MonitorType;
failedAt: number | null;
failCount: number;
failReason: string | null;
}>;
totalRuns: number;
totalFailures: number;
workers: Array<{
workerId: string;
mode: QueueMode;
lastSeenAt: number;
}>;
} = jobs.reduce(
(acc, job) => {
const runCount = job.runCount ?? 0;
const failCount = job.failCount ?? 0;
const lastFailedAt = job.lastFailedAt ?? 0;
const lastRunAt = job.lastRunAt ?? 0;
acc.totalRuns += runCount;
acc.totalFailures += failCount;
acc.jobs++;
if (failCount > 0 && lastFailedAt >= lastRunAt) {
acc.failingJobs++;
}
if (job.lockedAt) {
acc.activeJobs++;
}
if (failCount > 0) {
acc.jobsWithFailures.push({
monitorId: job.id,
monitorUrl: job.data?.url || null,
monitorType: job.data?.type || null,
failedAt: job.lastFailedAt ?? null,
failCount,
failReason: job.lastFailReason ?? null,
});
}
return acc;
},
{
jobs: 0,
activeJobs: 0,
failingJobs: 0,
jobsWithFailures: [] as Array<{
monitorId: string | number;
monitorUrl: string;
monitorType: MonitorType;
failedAt: number | null;
failCount: number;
failReason: string | null;
}>,
totalRuns: 0,
totalFailures: 0,
workers: [
{
workerId: this.workerId,
mode: "primary",
lastSeenAt: Date.now(),
},
],
}
);
return metrics;
};
getJobs = async ({ page = 0, rowsPerPage = 0 }: { page?: number; rowsPerPage?: number }): Promise<{ jobs: WorkerJobSummary[]; count: number }> => {
const jobs = await this.scheduler.getJobs();
const mapped = jobs.map((job) => {
return {
monitorId: job.id,
monitorUrl: job.data?.url || null,
monitorType: job.data?.type || null,
monitorInterval: job.data?.interval || null,
monitorGeoInterval: job.data?.geoCheckInterval || null,
monitorActive: job.data?.isActive ?? null,
active: job.active,
repeat: job.repeat ?? null,
lockedBy: job.lockedAt ? this.workerId : null,
lockedUntil: null,
lockedAt: job.lockedAt ?? null,
runCount: job.runCount ?? 0,
failCount: job.failCount ?? 0,
failReason: job.lastFailReason ?? null,
lastRunAt: job.lastRunAt ?? null,
lastFinishedAt: job.lastFinishedAt ?? null,
lastRunTook: job.lockedAt ? null : (job.lastFinishedAt ?? 0) - (job.lastRunAt ?? 0),
lastFailedAt: job.lastFailedAt ?? null,
};
});
const start = Math.max(page, 0) * rowsPerPage;
return {
jobs: rowsPerPage > 0 ? mapped.slice(start, start + rowsPerPage) : mapped,
count: mapped.length,
};
};
flushQueues = async () => {
const stopRes = await this.scheduler.stop();
const flushRes = await this.scheduler.flushJobs();
const initRes = await this.init();
return {
success: Boolean(stopRes && flushRes && initRes),
};
};
}
+7 -4
View File
@@ -2,6 +2,8 @@ import { jest } from "@jest/globals";
import { WorkerHelper } from "../../src/worker/worker.helper.ts";
import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts";
import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts";
import { CheckProducer } from "../../src/worker/worker.check-producer.ts";
import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts";
import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts";
import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts";
import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts";
@@ -131,7 +133,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
}),
};
const checkService = {
buildCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => {
toCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => {
return makeCheck(response.status, response.code);
}),
};
@@ -149,16 +151,16 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
const geoChecksService = { buildGeoCheck: jest.fn() };
const checkPipeline = new CheckPipeline(
const checkProducer = new CheckProducer(
monitorsRepo as any,
maintenanceWindowsRepo as any,
checkService as any,
networkService as any,
bufferStub as any,
new MonitorStatusPolicy(),
statusService as any,
logger
);
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger);
const notificationReactor = new NotificationReactor(notificationsService as any);
@@ -170,6 +172,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
checkService as any,
{ getDBSettings: jest.fn() } as any,
monitorsRepo as any,
{ deleteByMonitorIdsNotIn: jest.fn() } as any,
{ findAllTeamIds: jest.fn() } as any,
createStubMonitorStatsRepo() as any,
createStubChecksRepo() as any,
@@ -0,0 +1,64 @@
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import { MonitorModel } from "../../src/domain/monitors/monitor.model.ts";
import { backfillMonitorLastEvaluatedAt } from "../../src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// Legacy monitors are inserted through the raw driver, NOT MonitorModel.create — the
// schema default (lastEvaluatedAt: 0) would otherwise stamp the field and the migration's
// { $exists: false } filter would skip it, defeating the test.
let mongod: MongoMemoryServer;
beforeAll(async () => {
mongod = await MongoMemoryServer.create();
await mongoose.connect(mongod.getUri());
}, 120_000);
afterAll(async () => {
await mongoose.disconnect();
await mongod.stop();
});
beforeEach(async () => {
await MonitorModel.collection.deleteMany({});
});
describe("0008_backfillMonitorLastEvaluatedAt", () => {
it("stamps the upgrade moment (not 0) on monitors missing lastEvaluatedAt", async () => {
await MonitorModel.collection.insertOne({ name: "legacy" }); // pre-upgrade doc, no lastEvaluatedAt
const before = Date.now();
await backfillMonitorLastEvaluatedAt();
const doc = await MonitorModel.collection.findOne({ name: "legacy" });
// Must be a recent timestamp — backfilling 0 would replay the whole check history on the first evaluate.
expect(doc?.lastEvaluatedAt).not.toBe(0);
expect(doc?.lastEvaluatedAt).toBeGreaterThanOrEqual(before);
expect(doc?.lastEvaluatedAt).toBeLessThanOrEqual(Date.now());
});
it("leaves monitors that already have lastEvaluatedAt untouched (idempotent)", async () => {
await MonitorModel.collection.insertOne({ name: "current", lastEvaluatedAt: 12345 });
await backfillMonitorLastEvaluatedAt();
const doc = await MonitorModel.collection.findOne({ name: "current" });
expect(doc?.lastEvaluatedAt).toBe(12345);
});
it("drops the legacy jobs collection left behind by the old scheduler", async () => {
const db = mongoose.connection.db!;
await db.collection("jobs").insertOne({ id: "legacy-job", template: "monitor-job" }); // old-scheduler row
await backfillMonitorLastEvaluatedAt();
const exists = await db.listCollections({ name: "jobs" }).toArray();
expect(exists).toHaveLength(0);
});
it("does not throw when there is no jobs collection to drop", async () => {
await expect(backfillMonitorLastEvaluatedAt()).resolves.toBeUndefined();
});
});
@@ -0,0 +1,461 @@
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
import { BACKOFF_MS, LOCK_MS, type Job, type JobType } from "../../src/domain/jobs/job.type.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
// single-document atomicity on findOneAndUpdate, so it can only be exercised against a
// live engine. We boot a throwaway in-process mongod for this suite.
let mongod: MongoMemoryServer;
beforeAll(async () => {
mongod = await MongoMemoryServer.create();
await mongoose.connect(mongod.getUri());
await JobModel.init(); // build the indexes the claim scan relies on
}, 120_000);
afterAll(async () => {
await mongoose.disconnect();
await mongod.stop();
});
beforeEach(async () => {
await JobModel.deleteMany({});
});
// ── Helpers ──────────────────────────────────────────────────────────────────
const NOW = 1_000_000;
// Insert a job row directly, full control over every field. Defaults describe a due,
// unlocked, active check job.
const seedJob = (overrides: Partial<JobDocument> = {}): Promise<JobDocument> =>
JobModel.create({
_id: overrides._id ?? `check:${overrides.refId ?? "mon-1"}`,
type: "check" as JobType,
refId: "mon-1",
isActive: true,
nextScheduledAt: NOW,
intervalMs: 60_000,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
...overrides,
});
const readRow = (id: string) => JobModel.findById(id).lean<JobDocument>();
// record*/renewLocks are fenced on the repo's private workerId, so a pre-seeded "held"
// lock must be stamped with the same id the repo will check against.
const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string }).workerId;
const WORKER_ID = "worker-under-test";
describe("MongoJobsRepository", () => {
let repo: MongoJobsRepository;
beforeEach(() => {
repo = new MongoJobsRepository(WORKER_ID);
});
// ── claimDue ───────────────────────────────────────────────────────────────
describe("claimDue", () => {
it("claims a due, unlocked job and leases it", async () => {
await seedJob();
const claimed = await repo.claimDue("check", NOW);
expect(claimed?.id).toBe("check:mon-1");
const row = await readRow("check:mon-1");
expect(row?.lockedBy).not.toBeNull();
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
});
it("returns null when nothing is due yet", async () => {
await seedJob({ nextScheduledAt: NOW + 1 });
expect(await repo.claimDue("check", NOW)).toBeNull();
});
it("skips inactive (paused) jobs", async () => {
await seedJob({ isActive: false });
expect(await repo.claimDue("check", NOW)).toBeNull();
});
it("only claims the requested type", async () => {
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
expect(await repo.claimDue("check", NOW)).toBeNull();
});
it("does not claim a job held by a live lease", async () => {
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
expect(await repo.claimDue("check", NOW)).toBeNull();
});
it("reclaims a job whose lease has expired (crash recovery)", async () => {
await seedJob({ lockedBy: "dead-worker", lockedUntil: NOW - 1 });
const claimed = await repo.claimDue("check", NOW);
expect(claimed?.id).toBe("check:mon-1");
const row = await readRow("check:mon-1");
expect(row?.lockedBy).not.toBe("dead-worker");
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
});
it("returns the oldest-due job first", async () => {
await seedJob({ _id: "check:newer", refId: "newer", nextScheduledAt: NOW });
await seedJob({ _id: "check:older", refId: "older", nextScheduledAt: NOW - 5_000 });
const claimed = await repo.claimDue("check", NOW);
expect(claimed?.id).toBe("check:older");
});
it("lets exactly one of many concurrent claimers win a single due job", async () => {
await seedJob();
// Distinct repo instances = distinct workerIds = realistic competing workers.
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
const results = await Promise.all(workers.map((w) => w.claimDue("check", NOW)));
const winners = results.filter((r): r is Job => r !== null);
expect(winners).toHaveLength(1);
expect(winners[0].id).toBe("check:mon-1");
});
it("hands two concurrent claimers two distinct due jobs (no double-claim)", async () => {
await seedJob({ _id: "check:a", refId: "a" });
await seedJob({ _id: "check:b", refId: "b" });
const [w1, w2] = [new MongoJobsRepository("worker-1"), new MongoJobsRepository("worker-2")];
const [r1, r2] = await Promise.all([w1.claimDue("check", NOW), w2.claimDue("check", NOW)]);
expect(r1).not.toBeNull();
expect(r2).not.toBeNull();
expect(r1?.id).not.toBe(r2?.id);
expect([r1?.id, r2?.id].sort()).toEqual(["check:a", "check:b"]);
});
});
// ── recordSuccess (fixed-rate reschedule) ────────────────────────────────────
describe("recordSuccess", () => {
it("reschedules fixed-rate from the scheduled tick and releases the lease", async () => {
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, runCount: 4 });
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_500);
expect(ok).toBe(true);
const row = await readRow("check:mon-1");
// next tick measured from the scheduled time (NOW), not completion time
expect(row?.nextScheduledAt).toBe(NOW + 60_000);
expect(row?.lockedBy).toBeNull();
expect(row?.lockedUntil).toBeNull();
expect(row?.lastFinishedAt).toBe(NOW + 1_500);
expect(row?.runCount).toBe(5);
});
it("skips missed ticks when the worker has fallen behind, with jitter to avoid a herd", async () => {
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
// scheduled NOW, interval 60s, but completion is 200s late → NOW+60s is already past
const now = NOW + 200_000;
await repo.recordSuccess("check:mon-1", NOW, 60_000, now);
const row = await readRow("check:mon-1");
// no catch-up burst: rebased to one interval from now, plus up to one interval of jitter
// so behind-jobs don't re-synchronize. Never earlier than now + interval (no over-checking).
expect(row?.nextScheduledAt).toBeGreaterThanOrEqual(now + 60_000);
expect(row?.nextScheduledAt).toBeLessThan(now + 120_000);
});
it("returns false for an unknown id", async () => {
expect(await repo.recordSuccess("missing", NOW, 60_000, NOW)).toBe(false);
});
it("does not release a lock owned by another worker (fenced no-op)", async () => {
// A worker whose lease expired and was reclaimed must not clobber the new owner's lock.
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7 });
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000);
expect(ok).toBe(false);
const row = await readRow("check:mon-1");
expect(row?.lockedBy).toBe("other-worker"); // lock untouched
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
expect(row?.runCount).toBe(7); // not double-counted
});
});
// ── renewLocks (lease renewal) ───────────────────────────────────────────────
describe("renewLocks", () => {
it("extends the lease on a lock this worker holds", async () => {
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000);
expect(renewed).toBe(1);
expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS);
});
it("does not renew a lock held by another worker (fenced)", async () => {
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000);
expect(renewed).toBe(0);
expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + LOCK_MS); // unchanged
});
it("renews only the locks this worker holds out of a mixed batch", async () => {
await seedJob({ _id: "check:mine", refId: "mine", lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
await seedJob({ _id: "check:theirs", refId: "theirs", lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
const renewed = await repo.renewLocks(["check:mine", "check:theirs"], NOW + 1_000);
expect(renewed).toBe(1);
expect((await readRow("check:mine"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS);
expect((await readRow("check:theirs"))?.lockedUntil).toBe(NOW + LOCK_MS);
});
it("returns 0 for an empty id list without touching the db", async () => {
expect(await repo.renewLocks([], NOW)).toBe(0);
});
// Regression: a renewed lock keeps a slow-but-alive job from being reclaimed once its
// original lease would have expired — preventing the duplicate-execution path.
it("keeps a job from being reclaimed after the original lease window passes", async () => {
await seedJob();
await repo.claimDue("check", NOW); // this worker now holds the lease until NOW + LOCK_MS
// just before expiry, the still-running job renews its own lock
const renewAt = NOW + LOCK_MS - 1_000;
expect(await repo.renewLocks(["check:mon-1"], renewAt)).toBe(1);
// a second worker scans PAST the original lease end — without renewal it would reclaim
const other = new MongoJobsRepository("competing-worker");
expect(await other.claimDue("check", NOW + LOCK_MS + 1)).toBeNull();
const row = await readRow("check:mon-1");
expect(row?.lockedBy).toBe(ownedBy(repo)); // still held by the original worker
expect(row?.lockedUntil).toBe(renewAt + LOCK_MS);
});
});
// ── recordFailure (backoff) ──────────────────────────────────────────────────
describe("recordFailure", () => {
it("backs off, records the reason, bumps failCount, releases the lease", async () => {
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, failCount: 2 });
const ok = await repo.recordFailure("check:mon-1", new Error("connection refused"), NOW);
expect(ok).toBe(true);
const row = await readRow("check:mon-1");
expect(row?.nextScheduledAt).toBe(NOW + BACKOFF_MS);
expect(row?.lastFailReason).toBe("connection refused");
expect(row?.failCount).toBe(3);
expect(row?.lockedBy).toBeNull();
expect(row?.lockedUntil).toBeNull();
});
it("stringifies a non-Error reason", async () => {
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
await repo.recordFailure("check:mon-1", "weird non-error", NOW);
const row = await readRow("check:mon-1");
expect(row?.lastFailReason).toBe("weird non-error");
});
it("returns false for an unknown id", async () => {
expect(await repo.recordFailure("missing", new Error("x"), NOW)).toBe(false);
});
});
// ── upsertEvaluate (handoff + $min coalescing) ───────────────────────────────
describe("upsertEvaluate", () => {
it("creates a single evaluate row keyed by monitor", async () => {
await repo.upsertEvaluate("mon-1", NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.type).toBe("evaluate");
expect(row?.refId).toBe("mon-1");
expect(row?.nextScheduledAt).toBe(NOW);
expect(row?.intervalMs).toBeNull();
expect(row?.isActive).toBe(true);
});
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
await repo.upsertEvaluate("mon-1", NOW);
await repo.upsertEvaluate("mon-1", NOW + 9_000);
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
expect(rows).toHaveLength(1);
expect(rows[0].nextScheduledAt).toBe(NOW); // earliest pending wins
});
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
await repo.upsertEvaluate("mon-1", NOW + 9_000);
await repo.upsertEvaluate("mon-1", NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.nextScheduledAt).toBe(NOW);
});
it("concurrent upserts for one monitor produce exactly one row", async () => {
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW)));
const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" });
expect(count).toBe(1);
});
});
// ── monitor lifecycle CRUD ───────────────────────────────────────────────────
describe("CRUD", () => {
it("upsertJob inserts, then refreshes config without clobbering scheduling state", async () => {
const job: Job = {
id: "check:mon-1",
type: "check",
refId: "mon-1",
isActive: true,
nextScheduledAt: NOW,
intervalMs: 60_000,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
};
await repo.upsertJob(job);
// simulate a running job that has advanced + holds a lease
await JobModel.updateOne({ _id: "check:mon-1" }, { $set: { nextScheduledAt: NOW + 5_000, lockedBy: "live", runCount: 7 } });
// a reconcile sweep re-upserts the same id with new config
await repo.upsertJob({ ...job, intervalMs: 30_000, isActive: false });
const row = await readRow("check:mon-1");
expect(row?.intervalMs).toBe(30_000); // config refreshed ($set)
expect(row?.isActive).toBe(false);
expect(row?.nextScheduledAt).toBe(NOW + 5_000); // scheduling untouched ($setOnInsert)
expect(row?.lockedBy).toBe("live");
expect(row?.runCount).toBe(7);
});
it("upsertCleanupJob resets nextScheduledAt on an existing row so it is due again on restart", async () => {
const job: Job = {
id: "cleanup-orphaned",
type: "cleanup-orphaned",
refId: null,
isActive: true,
nextScheduledAt: NOW,
intervalMs: 86_400_000,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
};
await repo.upsertCleanupJob(job);
// row has already advanced a day into the future after a previous run
await JobModel.updateOne({ _id: "cleanup-orphaned" }, { $set: { nextScheduledAt: NOW + 86_400_000, runCount: 3 } });
// a restart reconcile re-upserts and pulls it due now
await repo.upsertCleanupJob({ ...job, nextScheduledAt: NOW + 1_000 });
const row = await readRow("cleanup-orphaned");
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // schedule reset ($set)
expect(row?.runCount).toBe(3); // run history untouched ($setOnInsert)
});
it("setActiveById flips check and geo rows together", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
await repo.setActiveById("mon-1", false);
const rows = await JobModel.find({ refId: "mon-1" }).lean<JobDocument[]>();
expect(rows).toHaveLength(2);
expect(rows.every((r) => r.isActive === false)).toBe(true);
});
it("updateScheduleById changes the interval on a single typed row", async () => {
await seedJob({ _id: "check:mon-1", type: "check", intervalMs: 60_000 });
await seedJob({ _id: "geo:mon-1", type: "geo-check", intervalMs: 60_000 });
await repo.updateScheduleById("mon-1", "check", 15_000);
expect((await readRow("check:mon-1"))?.intervalMs).toBe(15_000);
expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched
});
it("deleteById drops every row for the monitor", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
await repo.upsertEvaluate("mon-1", NOW);
await repo.deleteById("mon-1");
expect(await JobModel.countDocuments({ refId: "mon-1" })).toBe(0);
});
it("deleteByMonitorIdsNotIn drops jobs for missing monitors but keeps valid and global rows", async () => {
await seedJob({ _id: "check:keep", refId: "keep", type: "check" });
await seedJob({ _id: "geo:keep", refId: "keep", type: "geo-check" });
await seedJob({ _id: "check:gone", refId: "gone", type: "check" });
await seedJob({ _id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" });
await seedJob({ _id: "cleanup-retention", refId: null, type: "cleanup-retention" });
const deleted = await repo.deleteByMonitorIdsNotIn(["keep"]);
expect(deleted).toBe(1); // only check:gone
const remaining = (await JobModel.find().lean<JobDocument[]>()).map((d) => d._id).sort();
expect(remaining).toEqual(["check:keep", "cleanup-orphaned", "cleanup-retention", "geo:keep"]);
});
it("findById returns all of a monitor's rows", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
const jobs = await repo.findById("mon-1");
expect(jobs.map((j) => j.id).sort()).toEqual(["check:mon-1", "geo:mon-1"]);
});
});
// ── observability ────────────────────────────────────────────────────────────
describe("observability", () => {
it("findPage returns a page plus the total count", async () => {
await seedJob({ _id: "check:a", refId: "a", nextScheduledAt: NOW + 1 });
await seedJob({ _id: "check:b", refId: "b", nextScheduledAt: NOW + 2 });
await seedJob({ _id: "check:c", refId: "c", nextScheduledAt: NOW + 3 });
const { jobs, count } = await repo.findPage({ page: 0, rowsPerPage: 2 });
expect(count).toBe(3);
expect(jobs).toHaveLength(2);
expect(jobs.map((j) => j.id)).toEqual(["check:a", "check:b"]); // sorted by nextScheduledAt
});
it("findPage returns everything when rowsPerPage is 0", async () => {
await seedJob({ _id: "check:a", refId: "a" });
await seedJob({ _id: "check:b", refId: "b" });
const { jobs, count } = await repo.findPage({});
expect(count).toBe(2);
expect(jobs).toHaveLength(2);
});
it("findAll returns every row", async () => {
await seedJob({ _id: "check:a", refId: "a" });
await seedJob({ _id: "geo:b", refId: "b", type: "geo-check" });
expect(await repo.findAll()).toHaveLength(2);
});
});
});
@@ -1,5 +1,7 @@
import { describe, expect, it, jest } from "@jest/globals";
import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
@@ -30,23 +32,23 @@ const createCheckPipeline = (overrides?: Record<string, any>) => {
statusService: {
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }),
},
checkService: { buildCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
buffer: { addToBuffer: jest.fn() },
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) },
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
monitorStatusPolicy: new MonitorStatusPolicy(),
...overrides,
};
const pipeline = new CheckPipeline(
const checkProducer = new CheckProducer(
defaults.monitorsRepository as any,
defaults.maintenanceWindowsRepository as any,
defaults.checkService as any,
defaults.networkService as any,
defaults.buffer as any,
defaults.monitorStatusPolicy as any,
defaults.statusService as any,
defaults.logger as any
);
const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
const pipeline = new CheckPipeline(checkProducer, checkEvaluator);
return { pipeline, defaults };
};
@@ -85,9 +87,9 @@ describe("CheckPipeline", () => {
await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response");
});
it("returns null and warns when buildCheck returns null", async () => {
it("returns null and warns when toCheck returns null", async () => {
const { pipeline, defaults } = createCheckPipeline({
checkService: { buildCheck: jest.fn().mockReturnValue(null) },
checkService: { toCheck: jest.fn().mockReturnValue(null) },
});
const result = await pipeline.run(makeMonitor());
+96 -16
View File
@@ -4,6 +4,8 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts";
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -53,12 +55,12 @@ describe("CheckService", () => {
});
});
// ── buildCheck ───────────────────────────────────────────────────────────
// ── toCheck ──────────────────────────────────────────────────────────────
describe("buildCheck", () => {
describe("toCheck", () => {
it("builds a basic HTTP check with correct fields", () => {
const { service } = createService();
const check = service.buildCheck(makeStatusResponse());
const check = service.toCheck(makeStatusResponse());
expect(check).toBeDefined();
expect(check!.metadata).toEqual({ monitorId: "mon-1", teamId: "team-1", type: "http" });
@@ -72,14 +74,14 @@ describe("CheckService", () => {
it("defaults responseTime to 0 when falsy", () => {
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ responseTime: undefined }));
const check = service.toCheck(makeStatusResponse({ responseTime: undefined }));
expect(check!.responseTime).toBe(0);
});
it("includes timings when provided", () => {
const timings = { start: 0, socket: 10, lookup: 20 };
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ timings } as any));
const check = service.toCheck(makeStatusResponse({ timings } as any));
expect(check!.timings).toBe(timings);
});
@@ -106,7 +108,7 @@ describe("CheckService", () => {
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
expect(check!.accessibility).toBe(90);
expect(check!.bestPractices).toBe(80);
@@ -119,7 +121,7 @@ describe("CheckService", () => {
it("returns undefined when pagespeed payload is missing", () => {
const { service, logger } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any));
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any));
expect(check).toBeUndefined();
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build check" }));
@@ -127,7 +129,7 @@ describe("CheckService", () => {
it("defaults categories and audits to empty when lighthouseResult is missing", () => {
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any));
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any));
expect(check!.accessibility).toBe(0);
expect(check!.bestPractices).toBe(0);
@@ -146,7 +148,7 @@ describe("CheckService", () => {
},
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
expect(check!.audits!.cls!.score).toBe("informative");
});
@@ -161,7 +163,7 @@ describe("CheckService", () => {
},
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
expect(check!.audits!.cls!.score).toBeNull();
});
@@ -176,7 +178,7 @@ describe("CheckService", () => {
},
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
expect(check!.audits!.si!.numericValue).toBeUndefined();
});
@@ -192,7 +194,7 @@ describe("CheckService", () => {
},
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
expect(check!.audits!.cls).toBeUndefined();
expect(check!.audits!.si).toBeUndefined();
@@ -214,7 +216,7 @@ describe("CheckService", () => {
capture: { screenshot: "base64data" },
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
expect(check!.cpu).toEqual({ usage_percent: 0.5 });
expect(check!.memory).toEqual({ usage_percent: 0.6 });
@@ -230,7 +232,7 @@ describe("CheckService", () => {
errors: [{ message: "timeout" }],
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
expect(check!.errors).toEqual([{ message: "timeout" }]);
});
@@ -241,14 +243,14 @@ describe("CheckService", () => {
errors: { errors: [{ message: "nested error" }] },
} as any;
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
expect(check!.errors).toEqual([{ message: "nested error" }]);
});
it("handles undefined payload gracefully", () => {
const { service } = createService();
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any));
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any));
expect(check).toBeDefined();
expect(check!.cpu).toBeUndefined();
@@ -259,6 +261,84 @@ describe("CheckService", () => {
});
});
// ── toStatusResponse round-trip (Phase 2 §5 guard) ─────────────────────────
// The DB-queue split persists a Check, then the evaluator rebuilds the status response from it.
// toStatusResponse is only correct because every field a downstream consumer reads survives
// toCheck → toStatusResponse. These tests pin that: if a future change drops a consumed field
// from the round-trip, they fail instead of silently regressing the evaluator/reactors.
describe("toStatusResponse round-trip", () => {
const makeHardwareStatus = (overrides?: Partial<MonitorStatusResponse>): MonitorStatusResponse =>
makeStatusResponse({
type: "hardware",
status: true,
code: 200,
message: "OK",
responseTime: 42,
payload: {
data: {
cpu: { usage_percent: 0.92 },
memory: { usage_percent: 0.81 },
disk: [{ device: "/dev/sda", usage_percent: 0.75 }],
host: { os: "linux" },
net: [{ name: "eth0" }],
},
} as HardwareStatusPayload,
...overrides,
} as any);
it("uptime: preserves every consumed scalar field", () => {
const { service } = createService();
const status = makeStatusResponse({ type: "http", status: false, code: 503, message: "Service Unavailable", responseTime: 87 });
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
expect(rebuilt.monitorId).toBe(status.monitorId);
expect(rebuilt.teamId).toBe(status.teamId);
expect(rebuilt.type).toBe(status.type);
expect(rebuilt.status).toBe(status.status);
expect(rebuilt.code).toBe(status.code);
expect(rebuilt.message).toBe(status.message);
expect(rebuilt.responseTime).toBe(status.responseTime);
});
it("hardware: preserves payload.data metrics verbatim, same units (usage_percent stays a 0–1 decimal)", () => {
const { service } = createService();
const status = makeHardwareStatus();
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
const original = status.payload as HardwareStatusPayload;
const roundtripped = rebuilt.payload as HardwareStatusPayload;
expect(roundtripped.data).toEqual(original.data);
expect(roundtripped.data.cpu!.usage_percent).toBe(0.92); // not rescaled to 92
});
it("hardware: a real consumer (extractThresholdBreaches) behaves identically on raw vs round-tripped status", () => {
const { service } = createService();
const builder = new NotificationMessageBuilder();
const monitor = {
id: "mon-1",
type: "hardware",
cpuAlertThreshold: 90, // 0.92 → 92% breaches
memoryAlertThreshold: 90, // 0.81 → 81% does not
diskAlertThreshold: 90, // 0.75 → 75% does not
tempAlertThreshold: 100,
} as Monitor;
const status = makeHardwareStatus();
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
const fromRaw = builder.extractThresholdBreaches(monitor, status as MonitorStatusResponse<HardwareStatusPayload>);
const fromRebuilt = builder.extractThresholdBreaches(monitor, rebuilt as MonitorStatusResponse<HardwareStatusPayload>);
// sanity: the fixture actually triggers a breach, so the equality below is not vacuous
expect(fromRaw).toHaveLength(1);
expect(fromRaw[0].metric).toBe("cpu");
expect(fromRebuilt).toEqual(fromRaw);
});
});
// ── createChecks ─────────────────────────────────────────────────────────
describe("createChecks", () => {
@@ -0,0 +1,536 @@
import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals";
import { DBQueueWorker } from "../../../src/worker/worker.db-queue.ts";
import { LOCK_MS } from "../../../src/domain/jobs/job.type.ts";
import type { Job } from "../../../src/domain/jobs/job.type.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
import type { QueueMode } from "../../../src/domain/app-settings/app-settings.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
// ── Notes ──────────────────────────────────────────────────────────────────────
//
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
// renews their locks, and registers itself in the worker heartbeat. The repository
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
// test only the orchestration around it, with the repository fully mocked.
//
// Fake timers are mandatory: init() starts self-re-arming setTimeout loops and a
// heartbeat setInterval, so without them the loops would leak across tests. The
// loop re-arms at POLL_MS (250ms) and the lock renews at LOCK_MS / 3 (20s).
const POLL_MS = 250;
const LOCK_RENEW_MS = LOCK_MS / 3;
const deferred = <T>() => {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
};
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "m1",
teamId: "team",
type: "http",
interval: 60000,
status: "up",
isActive: true,
geoCheckEnabled: false,
geoCheckInterval: 300000,
lastEvaluatedAt: 0,
...overrides,
}) as Monitor;
const makeJob = (overrides?: Partial<Job>): Job => ({
id: "check:m1",
type: "check",
refId: "m1",
isActive: true,
nextScheduledAt: 1000,
intervalMs: 60000,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
...overrides,
});
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
const jobsRepository = {
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
upsertJob: jest.fn<any>().mockResolvedValue(true),
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
recordSuccess: jest.fn<any>().mockResolvedValue(true),
recordFailure: jest.fn<any>().mockResolvedValue(true),
recordOneShot: jest.fn<any>().mockResolvedValue(true),
renewLocks: jest.fn<any>().mockResolvedValue(1),
deleteById: jest.fn<any>().mockResolvedValue(true),
deleteByIdAndType: jest.fn<any>().mockResolvedValue(true),
setActiveById: jest.fn<any>().mockResolvedValue(true),
updateScheduleById: jest.fn<any>().mockResolvedValue(true),
findAll: jest.fn<any>().mockResolvedValue([]),
findPage: jest.fn<any>().mockResolvedValue({ jobs: [], count: 0 }),
};
const monitorsRepository = {
findByIds: jest.fn<any>().mockResolvedValue([makeMonitor()]),
findAll: jest.fn<any>().mockResolvedValue([]),
updateById: jest.fn<any>().mockResolvedValue({}),
};
const checksRepository = {
findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([]),
};
const checkService = {
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
toLastEvaluatedAt: jest.fn<any>().mockReturnValue(12345),
};
const checkProducer = {
produce: jest.fn<any>().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }),
};
const checkEvaluator = {
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), decision: {} }),
};
const geoCheckPipeline = { run: jest.fn<any>().mockResolvedValue(null) };
const dispatcher = { dispatch: jest.fn<any>().mockResolvedValue(undefined) };
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
const queueWorkersRepository = {
upsert: jest.fn<any>().mockResolvedValue(undefined),
deleteById: jest.fn<any>().mockResolvedValue(undefined),
findRecent: jest.fn<any>().mockResolvedValue([]),
};
const logger = createMockLogger();
const mocks = {
jobsRepository,
monitorsRepository,
checksRepository,
checkService,
checkProducer,
checkEvaluator,
geoCheckPipeline,
dispatcher,
helper,
queueWorkersRepository,
logger,
...overrides?.mocks,
};
const worker = new DBQueueWorker(
mocks.logger as any,
mocks.jobsRepository as any,
mocks.monitorsRepository as any,
mocks.checksRepository as any,
mocks.checkService as any,
mocks.checkProducer as any,
mocks.checkEvaluator as any,
mocks.geoCheckPipeline as any,
mocks.dispatcher as any,
mocks.helper as any,
mocks.queueWorkersRepository as any,
overrides?.queueMode ?? "worker",
overrides?.queuePrimaryProcesses ?? true,
"worker-1"
);
return { worker, mocks };
};
describe("DBQueueWorker", () => {
let active: DBQueueWorker | null = null;
beforeEach(() => {
jest.useFakeTimers();
});
afterEach(async () => {
if (active) await active.shutdown();
active = null;
jest.clearAllTimers();
jest.useRealTimers();
});
const start = async (opts?: Parameters<typeof createWorker>[0]) => {
const ctx = createWorker(opts);
active = ctx.worker;
await ctx.worker.init();
await jest.advanceTimersByTimeAsync(1); // flush the immediate tick + any job it launched
return ctx;
};
// ── init() wiring ─────────────────────────────────────────────────────────
describe("init", () => {
it("registers the worker in the heartbeat registry on startup", async () => {
const { mocks } = await start({ queueMode: "worker" });
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker");
});
it("primary mode reconciles: seeds a check row per monitor plus the two cleanup rows", async () => {
const { mocks } = await start({
queueMode: "primary",
queuePrimaryProcesses: false, // isolate reconcile from the polling loops
mocks: {
monitorsRepository: { findByIds: jest.fn<any>(), findAll: jest.fn<any>().mockResolvedValue([makeMonitor()]), updateById: jest.fn<any>() },
},
});
const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
expect(seededTypes).toContain("check");
const cleanupTypes = mocks.jobsRepository.upsertCleanupJob.mock.calls.map((c: any[]) => c[0].type);
expect(cleanupTypes).toContain("cleanup-orphaned");
expect(cleanupTypes).toContain("cleanup-retention");
});
it("primary mode also seeds a geo-check row for geo-enabled monitors", async () => {
const geoMonitor = makeMonitor({ type: "http", geoCheckEnabled: true });
const { mocks } = await start({
queueMode: "primary",
queuePrimaryProcesses: false,
mocks: {
monitorsRepository: { findByIds: jest.fn<any>(), findAll: jest.fn<any>().mockResolvedValue([geoMonitor]), updateById: jest.fn<any>() },
},
});
const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
expect(seededTypes).toContain("geo-check");
});
it("worker mode does not reconcile (never seeds jobs)", async () => {
const { mocks } = await start({ queueMode: "worker" });
expect(mocks.jobsRepository.upsertJob).not.toHaveBeenCalled();
expect(mocks.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("starts a polling loop per job type, each claiming up to its concurrency cap", async () => {
const { mocks } = await start({ queueMode: "worker" });
const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
});
it("does not start polling loops when primary and queuePrimaryProcesses is false", async () => {
const { mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false });
expect(mocks.jobsRepository.claimDueBatch).not.toHaveBeenCalled();
});
it("create() constructs the worker and runs init", async () => {
const { mocks } = createWorker(); // reuse the mock set; the un-init'd instance is discarded
const worker = await DBQueueWorker.create(
mocks.logger as any,
mocks.jobsRepository as any,
mocks.monitorsRepository as any,
mocks.checksRepository as any,
mocks.checkService as any,
mocks.checkProducer as any,
mocks.checkEvaluator as any,
mocks.geoCheckPipeline as any,
mocks.dispatcher as any,
mocks.helper as any,
mocks.queueWorkersRepository as any,
"worker",
true,
"worker-1"
);
active = worker;
await jest.advanceTimersByTimeAsync(1);
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker");
});
});
// ── runJob: the per-type execution paths ────────────────────────────────────
describe("runJob", () => {
// Hand the "check" loop exactly one job, then nothing, so the assertion target runs once.
const claimOnce = (job: Job) => {
let handed = false;
return jest.fn<any>(async (type: string) => {
if (type === job.type && !handed) {
handed = true;
return [job];
}
return [];
});
};
it("runs a check job and reschedules it on success", async () => {
const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]);
expect(mocks.checkProducer.produce).toHaveBeenCalled();
expect(mocks.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("m1", expect.any(Number));
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled();
});
it("records a failure (and not a success) when the job handler throws", async () => {
const job = makeJob({ type: "check" });
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: failingProducer },
});
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
});
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
});
it("evaluate job dispatches each unevaluated check and advances lastEvaluatedAt", async () => {
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
});
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
});
it("cleanup-orphaned job runs the helper's cleanup function", async () => {
const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 });
const cleanupFn = jest.fn<any>().mockResolvedValue(undefined);
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(cleanupFn),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
expect(cleanupFn).toHaveBeenCalled();
});
it("cleanup-retention job runs the helper's retention function", async () => {
const job = makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 });
const retentionFn = jest.fn<any>().mockResolvedValue(undefined);
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(retentionFn),
};
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
expect(retentionFn).toHaveBeenCalled();
});
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.geoCheckPipeline.run).toHaveBeenCalled();
expect(mocks.dispatcher.dispatch).not.toHaveBeenCalled();
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
});
it("renews the lock while a slow job is still running", async () => {
const job = makeJob({ type: "check" });
const gate = deferred<{ status: unknown; check: unknown }>();
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer },
});
// Job is parked on produce(); push past one renewal interval.
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledWith([job.id], expect.any(Number));
// Let it finish; success is still recorded.
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
await jest.advanceTimersByTimeAsync(1);
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled();
});
it("stops renewing once the lock is lost to another worker", async () => {
const job = makeJob({ type: "check" });
const gate = deferred<{ status: unknown; check: unknown }>();
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
const jobsRepository = {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
renewLocks: jest.fn<any>().mockResolvedValue(0),
};
const { mocks } = await start({ mocks: { jobsRepository, checkProducer: slowProducer } });
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); // lease lost → renewal timer stops
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Lost lock") }));
// A second interval passes without another renewal attempt.
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1);
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
await jest.advanceTimersByTimeAsync(1);
});
});
// ── monitor lifecycle delegation ────────────────────────────────────────────
describe("job lifecycle methods", () => {
it("addJob seeds a check row, plus a geo row for geo-enabled monitors", async () => {
const { worker, mocks } = createWorker();
active = worker;
await worker.addJob("m1", makeMonitor({ geoCheckEnabled: true }));
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
expect(types).toContain("check");
expect(types).toContain("geo-check");
});
it("addJob seeds only a check row for non-geo monitors", async () => {
const { worker, mocks } = createWorker();
active = worker;
await worker.addJob("m1", makeMonitor({ geoCheckEnabled: false }));
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
expect(types).toEqual(["check"]);
});
it("deleteJob removes every row for the monitor", async () => {
const { worker, mocks } = createWorker();
active = worker;
await worker.deleteJob(makeMonitor());
expect(mocks.jobsRepository.deleteById).toHaveBeenCalledWith("m1");
});
it("pauseJob and resumeJob flip the active flag", async () => {
const { worker, mocks } = createWorker();
active = worker;
await worker.pauseJob(makeMonitor());
expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", false);
await worker.resumeJob(makeMonitor());
expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", true);
});
it("updateJob reschedules the check row and drops the geo row when geo is disabled", async () => {
const { worker, mocks } = createWorker();
active = worker;
await worker.updateJob(makeMonitor({ geoCheckEnabled: false, interval: 30000 }));
expect(mocks.jobsRepository.updateScheduleById).toHaveBeenCalledWith("m1", "check", 30000);
expect(mocks.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith("m1", "geo-check");
});
it("updateJob upserts the geo row when geo is enabled", async () => {
const { worker, mocks } = createWorker();
active = worker;
await worker.updateJob(makeMonitor({ geoCheckEnabled: true }));
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
expect(types).toContain("geo-check");
expect(mocks.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled();
});
});
// ── shutdown ────────────────────────────────────────────────────────────────
describe("shutdown", () => {
it("deregisters the worker and stops the polling loops", async () => {
const { worker, mocks } = createWorker({ queueMode: "worker" });
await worker.init();
await jest.advanceTimersByTimeAsync(1);
const claimsAfterInit = mocks.jobsRepository.claimDueBatch.mock.calls.length;
await worker.shutdown();
expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1");
// Loops must not re-arm after shutdown.
await jest.advanceTimersByTimeAsync(POLL_MS * 4);
expect(mocks.jobsRepository.claimDueBatch.mock.calls.length).toBe(claimsAfterInit);
// already shut down; keep afterEach from double-shutting-down
active = null;
});
});
// ── flushQueues ───────────────────────────────────────────────────────────────
describe("flushQueues", () => {
it("restarts the worker and reports success", async () => {
const { worker, mocks } = createWorker({ queueMode: "worker" });
active = worker;
await worker.init();
await jest.advanceTimersByTimeAsync(1);
const result = await worker.flushQueues();
await jest.advanceTimersByTimeAsync(1);
expect(result).toEqual({ success: true });
expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1"); // shutdown leg
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledTimes(2); // init + re-init
});
});
// ── getMetrics ──────────────────────────────────────────────────────────────
describe("getMetrics", () => {
it("aggregates run/fail counts, active leases, and failing jobs across all rows", async () => {
const now = Date.now();
const rows = [
makeJob({ id: "a", runCount: 5, failCount: 0, lockedBy: "w", lockedUntil: now + 10000 }), // active lease
makeJob({ id: "b", runCount: 2, failCount: 3, lastFailReason: "boom", lockedBy: null, lockedUntil: null }), // failing
makeJob({ id: "c", runCount: 1, failCount: 0, lockedBy: "w", lockedUntil: now - 10000 }), // expired lease, not active
];
const { worker, mocks } = createWorker();
active = worker;
mocks.jobsRepository.findAll.mockResolvedValue(rows);
mocks.queueWorkersRepository.findRecent.mockResolvedValue([{ workerId: "w" }]);
const metrics = await worker.getMetrics();
expect(metrics.jobs).toBe(3);
expect(metrics.totalRuns).toBe(8);
expect(metrics.totalFailures).toBe(3);
expect(metrics.activeJobs).toBe(1);
expect(metrics.failingJobs).toBe(1);
expect(metrics.jobsWithFailures).toHaveLength(1);
expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ failCount: 3, failReason: "boom" }));
expect(metrics.workers).toEqual([{ workerId: "w" }]);
});
});
// ── getJobs ─────────────────────────────────────────────────────────────────
describe("getJobs", () => {
it("maps rows to summaries and passes the total count through", async () => {
const { worker, mocks } = createWorker();
active = worker;
mocks.jobsRepository.findPage.mockResolvedValue({
jobs: [makeJob({ id: "check:m1", refId: "m1", type: "check", intervalMs: 60000 })],
count: 1,
});
const page = await worker.getJobs({ page: 0, rowsPerPage: 10 });
expect(page.count).toBe(1);
expect(page.jobs).toHaveLength(1);
expect(page.jobs[0]).toEqual(expect.objectContaining({ monitorId: "m1", monitorType: "check", monitorInterval: 60000, repeat: 60000 }));
});
it("falls back to the job id for monitorId when refId is null (global jobs)", async () => {
const { worker, mocks } = createWorker();
active = worker;
mocks.jobsRepository.findPage.mockResolvedValue({
jobs: [makeJob({ id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" })],
count: 1,
});
const page = await worker.getJobs({ page: 0, rowsPerPage: 10 });
expect(page.jobs[0].monitorId).toBe("cleanup-orphaned");
});
});
});
@@ -1,791 +0,0 @@
import { describe, expect, it, jest } from "@jest/globals";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
// ── Mock Scheduler / MongoStore ──────────────────────────────────────────────
const createScheduler = () => {
const listeners: Record<string, Function[]> = {};
return {
workerId: "host-1:1234:abcd",
on: jest.fn((event: string, cb: Function) => {
listeners[event] = listeners[event] || [];
listeners[event].push(cb);
}),
off: jest.fn((event: string, cb: Function) => {
listeners[event] = (listeners[event] || []).filter((fn) => fn !== cb);
}),
emit: (event: string, ...args: unknown[]) => {
(listeners[event] || []).forEach((cb) => cb(...args));
},
start: jest.fn().mockResolvedValue(true),
stop: jest.fn().mockResolvedValue(true),
addTemplate: jest.fn(),
addJob: jest.fn(),
addJobs: jest.fn().mockResolvedValue({ inserted: 0, upserted: 0, modified: 0, failures: [] }),
removeJob: jest.fn(),
getJob: jest.fn().mockResolvedValue(null),
getJobs: jest.fn().mockResolvedValue([]),
countJobs: jest.fn().mockResolvedValue(0),
getStats: jest.fn().mockResolvedValue({ jobs: 0, activeJobs: 0, failingJobs: 0, totalRuns: 0, totalFailures: 0, jobsWithFailures: [] }),
pauseJob: jest.fn().mockResolvedValue(true),
resumeJob: jest.fn().mockResolvedValue(true),
updateJob: jest.fn(),
flushJobs: jest.fn().mockResolvedValue(true),
};
};
const mockSchedulerInstance = createScheduler();
const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance);
const MockMongoStore = jest.fn();
jest.unstable_mockModule("less-simple-scheduler", () => ({
Scheduler: MockScheduler,
MongoStore: MockMongoStore,
}));
const { LessSimpleWorker: LessSimpleQueue } = await import("../../../src/worker/worker.less-simple.ts");
const createQueueHelper = () => ({
getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()),
getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()),
getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()),
getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()),
});
const createMonitorsRepo = () => ({
findAll: jest.fn().mockResolvedValue([]),
});
const createWorkersRepo = () => ({
upsert: jest.fn().mockResolvedValue(undefined),
findRecent: jest.fn().mockResolvedValue([]),
deleteById: jest.fn().mockResolvedValue(undefined),
});
const createQueue = (overrides?: Record<string, unknown>) => {
const logger = createMockLogger();
const helper = createQueueHelper();
const monitorsRepository = createMonitorsRepo();
const workersRepository = createWorkersRepo();
const scheduler = createScheduler();
const defaults = { logger, helper, monitorsRepository, workersRepository, scheduler, queueMode: "primary", ...overrides };
const queue = new LessSimpleQueue(
defaults.logger as any,
defaults.helper as any,
defaults.monitorsRepository as any,
defaults.workersRepository as any,
defaults.scheduler as any,
defaults.queueMode as any
);
return { queue, ...defaults };
};
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "mon-1",
teamId: "team-1",
type: "http",
interval: 60000,
isActive: true,
geoCheckEnabled: false,
geoCheckInterval: 300000,
geoCheckLocations: [],
...overrides,
}) as Monitor;
// ── Tests ────────────────────────────────────────────────────────────────────
describe("LessSimpleQueue", () => {
describe("serviceName", () => {
it("returns JobQueue", () => {
const { queue } = createQueue();
expect(queue.serviceName).toBe("JobQueue");
});
});
// ── init (primary) ─────────────────────────────────────────────────────────
describe("init - primary mode", () => {
it("starts scheduler, registers all templates, seeds monitors and system jobs", async () => {
const { queue, scheduler, monitorsRepository } = createQueue();
const result = await queue.init();
expect(result).toBe(true);
expect(scheduler.start).toHaveBeenCalled();
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
expect(monitorsRepository.findAll).toHaveBeenCalled();
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
});
it("registers existing monitors in a single bulk write", async () => {
const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })];
const { queue, scheduler, monitorsRepository } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors);
await queue.init();
expect(scheduler.addJobs).toHaveBeenCalledTimes(1);
expect(scheduler.addJobs).toHaveBeenCalledWith(
expect.arrayContaining([
expect.objectContaining({ id: "m1", template: "monitor-job", jitter: true }),
expect.objectContaining({ id: "m2", template: "monitor-job", jitter: true }),
])
);
});
it("returns true and skips system jobs when findAll returns null", async () => {
const { queue, monitorsRepository, scheduler } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(null);
const result = await queue.init();
expect(result).toBe(true);
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
});
it("returns false and logs error on failure", async () => {
const { queue, logger, monitorsRepository } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down"));
const result = await queue.init();
expect(result).toBe(false);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") }));
});
it("logs error with String(error) for non-Error exceptions", async () => {
const { queue, logger, monitorsRepository } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error");
const result = await queue.init();
expect(result).toBe(false);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") }));
});
});
// ── init (worker) ──────────────────────────────────────────────────────────
describe("init - worker mode", () => {
it("starts scheduler and registers all templates", async () => {
const { queue, scheduler } = createQueue();
const result = await queue.init("worker");
expect(result).toBe(true);
expect(scheduler.start).toHaveBeenCalled();
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
});
it("does NOT seed monitor jobs (no findAll, primary populates the queue)", async () => {
const { queue, monitorsRepository } = createQueue();
await queue.init("worker");
expect(monitorsRepository.findAll).not.toHaveBeenCalled();
});
it("does NOT seed cleanup system jobs", async () => {
const { queue, scheduler } = createQueue();
await queue.init("worker");
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
expect(scheduler.addJob).not.toHaveBeenCalled();
});
it("logs the initialized worker with its workerId", async () => {
const { queue, logger } = createQueue();
await queue.init("worker");
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Worker queue: host-1:1234:abcd initialized" }));
});
});
// ── registerListeners ──────────────────────────────────────────────────────
describe("registerListeners (via constructor)", () => {
it("logs on scheduler:start", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("scheduler:start", "host-1:1234:abcd");
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" }));
});
it("logs on scheduler:stop", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("scheduler:stop", "host-1:1234:abcd");
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" }));
});
it("logs on scheduler:error with Error", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("scheduler:error", "w1", new Error("boom"));
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: boom" }));
});
it("logs on scheduler:error with non-Error", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("scheduler:error", "w1", "string error");
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: string error", stack: undefined }));
});
it("logs on job:locked", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:locked", "w1", { id: "m1" });
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 locked" }));
});
it("logs on job:abort", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:abort", "w1", { id: "m1" }, "timeout");
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 aborted: timeout" }));
});
it("logs on job:attempt", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:attempt", "w1", { id: "m1" }, 2);
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 attempt 2" }));
});
it("logs on job:complete", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:complete", "w1", { id: "m1" });
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 completed successfully" }));
});
it("logs on job:exhausted with Error", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:exhausted", "w1", { id: "m1" }, new Error("gave up"));
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up" }));
});
it("logs on job:exhausted with non-Error", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:exhausted", "w1", { id: "m1" }, "gave up");
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up", stack: undefined }));
});
it("logs on job:fail with Error", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:fail", "w1", { id: "m1" }, new Error("oops"), 1);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops" }));
});
it("logs on job:fail with non-Error", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:fail", "w1", { id: "m1" }, "oops", 1);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops", stack: undefined }));
});
it("logs on job:start", () => {
const { scheduler, logger } = createQueue();
scheduler.emit("job:start", "w1", { id: "m1" });
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 started" }));
});
});
// ── addJob ─────────────────────────────────────────────────────────────────
describe("addJob", () => {
it("adds monitor job to scheduler", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor());
expect(scheduler.addJobs).toHaveBeenCalledWith(
expect.arrayContaining([expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true })])
);
});
it("adds geo check job when geoCheckEnabled and type supports it", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }));
expect(scheduler.addJobs).toHaveBeenCalledWith(
expect.arrayContaining([expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 })])
);
});
it("skips geo job when type does not support geo checks", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })]));
});
it("skips geo job when geoCheckEnabled is false", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" }));
expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })]));
});
});
// ── deleteJob ──────────────────────────────────────────────────────────────
describe("deleteJob", () => {
it("removes monitor job", async () => {
const { queue, scheduler } = createQueue();
await queue.deleteJob(makeMonitor());
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1");
});
it("removes geo job if it exists", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.deleteJob(makeMonitor());
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
});
it("does not remove geo job if it does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.deleteJob(makeMonitor());
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
});
});
// ── pauseJob ───────────────────────────────────────────────────────────────
describe("pauseJob", () => {
it("pauses monitor and geo job when geo exists", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.pauseJob(makeMonitor());
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" }));
});
it("does not pause geo job when it does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.pauseJob(makeMonitor());
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo");
});
it("throws when scheduler.pauseJob returns false for the main job", async () => {
const { queue, scheduler } = createQueue();
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor");
});
it("logs error but does not throw when geo pauseJob returns false", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
(scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined();
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" }));
});
});
// ── resumeJob ──────────────────────────────────────────────────────────────
describe("resumeJob", () => {
it("resumes monitor and geo job when geo exists", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.resumeJob(makeMonitor());
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" }));
});
it("does not resume geo job when it does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.resumeJob(makeMonitor());
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo");
});
it("throws when scheduler.resumeJob returns false for the main job", async () => {
const { queue, scheduler } = createQueue();
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor");
});
it("logs error but does not throw when geo resumeJob returns false", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
(scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined();
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" }));
});
});
// ── updateJob / syncGeoJob ─────────────────────────────────────────────────
describe("updateJob", () => {
it("updates monitor job with new interval", async () => {
const { queue, scheduler } = createQueue();
const monitor = makeMonitor({ interval: 120000 });
await queue.updateJob(monitor);
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor }));
});
it("removes existing geo job when geoCheckEnabled is false", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" }));
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
});
it("removes existing geo job when type does not support geo checks", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
});
it("updates existing geo job when geoCheckEnabled", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
await queue.updateJob(monitor);
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 }));
});
it("creates new geo job when enabled but no existing job", async () => {
const { queue, scheduler } = createQueue();
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
await queue.updateJob(monitor);
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" }));
});
});
// ── shutdown ───────────────────────────────────────────────────────────────
describe("shutdown", () => {
it("stops the scheduler", async () => {
const { queue, scheduler } = createQueue();
await queue.shutdown();
expect(scheduler.stop).toHaveBeenCalled();
});
});
// ── getMetrics ─────────────────────────────────────────────────────────────
describe("getMetrics", () => {
it("returns empty metrics when no jobs", async () => {
const { queue } = createQueue();
const metrics = await queue.getMetrics();
expect(metrics).toEqual({ jobs: 0, activeJobs: 0, failingJobs: 0, jobsWithFailures: [], totalRuns: 0, totalFailures: 0, workers: [] });
});
it("maps the store's aggregate counters through", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getStats as jest.Mock).mockResolvedValue({
jobs: 2,
activeJobs: 1,
failingJobs: 1,
totalRuns: 15,
totalFailures: 2,
jobsWithFailures: [],
});
const metrics = await queue.getMetrics();
expect(metrics).toMatchObject({ jobs: 2, activeJobs: 1, failingJobs: 1, totalRuns: 15, totalFailures: 2 });
});
it("derives monitor fields for failing monitor jobs from job data", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getStats as jest.Mock).mockResolvedValue({
jobs: 1,
activeJobs: 0,
failingJobs: 1,
totalRuns: 10,
totalFailures: 2,
jobsWithFailures: [{ id: "m1", data: { url: "http://a.com", type: "http" }, failedAt: 200, failCount: 2, failReason: "timeout" }],
});
const metrics = await queue.getMetrics();
expect(metrics.jobsWithFailures).toEqual([
{ monitorId: "m1", monitorUrl: "http://a.com", monitorType: "http", failedAt: 200, failCount: 2, failReason: "timeout" },
]);
});
it("leaves monitor fields null for non-monitor (cleanup) failures", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getStats as jest.Mock).mockResolvedValue({
jobs: 1,
activeJobs: 0,
failingJobs: 1,
totalRuns: 1,
totalFailures: 1,
jobsWithFailures: [{ id: "cleanup-orphaned", data: undefined, failedAt: 100, failCount: 1, failReason: "boom" }],
});
const metrics = await queue.getMetrics();
expect(metrics.jobsWithFailures[0]).toEqual({
monitorId: "cleanup-orphaned",
monitorUrl: null,
monitorType: null,
failedAt: 100,
failCount: 1,
failReason: "boom",
});
});
});
// ── getMetrics: worker registry ──────────────────────────────────────────────
describe("getMetrics - worker registry", () => {
it("returns the alive workers from the registry", async () => {
const { queue, workersRepository } = createQueue();
const alive = [
{ workerId: "host:1:aaa", mode: "primary", lastSeenAt: 1000 },
{ workerId: "host:2:bbb", mode: "worker", lastSeenAt: 2000 },
];
(workersRepository.findRecent as jest.Mock).mockResolvedValue(alive);
const metrics = await queue.getMetrics();
expect(workersRepository.findRecent).toHaveBeenCalledWith(30000);
expect(metrics.workers).toEqual(alive);
});
});
// ── getJobs ────────────────────────────────────────────────────────────────
describe("getJobs", () => {
it("maps scheduler jobs to summaries", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{
id: "m1",
template: "monitor-job",
active: true,
repeat: 60000,
lockedAt: null,
runCount: 5,
failCount: 1,
lastError: "timeout",
lastStartedAt: 1000,
lastFinishedAt: 1100,
lastFailedAt: 900,
data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true },
},
]);
const { jobs } = await queue.getJobs({});
expect(jobs).toHaveLength(1);
expect(jobs[0]).toEqual({
monitorId: "m1",
monitorUrl: "http://a.com",
monitorType: "http",
monitorInterval: 60000,
monitorGeoInterval: 300000,
monitorActive: true,
active: true,
repeat: 60000,
lockedBy: null,
lockedUntil: null,
lockedAt: null,
runCount: 5,
failCount: 1,
failReason: "timeout",
lastRunAt: 1000,
lastFinishedAt: 1100,
lastRunTook: 100,
lastFailedAt: 900,
});
});
it("returns null lastRunTook when job is locked", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{
id: "m1",
template: "monitor-job",
active: true,
lockedAt: 999,
runCount: 1,
lastStartedAt: 1000,
lastFinishedAt: 1100,
data: { url: "a", type: "http", interval: 60000 },
},
]);
const { jobs } = await queue.getJobs({});
expect(jobs[0].lastRunTook).toBeNull();
});
it("leaves monitor fields null for non-monitor jobs", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, data: undefined }]);
const { jobs } = await queue.getJobs({});
expect(jobs[0].monitorUrl).toBeNull();
expect(jobs[0].monitorType).toBeNull();
expect(jobs[0].monitorInterval).toBeNull();
expect(jobs[0].monitorGeoInterval).toBeNull();
expect(jobs[0].monitorActive).toBeNull();
});
it("pushes pagination to the scheduler and returns its total count", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{ id: "m1", template: "monitor-job", active: true, data: { url: "a", type: "http", interval: 60000 } },
]);
(scheduler.countJobs as jest.Mock).mockResolvedValue(42);
const { jobs, count } = await queue.getJobs({ page: 2, rowsPerPage: 10 });
expect(scheduler.getJobs).toHaveBeenCalledWith({ skip: 20, limit: 10 });
expect(count).toBe(42);
expect(jobs).toHaveLength(1);
});
it("requests all jobs when rowsPerPage is omitted", async () => {
const { queue, scheduler } = createQueue();
(scheduler.countJobs as jest.Mock).mockResolvedValue(7);
const { count } = await queue.getJobs({});
expect(scheduler.getJobs).toHaveBeenCalledWith(undefined);
expect(count).toBe(7);
});
});
// ── flushQueues ────────────────────────────────────────────────────────────
describe("flushQueues", () => {
it("stops, flushes, and reinitializes", async () => {
const { queue, scheduler } = createQueue();
const result = await queue.flushQueues();
expect(scheduler.stop).toHaveBeenCalled();
expect(scheduler.flushJobs).toHaveBeenCalled();
expect(result.success).toBe(true);
});
it("flushes before stopping, so the store is still connected for removeAll", async () => {
const { queue, scheduler } = createQueue();
const order: string[] = [];
(scheduler.flushJobs as jest.Mock).mockImplementation(async () => {
order.push("flush");
return true;
});
(scheduler.stop as jest.Mock).mockImplementation(async () => {
order.push("stop");
return true;
});
await queue.flushQueues();
expect(order).toEqual(["flush", "stop"]);
});
it("refuses to flush on a non-primary worker without touching the scheduler", async () => {
const { queue, scheduler } = createQueue({ queueMode: "worker" });
const result = await queue.flushQueues();
expect(result.success).toBe(false);
expect(scheduler.flushJobs).not.toHaveBeenCalled();
expect(scheduler.stop).not.toHaveBeenCalled();
});
it("returns false when flush fails", async () => {
const { queue, scheduler } = createQueue();
(scheduler.flushJobs as jest.Mock).mockResolvedValue(false);
const result = await queue.flushQueues();
expect(result.success).toBe(false);
});
});
// ── static create ──────────────────────────────────────────────────────────
describe("static create", () => {
const envSettings = { dbConnectionString: "mongodb://localhost:27017/test_db" } as any;
it("creates a MongoStore + Scheduler and inits in primary mode", async () => {
const logger = createMockLogger();
const helper = createQueueHelper();
const monitorsRepository = createMonitorsRepo();
const workersRepository = createWorkersRepo();
const instance = await LessSimpleQueue.create(
logger as any,
helper as any,
monitorsRepository as any,
workersRepository as any,
envSettings,
"primary"
);
expect(MockMongoStore).toHaveBeenCalledWith(expect.objectContaining({ url: "mongodb://localhost:27017/test_db" }));
expect(MockScheduler).toHaveBeenCalled();
expect(instance).toBeInstanceOf(LessSimpleQueue);
expect(mockSchedulerInstance.start).toHaveBeenCalled();
expect(monitorsRepository.findAll).toHaveBeenCalled();
});
it("inits in worker mode without seeding monitors", async () => {
const logger = createMockLogger();
const helper = createQueueHelper();
const monitorsRepository = createMonitorsRepo();
const workersRepository = createWorkersRepo();
const instance = await LessSimpleQueue.create(
logger as any,
helper as any,
monitorsRepository as any,
workersRepository as any,
envSettings,
"worker"
);
expect(instance).toBeInstanceOf(LessSimpleQueue);
expect(monitorsRepository.findAll).not.toHaveBeenCalled();
});
});
// ── worker heartbeat / deregistration ────────────────────────────────────────
describe("worker heartbeat", () => {
it("writes a heartbeat on init with the workerId and mode (primary)", async () => {
const { queue, workersRepository } = createQueue();
await queue.init("primary");
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "primary");
});
it("writes a heartbeat on init with the workerId and mode (worker)", async () => {
const { queue, workersRepository } = createQueue();
await queue.init("worker");
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker");
});
it("beats again on each scheduler:heartbeat event", async () => {
const { queue, scheduler, workersRepository } = createQueue();
await queue.init("worker");
(workersRepository.upsert as jest.Mock).mockClear();
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
expect(workersRepository.upsert).toHaveBeenCalledTimes(2);
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker");
});
it("stops beating after shutdown", async () => {
const { queue, scheduler, workersRepository } = createQueue();
await queue.init("worker");
await queue.shutdown();
(workersRepository.upsert as jest.Mock).mockClear();
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
expect(workersRepository.upsert).not.toHaveBeenCalled();
});
it("deregisters the worker and stops the scheduler on shutdown", async () => {
const { queue, scheduler, workersRepository } = createQueue();
await queue.init("worker");
await queue.shutdown();
expect(workersRepository.deleteById).toHaveBeenCalledWith("host-1:1234:abcd");
expect(scheduler.stop).toHaveBeenCalled();
});
});
});
@@ -1,712 +0,0 @@
import { describe, expect, it, jest, beforeEach } from "@jest/globals";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
// ── Mock Scheduler ───────────────────────────────────────────────────────────
const createScheduler = () => {
const listeners: Record<string, Function[]> = {};
return {
on: jest.fn((event: string, cb: Function) => {
listeners[event] = listeners[event] || [];
listeners[event].push(cb);
}),
emit: (event: string, ...args: unknown[]) => {
(listeners[event] || []).forEach((cb) => cb(...args));
},
start: jest.fn(),
stop: jest.fn().mockResolvedValue(true),
addTemplate: jest.fn(),
addJob: jest.fn(),
removeJob: jest.fn(),
getJob: jest.fn().mockResolvedValue(null),
getJobs: jest.fn().mockResolvedValue([]),
pauseJob: jest.fn().mockResolvedValue(true),
resumeJob: jest.fn().mockResolvedValue(true),
updateJob: jest.fn(),
flushJobs: jest.fn().mockResolvedValue(true),
};
};
const mockSchedulerInstance = createScheduler();
const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance);
jest.unstable_mockModule("super-simple-scheduler", () => ({
default: MockScheduler,
}));
const { SuperSimpleQueue } = await import("../../../src/worker/worker.super-simple.ts");
const createQueueHelper = () => ({
getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()),
getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()),
getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()),
getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()),
});
const createMonitorsRepo = () => ({
findAll: jest.fn().mockResolvedValue([]),
});
const createQueue = (overrides?: Record<string, unknown>) => {
const logger = createMockLogger();
const helper = createQueueHelper();
const monitorsRepository = createMonitorsRepo();
const scheduler = createScheduler();
const defaults = { logger, helper, monitorsRepository, scheduler, ...overrides };
const queue = new SuperSimpleQueue(defaults.logger as any, defaults.helper as any, defaults.monitorsRepository as any, defaults.scheduler as any);
return { queue, ...defaults };
};
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "mon-1",
teamId: "team-1",
type: "http",
interval: 60000,
isActive: true,
geoCheckEnabled: false,
geoCheckInterval: 300000,
geoCheckLocations: [],
...overrides,
}) as Monitor;
// ── Tests ────────────────────────────────────────────────────────────────────
describe("SuperSimpleQueue", () => {
describe("serviceName", () => {
it("returns JobQueue", () => {
const { queue } = createQueue();
expect(queue.serviceName).toBe("JobQueue");
});
});
// ── init ─────────────────────────────────────────────────────────────────
describe("init", () => {
it("registers listeners, starts scheduler, adds templates and system jobs", async () => {
const { queue, scheduler, helper } = createQueue();
const result = await queue.init();
expect(result).toBe(true);
expect(scheduler.start).toHaveBeenCalled();
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
});
it("schedules existing monitors with staggered offsets", async () => {
jest.useFakeTimers();
const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })];
const { queue, scheduler, monitorsRepository } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors);
await queue.init();
jest.runAllTimers();
// Each monitor gets addJob called (once for the monitor itself via addJob method)
// The init schedules via setTimeout → addJob, plus the two system jobs
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m1", template: "monitor-job" }));
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m2", template: "monitor-job" }));
jest.useRealTimers();
});
it("returns true when findAll returns null", async () => {
const { queue, monitorsRepository, scheduler } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(null);
const result = await queue.init();
expect(result).toBe(true);
// System jobs should not be added when monitors is null (early return)
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
});
it("returns false and logs error on failure", async () => {
const { queue, logger, monitorsRepository } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down"));
const result = await queue.init();
expect(result).toBe(false);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") }));
});
it("logs error with String(error) for non-Error exceptions", async () => {
const { queue, logger, monitorsRepository } = createQueue();
(monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error");
const result = await queue.init();
expect(result).toBe(false);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") }));
});
});
// ── registerListeners ────────────────────────────────────────────────────
describe("registerListeners (via init)", () => {
it("logs on scheduler:start", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("scheduler:start");
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" }));
});
it("logs on scheduler:stop", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("scheduler:stop");
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" }));
});
it("logs on scheduler:error with Error", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("scheduler:error", new Error("boom"));
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: boom" }));
});
it("logs on scheduler:error with non-Error", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("scheduler:error", "string error");
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: string error", stack: undefined }));
});
it("logs on job:abort", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:abort", { id: "m1" }, "timeout");
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 aborted: timeout" }));
});
it("logs on job:attempt", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:attempt", { id: "m1" }, 2);
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 attempt 2" }));
});
it("logs on job:complete", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:complete", { id: "m1" });
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 completed successfully" }));
});
it("logs on job:exhausted with Error", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:exhausted", { id: "m1" }, new Error("gave up"));
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up" }));
});
it("logs on job:exhausted with non-Error", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:exhausted", { id: "m1" }, "gave up");
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up", stack: undefined }));
});
it("logs on job:fail with Error", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:fail", { id: "m1" }, new Error("oops"), 1);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops" }));
});
it("logs on job:fail with non-Error", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:fail", { id: "m1" }, "oops", 1);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops", stack: undefined }));
});
it("logs on job:start", async () => {
const { queue, scheduler, logger } = createQueue();
await queue.init();
scheduler.emit("job:start", { id: "m1" });
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 started" }));
});
});
// ── addJob ───────────────────────────────────────────────────────────────
describe("addJob", () => {
it("adds monitor job to scheduler", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor());
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true }));
});
it("adds geo check job when geoCheckEnabled and type supports it", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }));
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 }));
});
it("skips geo job when type does not support geo checks", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" }));
});
it("skips geo job when geoCheckEnabled is false", async () => {
const { queue, scheduler } = createQueue();
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" }));
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" }));
});
});
// ── deleteJob ────────────────────────────────────────────────────────────
describe("deleteJob", () => {
it("removes monitor job", async () => {
const { queue, scheduler } = createQueue();
await queue.deleteJob(makeMonitor());
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1");
});
it("removes geo job if it exists", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.deleteJob(makeMonitor());
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
});
it("does not remove geo job if it does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.deleteJob(makeMonitor());
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
});
});
// ── pauseJob ─────────────────────────────────────────────────────────────
describe("pauseJob", () => {
it("pauses monitor and pauses geo job when geo exists", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.pauseJob(makeMonitor());
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" }));
});
it("refreshes scheduler data with current monitor on pause", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
const monitor = makeMonitor({ isActive: false, status: "paused" });
await queue.pauseJob(monitor);
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor });
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor });
});
it("does not pause geo job when it does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.pauseJob(makeMonitor());
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo");
});
it("does not refresh geo data when geo job does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.pauseJob(makeMonitor());
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() }));
expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything());
});
it("throws when scheduler.pauseJob returns false for the main job", async () => {
const { queue, scheduler } = createQueue();
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor");
});
it("does not refresh data when main pauseJob throws", async () => {
const { queue, scheduler } = createQueue();
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow();
expect(scheduler.updateJob).not.toHaveBeenCalled();
});
it("logs error but does not throw when geo pauseJob returns false", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
(scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined();
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" }));
});
});
// ── resumeJob ────────────────────────────────────────────────────────────
describe("resumeJob", () => {
it("resumes monitor and geo job when geo exists", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.resumeJob(makeMonitor());
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" }));
});
it("refreshes scheduler data with current monitor on resume", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
const monitor = makeMonitor({ isActive: true, status: "initializing" });
await queue.resumeJob(monitor);
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor });
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor });
});
it("does not resume geo job when it does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.resumeJob(makeMonitor());
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo");
});
it("does not refresh geo data when geo job does not exist", async () => {
const { queue, scheduler } = createQueue();
await queue.resumeJob(makeMonitor());
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() }));
expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything());
});
it("throws when scheduler.resumeJob returns false for the main job", async () => {
const { queue, scheduler } = createQueue();
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor");
});
it("does not refresh data when main resumeJob throws", async () => {
const { queue, scheduler } = createQueue();
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow();
expect(scheduler.updateJob).not.toHaveBeenCalled();
});
it("logs error but does not throw when geo resumeJob returns false", async () => {
const { queue, scheduler, logger } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
(scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined();
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" }));
});
it("preserves geo job through a pause/resume cycle", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.pauseJob(makeMonitor());
await queue.resumeJob(makeMonitor());
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
});
});
// ── updateJob ────────────────────────────────────────────────────────────
describe("updateJob", () => {
it("updates monitor job and syncs geo job", async () => {
const { queue, scheduler } = createQueue();
const monitor = makeMonitor({ interval: 120000 });
await queue.updateJob(monitor);
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor }));
});
});
// ── syncGeoJob (via updateJob) ───────────────────────────────────────────
describe("syncGeoJob (via updateJob)", () => {
it("removes existing geo job when geoCheckEnabled is false", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" }));
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
});
it("removes existing geo job when type does not support geo checks", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
});
it("does not remove when geo disabled and no existing geo job", async () => {
const { queue, scheduler } = createQueue();
await queue.updateJob(makeMonitor({ geoCheckEnabled: false }));
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
});
it("updates existing geo job when geoCheckEnabled", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
await queue.updateJob(monitor);
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 }));
});
it("creates new geo job when enabled but no existing job", async () => {
const { queue, scheduler } = createQueue();
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
await queue.updateJob(monitor);
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" }));
});
});
// ── shutdown ──────────────────────────────────────────────────────────────
describe("shutdown", () => {
it("stops the scheduler", async () => {
const { queue, scheduler } = createQueue();
await queue.shutdown();
expect(scheduler.stop).toHaveBeenCalled();
});
});
// ── getMetrics ───────────────────────────────────────────────────────────
describe("getMetrics", () => {
it("returns empty job metrics plus this process as the sole worker when no jobs", async () => {
const { queue } = createQueue();
const metrics = await queue.getMetrics();
expect(metrics).toEqual({
jobs: 0,
activeJobs: 0,
failingJobs: 0,
jobsWithFailures: [],
totalRuns: 0,
totalFailures: 0,
workers: [
{
// hostname:pid:uuid, matching less-simple-scheduler's workerId format
workerId: expect.stringMatching(/^.+:\d+:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/),
mode: "primary",
lastSeenAt: expect.any(Number),
},
],
});
});
it("aggregates metrics from jobs", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{ id: "m1", runCount: 10, failCount: 2, lastFailedAt: 200, lastRunAt: 100, lockedAt: null, data: { url: "http://a.com", type: "http" } },
{ id: "m2", runCount: 5, failCount: 0, lastFailedAt: 0, lastRunAt: 50, lockedAt: 999, data: { url: "http://b.com", type: "ping" } },
]);
const metrics = await queue.getMetrics();
expect(metrics.jobs).toBe(2);
expect(metrics.totalRuns).toBe(15);
expect(metrics.totalFailures).toBe(2);
expect(metrics.activeJobs).toBe(1);
expect(metrics.failingJobs).toBe(1);
expect(metrics.jobsWithFailures).toHaveLength(1);
expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ monitorId: "m1", failCount: 2, monitorUrl: "http://a.com" }));
});
it("handles jobs with undefined fields", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", data: null }]);
const metrics = await queue.getMetrics();
expect(metrics.jobs).toBe(1);
expect(metrics.totalRuns).toBe(0);
});
it("handles failing job with null data", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{ id: "m1", runCount: 1, failCount: 1, lastFailedAt: 100, lastRunAt: 50, data: null, lastFailReason: "timeout" },
]);
const metrics = await queue.getMetrics();
expect(metrics.jobsWithFailures).toHaveLength(1);
expect(metrics.jobsWithFailures[0].monitorUrl).toBeNull();
expect(metrics.jobsWithFailures[0].monitorType).toBeNull();
expect(metrics.jobsWithFailures[0].failReason).toBe("timeout");
});
it("handles failing job with undefined lastFailedAt and lastFailReason", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", runCount: 1, failCount: 1, data: { url: "http://a.com", type: "http" } }]);
const metrics = await queue.getMetrics();
expect(metrics.jobsWithFailures[0].failedAt).toBeNull();
expect(metrics.jobsWithFailures[0].failReason).toBeNull();
});
it("does not count as failing when lastFailedAt < lastRunAt", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{ id: "m1", runCount: 10, failCount: 1, lastFailedAt: 50, lastRunAt: 100, data: { url: "a" } },
]);
const metrics = await queue.getMetrics();
expect(metrics.failingJobs).toBe(0);
expect(metrics.jobsWithFailures).toHaveLength(1);
});
});
// ── getJobs ──────────────────────────────────────────────────────────────
describe("getJobs", () => {
it("maps scheduler jobs to summaries", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{
id: "m1",
active: true,
repeat: 60000,
lockedAt: null,
runCount: 5,
failCount: 1,
lastFailReason: "timeout",
lastRunAt: 1000,
lastFinishedAt: 1100,
lastFailedAt: 900,
data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true },
},
]);
const { jobs } = await queue.getJobs({});
expect(jobs).toHaveLength(1);
expect(jobs[0]).toEqual({
monitorId: "m1",
monitorUrl: "http://a.com",
monitorType: "http",
monitorInterval: 60000,
monitorGeoInterval: 300000,
monitorActive: true,
active: true,
repeat: 60000,
lockedBy: null,
lockedUntil: null,
lockedAt: null,
runCount: 5,
failCount: 1,
failReason: "timeout",
lastRunAt: 1000,
lastFinishedAt: 1100,
lastRunTook: 100,
lastFailedAt: 900,
});
});
it("returns null lastRunTook when job is locked", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{ id: "m1", active: true, lockedAt: 999, runCount: 1, data: { url: "a", type: "http", interval: 60000 } },
]);
const { jobs } = await queue.getJobs({});
expect(jobs[0].lastRunTook).toBeNull();
});
it("handles jobs with missing data", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", active: true, data: null }]);
const { jobs } = await queue.getJobs({});
expect(jobs[0].monitorUrl).toBeNull();
expect(jobs[0].monitorType).toBeNull();
expect(jobs[0].monitorInterval).toBeNull();
expect(jobs[0].monitorGeoInterval).toBeNull();
expect(jobs[0].monitorActive).toBeNull();
expect(jobs[0].repeat).toBeNull();
});
it("exposes monitor.isActive as monitorActive in the summary", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{ id: "m1", active: true, data: { url: "a", type: "http", interval: 60000, isActive: true } },
{ id: "m2", active: false, data: { url: "b", type: "http", interval: 60000, isActive: false } },
]);
const { jobs } = await queue.getJobs({});
expect(jobs[0].monitorActive).toBe(true);
expect(jobs[1].monitorActive).toBe(false);
});
it("exposes geo check interval and repeat", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue([
{ id: "m1", active: true, repeat: 120000, data: { url: "a", type: "http", interval: 60000, geoCheckInterval: 600000 } },
]);
const { jobs } = await queue.getJobs({});
expect(jobs[0].repeat).toBe(120000);
expect(jobs[0].monitorGeoInterval).toBe(600000);
});
it("paginates jobs and returns the total count", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue(
Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } }))
);
const { jobs, count } = await queue.getJobs({ page: 1, rowsPerPage: 10 });
expect(count).toBe(25);
expect(jobs).toHaveLength(10);
expect(jobs[0].monitorId).toBe("m10");
});
it("returns all jobs when rowsPerPage is omitted", async () => {
const { queue, scheduler } = createQueue();
(scheduler.getJobs as jest.Mock).mockResolvedValue(
Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } }))
);
const { jobs, count } = await queue.getJobs({});
expect(count).toBe(25);
expect(jobs).toHaveLength(25);
});
});
// ── flushQueues ──────────────────────────────────────────────────────────
describe("flushQueues", () => {
it("stops, flushes, and reinitializes", async () => {
const { queue, scheduler } = createQueue();
const result = await queue.flushQueues();
expect(scheduler.stop).toHaveBeenCalled();
expect(scheduler.flushJobs).toHaveBeenCalled();
expect(result.success).toBe(true);
});
it("returns false when any step fails", async () => {
const { queue, scheduler } = createQueue();
(scheduler.flushJobs as jest.Mock).mockResolvedValue(false);
const result = await queue.flushQueues();
expect(result.success).toBe(false);
});
});
// ── static create ────────────────────────────────────────────────────────
describe("static create", () => {
it("creates a SuperSimpleQueue instance with a Scheduler and calls init", async () => {
const logger = createMockLogger();
const helper = createQueueHelper();
const monitorsRepository = createMonitorsRepo();
const instance = await SuperSimpleQueue.create(logger as any, helper as any, monitorsRepository as any);
expect(MockScheduler).toHaveBeenCalled();
expect(instance).toBeInstanceOf(SuperSimpleQueue);
// init was called, which calls scheduler.start
expect(mockSchedulerInstance.start).toHaveBeenCalled();
});
});
});
@@ -17,6 +17,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
findAllMonitorIds: jest.fn().mockResolvedValue(["m1"]),
deleteByTeamIdsNotIn: jest.fn().mockResolvedValue(0),
};
const jobsRepository = {
deleteByMonitorIdsNotIn: jest.fn().mockResolvedValue(0),
};
const teamsRepository = {
findAllTeamIds: jest.fn().mockResolvedValue(["team"]),
};
@@ -48,6 +51,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
checkService,
settingsService,
monitorsRepository,
jobsRepository,
teamsRepository,
monitorStatsRepository,
checksRepository,
@@ -64,6 +68,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
defaults.checkService as any,
defaults.settingsService as any,
defaults.monitorsRepository as any,
defaults.jobsRepository as any,
defaults.teamsRepository as any,
defaults.monitorStatsRepository as any,
defaults.checksRepository as any,
@@ -170,6 +175,7 @@ describe("WorkerHelper", () => {
(defaults.checksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(4);
(defaults.incidentsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(1);
(defaults.geoChecksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(5);
(defaults.jobsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(6);
const job = helper.getCleanupOrphanedJob();
await job();
@@ -181,11 +187,13 @@ describe("WorkerHelper", () => {
expect(defaults.checksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.incidentsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.geoChecksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.jobsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("2 orphaned monitors") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("3 orphaned monitor stats") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("4 orphaned checks") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 orphaned incidents") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("5 orphaned geo checks") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("6 orphaned jobs") }));
});
it("skips info logs when no orphaned data is found", async () => {