Merge pull request #3728 from bluewave-labs/feat/db-queue
feat: db queue
This commit is contained in:
42 files changed
+3499
-3514
No files matched your search
+2
-1
@@ -11,7 +11,8 @@ LOG_LEVEL=debug
|
||||
# Database
|
||||
# --------
|
||||
DB_CONNECTION_STRING=mongodb://localhost:27017/uptime_db
|
||||
QUEUE_TYPE=superSimpleQueue
|
||||
QUEUE_MODE="primary"
|
||||
QUEUE_PRIMARY_PROCESSES="true"
|
||||
|
||||
# JWT Authentication
|
||||
# ------------------
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"ignore": ["node_modules/*"],
|
||||
"watch": ["src/**/*.ts", "src/**/*.js", "*.json"],
|
||||
"watch": ["src/**/*.ts", "src/**/*.js", "*.json", ".env"],
|
||||
"ext": "ts,js,json"
|
||||
}
|
||||
Generated
+1181
-988
File diff suppressed because it is too large.
Load diff
+1
-4
@@ -33,7 +33,6 @@
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.14.3",
|
||||
"@grpc/proto-loader": "^0.8.0",
|
||||
"@pulsecron/pulse": "1.6.8",
|
||||
"axios": "^1.15.2",
|
||||
"bcryptjs": "3.0.2",
|
||||
"cacheable-lookup": "7.0.0",
|
||||
@@ -48,12 +47,10 @@
|
||||
"got": "14.4.7",
|
||||
"handlebars": "^4.7.8",
|
||||
"helmet": "^8.0.0",
|
||||
"ioredis": "^5.4.2",
|
||||
"isomorphic-dompurify": "^2.26.0",
|
||||
"jmespath": "^0.16.0",
|
||||
"jsdom": "^26.1.0",
|
||||
"jsonwebtoken": "9.0.2",
|
||||
"less-simple-scheduler": "2.6.0",
|
||||
"mailersend": "^2.2.0",
|
||||
"mjml": "^5.0.0-alpha.4",
|
||||
"mongoose": "^8.3.3",
|
||||
@@ -62,7 +59,6 @@
|
||||
"ping": "0.4.4",
|
||||
"sharp": "0.33.5",
|
||||
"ssl-checker": "2.0.10",
|
||||
"super-simple-scheduler": "1.9.2",
|
||||
"swagger-ui-express": "5.0.1",
|
||||
"winston": "^3.13.0",
|
||||
"ws": "^8.19.0",
|
||||
@@ -96,6 +92,7 @@
|
||||
"globals": "^15.14.0",
|
||||
"jest": "^30.2.0",
|
||||
"lint-staged": "^16.2.7",
|
||||
"mongodb-memory-server": "11.2.0",
|
||||
"nodemon": "^3.1.11",
|
||||
"prettier": "^3.3.3",
|
||||
"prettier-plugin-ini": "1.3.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import mongoose from "mongoose";
|
||||
import { MonitorModel } from "../dist/db/models/Monitor.js";
|
||||
import { UserModel } from "../dist/db/models/User.js";
|
||||
import { MonitorModel } from "../dist/domain/monitors/monitor.model.js";
|
||||
import { UserModel } from "../dist/domain/users/user.model.js";
|
||||
|
||||
// Seeds a large number of HTTP monitors against real, resolving domains so the
|
||||
// scheduler can be exercised under load. Run after `npm run build` (imports the
|
||||
@@ -241,6 +241,7 @@ async function run() {
|
||||
url: `https://${domain}`,
|
||||
isActive: true,
|
||||
interval,
|
||||
lastEvaluatedAt: 0, // evaluator watermark; without it findUnevaluatedByMonitorId gets new Date(undefined)
|
||||
});
|
||||
|
||||
if (docs.length === batchSize) {
|
||||
|
||||
@@ -272,5 +272,6 @@ export const monitorResponseSchema = z
|
||||
userId: z.string(),
|
||||
createdAt: z.string(),
|
||||
updatedAt: z.string(),
|
||||
lastEvaluatedAt: z.number(),
|
||||
})
|
||||
.passthrough();
|
||||
@@ -1,5 +1,5 @@
|
||||
import { z } from "zod";
|
||||
import { DbTypes, QueueTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { booleanCoercion } from "@/api/validation/shared.js";
|
||||
|
||||
const envSchema = z.object({
|
||||
@@ -12,8 +12,8 @@ const envSchema = z.object({
|
||||
DB_CONNECTION_STRING: z.string().min(1, "Database connection string is required"),
|
||||
DB_TYPE: z.enum(DbTypes).default("mongodb"),
|
||||
|
||||
QUEUE_TYPE: z.enum(QueueTypes).default("superSimpleQueue"),
|
||||
QUEUE_MODE: z.enum(QueueModes).default("primary"),
|
||||
QUEUE_PRIMARY_PROCESSES: booleanCoercion.default(true),
|
||||
|
||||
// JWT Authentication
|
||||
JWT_SECRET: z.string().min(1, "JWT_SECRET is required"),
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Mongoose } from "mongoose";
|
||||
import { hostname } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import MongoDB from "../db/db.mongo.js";
|
||||
import { IDb } from "@/db/db.interface.js";
|
||||
import { EnvConfig, ISettingsService, SettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
@@ -30,8 +32,6 @@ import { EmailService, IEmailService } from "@/service/emailService.js";
|
||||
import { GlobalPingService } from "@/service/globalPingService.js";
|
||||
import { WorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IWorker } from "@/worker/worker.interface.js";
|
||||
import { LessSimpleWorker } from "@/worker/worker.less-simple.js";
|
||||
import { SuperSimpleQueue } from "@/worker/worker.super-simple.js";
|
||||
import { INetworkService, NetworkService } from "@/service/networkService.js";
|
||||
import { IStatusService, StatusService } from "@/service/statusService.js";
|
||||
import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
@@ -89,7 +89,6 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
|
||||
import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js";
|
||||
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
|
||||
import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js";
|
||||
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
|
||||
import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js";
|
||||
import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js";
|
||||
import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js";
|
||||
@@ -101,12 +100,15 @@ import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
|
||||
import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
|
||||
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
|
||||
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { CheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
|
||||
import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js";
|
||||
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
|
||||
|
||||
export type InitializedServices = {
|
||||
settingsService: ISettingsService;
|
||||
@@ -151,12 +153,10 @@ export const initializeServices = async ({
|
||||
logger,
|
||||
envSettings,
|
||||
settingsService,
|
||||
queueMode = "primary",
|
||||
}: {
|
||||
logger: ILogger;
|
||||
envSettings: EnvConfig;
|
||||
settingsService: ISettingsService;
|
||||
queueMode?: QueueMode;
|
||||
}): Promise<InitializedServices> => {
|
||||
// Create DB
|
||||
|
||||
@@ -164,58 +164,6 @@ export const initializeServices = async ({
|
||||
db = new MongoDB(logger, envSettings);
|
||||
await db.connect();
|
||||
|
||||
// ** NOTE **
|
||||
// const dbType = envSettings.dbType;
|
||||
// DB type has been fixed to MongoDB for now
|
||||
|
||||
// if (dbType === "mongodb") {
|
||||
// db = new MongoDB(logger, envSettings);
|
||||
// }
|
||||
|
||||
// if (!db) {
|
||||
// throw new AppError({ message: "Unsupported database type", status: 500 });
|
||||
// }
|
||||
|
||||
// await db.connect();
|
||||
|
||||
// Repositories
|
||||
|
||||
// ** NOTE **
|
||||
// DB Type fixed to MongoDB for now
|
||||
// let monitorsRepository: IMonitorsRepository;
|
||||
// let checksRepository: IChecksRepository;
|
||||
// let geoChecksRepository: IGeoChecksRepository;
|
||||
// let monitorStatsRepository: IMonitorStatsRepository;
|
||||
// let statusPagesRepository: IStatusPagesRepository;
|
||||
// let usersRepository: IUsersRepository;
|
||||
// let invitesRepository: IInvitesRepository;
|
||||
// let recoveryTokensRepository: IRecoveryTokensRepository;
|
||||
// let settingsRepository: ISettingsRepository;
|
||||
// let notificationsRepository: INotificationsRepository;
|
||||
// let tagsRepository: ITagsRepository;
|
||||
// let incidentsRepository: IIncidentsRepository;
|
||||
// let teamsRepository: ITeamsRepository;
|
||||
// let maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
|
||||
// if (dbType === "mongodb") {
|
||||
// monitorsRepository = new MongoMonitorsRepository();
|
||||
// checksRepository = new MongoChecksRepository(logger);
|
||||
// geoChecksRepository = new MongoGeoChecksRepository(logger);
|
||||
// monitorStatsRepository = new MongoMonitorStatsRepository();
|
||||
// statusPagesRepository = new MongoStatusPagesRepository();
|
||||
// usersRepository = new MongoUsersRepository();
|
||||
// invitesRepository = new MongoInvitesRepository();
|
||||
// recoveryTokensRepository = new MongoRecoveryTokensRepository();
|
||||
// settingsRepository = new MongoSettingsRepository();
|
||||
// notificationsRepository = new MongoNotificationsRepository();
|
||||
// tagsRepository = new MongoTagsRepository();
|
||||
// incidentsRepository = new MongoIncidentsRepository();
|
||||
// teamsRepository = new MongoTeamsRepository();
|
||||
// maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
|
||||
// } else {
|
||||
// throw new AppError({ message: "Unsupported database type", status: 500 });
|
||||
// }
|
||||
|
||||
const monitorsRepository = new MongoMonitorsRepository();
|
||||
const checksRepository = new MongoChecksRepository(logger);
|
||||
const geoChecksRepository = new MongoGeoChecksRepository(logger);
|
||||
@@ -230,6 +178,9 @@ export const initializeServices = async ({
|
||||
const incidentsRepository = new MongoIncidentsRepository();
|
||||
const teamsRepository = new MongoTeamsRepository();
|
||||
const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
|
||||
// One identity per process
|
||||
const workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
|
||||
const jobsRepository = new MongoJobsRepository(workerId);
|
||||
const queueWorkersRepository = new MongoQueueWorkersRepository();
|
||||
|
||||
// Inject settings repository into settings service (now that DB is connected)
|
||||
@@ -320,17 +271,12 @@ export const initializeServices = async ({
|
||||
const incidentReactor = new IncidentReactor(incidentService);
|
||||
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
|
||||
|
||||
// Check producer/evaluator
|
||||
const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger);
|
||||
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
|
||||
|
||||
// pipelines
|
||||
const checkPipeline = new CheckPipeline(
|
||||
monitorsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
checkService,
|
||||
networkService,
|
||||
bufferService,
|
||||
monitorStatusPolicy,
|
||||
statusService,
|
||||
logger
|
||||
);
|
||||
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
|
||||
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
|
||||
|
||||
@@ -339,6 +285,7 @@ export const initializeServices = async ({
|
||||
checkService,
|
||||
settingsService,
|
||||
monitorsRepository,
|
||||
jobsRepository,
|
||||
teamsRepository,
|
||||
monitorStatsRepository,
|
||||
checksRepository,
|
||||
@@ -349,23 +296,22 @@ export const initializeServices = async ({
|
||||
geoCheckPipeline
|
||||
);
|
||||
|
||||
if (queueMode === "worker" && envSettings.queueType !== "lessSimpleQueue") {
|
||||
throw new AppError({
|
||||
message: `Worker mode requires QUEUE_TYPE="lessSimpleQueue" got ${envSettings.queueType}`,
|
||||
status: 500,
|
||||
service: "config services",
|
||||
method: "initializeServices",
|
||||
});
|
||||
}
|
||||
|
||||
let worker: IWorker;
|
||||
switch (envSettings.queueType) {
|
||||
case "lessSimpleQueue":
|
||||
worker = await LessSimpleWorker.create(logger, workerHelper, monitorsRepository, queueWorkersRepository, envSettings, queueMode);
|
||||
break;
|
||||
default:
|
||||
worker = await SuperSimpleQueue.create(logger, workerHelper, monitorsRepository);
|
||||
}
|
||||
const worker = await DBQueueWorker.create(
|
||||
logger,
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
reactorDispatcher,
|
||||
workerHelper,
|
||||
queueWorkersRepository,
|
||||
envSettings.queueMode,
|
||||
envSettings.queuePrimaryProcesses,
|
||||
workerId
|
||||
);
|
||||
|
||||
// Business services
|
||||
const userService = new UserService({
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import mongoose from "mongoose";
|
||||
import { MonitorModel } from "../../domain/monitors/monitor.model.js";
|
||||
import { logger } from "@/utils/logger.js";
|
||||
|
||||
const JOBS_COLLECTION = "jobs";
|
||||
|
||||
/**
|
||||
* Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed.
|
||||
*
|
||||
* Stamp the upgrade moment, not 0. If set to 0, all checks will be replayed
|
||||
*/
|
||||
export async function backfillMonitorLastEvaluatedAt(): Promise<void> {
|
||||
const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt";
|
||||
|
||||
try {
|
||||
// Drop the legacy jobs collection first. The old super-simple-scheduler persisted jobs to the
|
||||
// same "jobs" collection the new db-queue uses, so any leftover rows carry an incompatible schema.
|
||||
// Migrations run before syncIndexes(), so the new Job indexes are rebuilt right after this drop.
|
||||
const db = mongoose.connection.db;
|
||||
if (db) {
|
||||
const existing = await db.listCollections({ name: JOBS_COLLECTION }).toArray();
|
||||
if (existing.length > 0) {
|
||||
await db.collection(JOBS_COLLECTION).drop();
|
||||
logger.info({ service: SERVICE_NAME, message: `Dropped legacy "${JOBS_COLLECTION}" collection` });
|
||||
}
|
||||
}
|
||||
|
||||
logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" });
|
||||
|
||||
const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: Date.now() } });
|
||||
|
||||
logger.info({
|
||||
service: SERVICE_NAME,
|
||||
message: `Backfilled lastEvaluatedAt on ${result.modifiedCount} monitors`,
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
logger.error({ service: SERVICE_NAME, message: `Error backfilling lastEvaluatedAt: ${errorMessage}` });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import MigrationModel from "../../domain/migrations/migration.model.js";
|
||||
import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js";
|
||||
import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js";
|
||||
import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js";
|
||||
import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
|
||||
type MigrationEntry = {
|
||||
@@ -21,6 +22,7 @@ const migrations: MigrationEntry[] = [
|
||||
{ name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray },
|
||||
{ name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex },
|
||||
{ name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray },
|
||||
{ name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt },
|
||||
];
|
||||
|
||||
const runMigrations = async (logger?: ILogger) => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
|
||||
import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ValidatedEnv } from "@/config/envValidation.js";
|
||||
import type { StringValue } from "ms";
|
||||
@@ -13,8 +13,8 @@ export type EnvConfig = {
|
||||
clientHost: string;
|
||||
dbConnectionString: string;
|
||||
dbType: DbType;
|
||||
queueType: QueueType;
|
||||
queueMode: QueueMode;
|
||||
queuePrimaryProcesses: boolean;
|
||||
statusPageThemesEnabled: boolean;
|
||||
};
|
||||
|
||||
@@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService {
|
||||
clientHost: env.CLIENT_HOST,
|
||||
dbConnectionString: env.DB_CONNECTION_STRING,
|
||||
dbType: env.DB_TYPE,
|
||||
queueType: env.QUEUE_TYPE,
|
||||
queueMode: env.QUEUE_MODE,
|
||||
queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES,
|
||||
statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
export const DbTypes = ["mongodb"] as const;
|
||||
export type DbType = (typeof DbTypes)[number];
|
||||
|
||||
export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const;
|
||||
export type QueueType = (typeof QueueTypes)[number];
|
||||
|
||||
export const QueueModes = ["primary", "worker"] as const;
|
||||
export type QueueMode = (typeof QueueModes)[number];
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ export interface IChecksRepository {
|
||||
options?: { type?: MonitorType }
|
||||
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult>;
|
||||
findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
|
||||
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
|
||||
// update
|
||||
//delete
|
||||
deleteByMonitorId(monitorId: string): Promise<number>;
|
||||
|
||||
@@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
};
|
||||
};
|
||||
|
||||
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
|
||||
const docs = await CheckModel.find({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
createdAt: { $gt: new Date(since) },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.lean<CheckDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
deleteByMonitorId = async (monitorId: string): Promise<number> => {
|
||||
const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
|
||||
return result.deletedCount;
|
||||
|
||||
@@ -11,7 +11,9 @@ const SERVICE_NAME = "checkService";
|
||||
|
||||
export interface ICheckService {
|
||||
createChecks(checks: Check[]): Promise<Check[]>;
|
||||
buildCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toLastEvaluatedAt(check: Check): number;
|
||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||
getChecksByMonitor(params: {
|
||||
monitorId: string;
|
||||
teamId: string;
|
||||
@@ -56,7 +58,11 @@ export class CheckService implements ICheckService {
|
||||
return this.checksRepository.createChecks(checks);
|
||||
};
|
||||
|
||||
buildCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
toLastEvaluatedAt = (check: Check) => {
|
||||
return new Date(check.createdAt).getTime();
|
||||
};
|
||||
|
||||
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
||||
|
||||
const now = new Date().toISOString();
|
||||
@@ -82,7 +88,7 @@ export class CheckService implements ICheckService {
|
||||
this.logger.warn({
|
||||
message: "Failed to build check",
|
||||
service: SERVICE_NAME,
|
||||
method: "buildCheck",
|
||||
method: "toCheck",
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
@@ -131,6 +137,24 @@ export class CheckService implements ICheckService {
|
||||
return check;
|
||||
};
|
||||
|
||||
toStatusResponse = (check: Check) => {
|
||||
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
|
||||
monitorId: check.metadata.monitorId,
|
||||
teamId: check.metadata.teamId,
|
||||
type: check.metadata.type,
|
||||
status: check.status,
|
||||
code: check.statusCode,
|
||||
message: check.message,
|
||||
responseTime: check.responseTime,
|
||||
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
|
||||
payload:
|
||||
check.metadata.type === "hardware"
|
||||
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
|
||||
: undefined,
|
||||
};
|
||||
return statusResponse;
|
||||
};
|
||||
|
||||
getChecksByMonitor = async ({
|
||||
monitorId,
|
||||
teamId,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { Schema, model } from "mongoose";
|
||||
import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export interface JobDocument extends Omit<Job, "id"> {
|
||||
_id: string;
|
||||
}
|
||||
|
||||
const JobSchema = new Schema<JobDocument>(
|
||||
{
|
||||
_id: {
|
||||
type: String,
|
||||
required: true,
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
enum: JOB_TYPES,
|
||||
required: true,
|
||||
},
|
||||
// monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate).
|
||||
refId: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
// paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob)
|
||||
isActive: {
|
||||
type: Boolean,
|
||||
default: true,
|
||||
},
|
||||
|
||||
// Scheduling — epoch ms, so the claim/$min queries operate on plain numbers
|
||||
nextScheduledAt: {
|
||||
type: Number,
|
||||
required: true,
|
||||
},
|
||||
intervalMs: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Lock / lease
|
||||
lockedBy: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
lockedUntil: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Observability
|
||||
runCount: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
failCount: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
lastFinishedAt: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
lastFailReason: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
// Used for scanning for claims
|
||||
JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 });
|
||||
// for findById, upsertEvaluate, deleteById
|
||||
JobSchema.index({ refId: 1, type: 1 });
|
||||
|
||||
const JobModel = model<JobDocument>("Job", JobSchema);
|
||||
|
||||
export { JobModel };
|
||||
export default JobModel;
|
||||
@@ -0,0 +1,72 @@
|
||||
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export type JobPageQuery = {
|
||||
page?: number;
|
||||
rowsPerPage?: number;
|
||||
};
|
||||
export type JobPage = {
|
||||
jobs: Job[];
|
||||
count: number;
|
||||
};
|
||||
|
||||
export interface IJobsRepository {
|
||||
// ********************
|
||||
// Claims
|
||||
// ********************
|
||||
|
||||
// Atomically claim a job
|
||||
claimDue(type: JobType, now: number): Promise<Job | null>;
|
||||
|
||||
// locks up to `limit` due rows in one batch
|
||||
claimDueBatch(type: JobType, limit: number, now: number): Promise<Job[]>;
|
||||
|
||||
// Extends the locks we still hold. Returns how many were renewed
|
||||
renewLocks(ids: string[], now: number): Promise<number>;
|
||||
|
||||
// Completes a repeating job
|
||||
recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise<boolean>;
|
||||
|
||||
// Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry
|
||||
recordFailure(id: string, error: unknown, now: number): Promise<boolean>;
|
||||
|
||||
recordOneShot(id: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
|
||||
// ********************
|
||||
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Job CRUD
|
||||
// ********************
|
||||
|
||||
// Create jobs. Takes a JobSeed
|
||||
upsertJob(job: JobSeed): Promise<boolean>;
|
||||
|
||||
// Creates cleanup jobs, these are executed immediately so need their own method
|
||||
upsertCleanupJob(job: JobSeed): Promise<boolean>;
|
||||
|
||||
// Pause/Resume
|
||||
setActiveById(refId: string, isActive: boolean): Promise<boolean>;
|
||||
|
||||
// Edit job schedule
|
||||
updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
|
||||
|
||||
// Delete, drop all rows
|
||||
deleteById(refId: string): Promise<boolean>;
|
||||
|
||||
// Delete a single job of type
|
||||
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
|
||||
|
||||
// Delete jobs that reference monitors that no longer exist
|
||||
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
|
||||
|
||||
// Get job
|
||||
findById(refId: string): Promise<Job[]>;
|
||||
|
||||
// ********************
|
||||
// Observability
|
||||
// ********************
|
||||
findPage(pagination: JobPageQuery): Promise<JobPage>;
|
||||
findAll(): Promise<Job[]>;
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
|
||||
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
|
||||
|
||||
class MongoJobsRepository implements IJobsRepository {
|
||||
constructor(private readonly workerId: string) {}
|
||||
|
||||
private toEntity(doc: JobDocument): Job {
|
||||
return {
|
||||
id: doc._id,
|
||||
type: doc.type,
|
||||
refId: doc.refId,
|
||||
isActive: doc.isActive,
|
||||
nextScheduledAt: doc.nextScheduledAt,
|
||||
intervalMs: doc.intervalMs,
|
||||
lockedBy: doc.lockedBy,
|
||||
lockedUntil: doc.lockedUntil,
|
||||
runCount: doc.runCount,
|
||||
failCount: doc.failCount,
|
||||
lastFinishedAt: doc.lastFinishedAt,
|
||||
lastFailReason: doc.lastFailReason,
|
||||
};
|
||||
}
|
||||
|
||||
claimDue = async (type: JobType, now: number) => {
|
||||
const doc = await JobModel.findOneAndUpdate(
|
||||
{
|
||||
type,
|
||||
isActive: true,
|
||||
nextScheduledAt: { $lte: now }, // Find due jobs
|
||||
$or: [
|
||||
{ lockedUntil: null }, // Not locked
|
||||
{ lockedUntil: { $lt: now } }, // Free or expired lock
|
||||
],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
lockedBy: this.workerId,
|
||||
lockedUntil: now + LOCK_MS,
|
||||
},
|
||||
},
|
||||
{
|
||||
new: true,
|
||||
sort: { nextScheduledAt: 1 }, // Get the oldest due job
|
||||
}
|
||||
).lean<JobDocument>();
|
||||
return doc ? this.toEntity(doc) : null;
|
||||
};
|
||||
|
||||
claimDueBatch = async (type: JobType, limit: number, now: number) => {
|
||||
if (limit <= 0) return [];
|
||||
const dueAndFree = {
|
||||
type,
|
||||
isActive: true,
|
||||
nextScheduledAt: { $lte: now }, // due
|
||||
$or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock
|
||||
};
|
||||
|
||||
// 1. Pick the oldest-due candidates
|
||||
const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>();
|
||||
if (candidates.length === 0) return [];
|
||||
const ids = candidates.map((d) => d._id);
|
||||
|
||||
// 2. lock them. Re-check free/expired so we never steal another worker's live lock
|
||||
// only the rows this updateMany actually flips end up with claimed
|
||||
const lockedUntil = now + LOCK_MS;
|
||||
await JobModel.updateMany(
|
||||
{ _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] },
|
||||
{ $set: { lockedBy: this.workerId, lockedUntil } }
|
||||
);
|
||||
|
||||
// 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time
|
||||
const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
renewLocks = async (ids: string[], now: number) => {
|
||||
if (ids.length === 0) return 0;
|
||||
const res = await JobModel.updateMany(
|
||||
{ _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed
|
||||
{ $set: { lockedUntil: now + LOCK_MS } }
|
||||
);
|
||||
return res.modifiedCount;
|
||||
};
|
||||
|
||||
recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => {
|
||||
// Fixed rate job scheduling
|
||||
|
||||
let nextScheduledAt = scheduledRunAt + intervalMs;
|
||||
|
||||
// If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd
|
||||
if (nextScheduledAt <= now) {
|
||||
nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs);
|
||||
}
|
||||
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
recordFailure = async (id: string, error: unknown, now: number) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: now + BACKOFF_MS,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFailReason: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
$inc: {
|
||||
failCount: 1, // Increment fail count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
recordOneShot = async (id: string, now: number) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: PARKED,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
upsertEvaluate = async (monitorId: string, now: number) => {
|
||||
const filter = { type: "evaluate", refId: monitorId };
|
||||
const update = {
|
||||
$setOnInsert: {
|
||||
_id: jobId("evaluate", monitorId),
|
||||
// refId and Type inferred from filter
|
||||
isActive: true,
|
||||
intervalMs: null,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
$min: {
|
||||
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
||||
},
|
||||
};
|
||||
try {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
return res.acknowledged;
|
||||
} catch (error) {
|
||||
// Two callers can race the first insert for the same monitor, so the
|
||||
// loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min.
|
||||
if (error && typeof error === "object" && (error as { code?: number }).code === 11000) {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
return res.acknowledged;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
upsertJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
{
|
||||
$set: {
|
||||
// These fields are owned by the monitor, always safe to set
|
||||
type: job.type,
|
||||
refId: job.refId,
|
||||
isActive: job.isActive,
|
||||
intervalMs: job.intervalMs,
|
||||
},
|
||||
$setOnInsert: {
|
||||
// These fields are scheduling related, only set on insert so they aren't clobbered
|
||||
nextScheduledAt: job.nextScheduledAt,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
return res.acknowledged;
|
||||
};
|
||||
|
||||
upsertCleanupJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
{
|
||||
$set: {
|
||||
type: job.type,
|
||||
refId: job.refId,
|
||||
isActive: job.isActive,
|
||||
intervalMs: job.intervalMs,
|
||||
nextScheduledAt: job.nextScheduledAt,
|
||||
},
|
||||
$setOnInsert: {
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
return res.acknowledged;
|
||||
};
|
||||
|
||||
setActiveById = async (refId: string, isActive: boolean) => {
|
||||
const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together
|
||||
return res.modifiedCount > 0;
|
||||
};
|
||||
|
||||
updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => {
|
||||
const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } });
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
deleteById = async (refId: string) => {
|
||||
const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByIdAndType = async (refId: string, type: JobType) => {
|
||||
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
|
||||
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
|
||||
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
|
||||
return res.deletedCount ?? 0;
|
||||
};
|
||||
|
||||
findById = async (refId: string) => {
|
||||
const docs = await JobModel.find({ refId }).lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => {
|
||||
const count = await JobModel.countDocuments();
|
||||
const query = JobModel.find().sort({ nextScheduledAt: 1 });
|
||||
if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage);
|
||||
const docs = await query.lean<JobDocument[]>();
|
||||
return { jobs: docs.map(this.toEntity), count };
|
||||
};
|
||||
|
||||
findAll = async () => {
|
||||
const docs = await JobModel.find().lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
}
|
||||
|
||||
export default MongoJobsRepository;
|
||||
@@ -0,0 +1,32 @@
|
||||
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
|
||||
export type JobType = (typeof JOB_TYPES)[number];
|
||||
export const LOCK_MS = 60_000;
|
||||
export const BACKOFF_MS = 5_000;
|
||||
export const PARKED = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
|
||||
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
|
||||
|
||||
export type Job = {
|
||||
id: string;
|
||||
type: JobType;
|
||||
refId: string | null;
|
||||
isActive: boolean;
|
||||
|
||||
// Scheduling fields
|
||||
nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now
|
||||
intervalMs: number | null; // repeat period in ms; null = one time job
|
||||
|
||||
// Lock fields
|
||||
lockedBy: string | null; // workerId holding the lease
|
||||
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
|
||||
|
||||
// observability (replaces what getMetrics/getJobs)
|
||||
runCount: number;
|
||||
failCount: number;
|
||||
lastFinishedAt: number | null;
|
||||
lastFailReason: string | null;
|
||||
};
|
||||
|
||||
// These are the only values to set when creating a job. Repo owns the rest
|
||||
export type JobSeed = Pick<Job, "id" | "type" | "refId" | "isActive" | "intervalMs" | "nextScheduledAt">;
|
||||
@@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase {
|
||||
teamId: Types.ObjectId;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
lastEvaluatedAt: number; // epoch ms
|
||||
}
|
||||
|
||||
const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
|
||||
@@ -377,6 +378,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
|
||||
type: [checkSnapshotSchema],
|
||||
default: [],
|
||||
},
|
||||
lastEvaluatedAt: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
|
||||
@@ -571,6 +571,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
dnsRecordType: doc.dnsRecordType ?? undefined,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
lastEvaluatedAt: doc.lastEvaluatedAt,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -636,6 +637,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
dnsRecordType: doc.dnsRecordType ?? undefined,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
lastEvaluatedAt: doc.lastEvaluatedAt,
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService {
|
||||
recentChecks: [],
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
lastEvaluatedAt: 0,
|
||||
}));
|
||||
|
||||
const createdMonitors = await this.createMonitors(cleanedMonitors);
|
||||
|
||||
@@ -88,6 +88,7 @@ export interface Monitor {
|
||||
recentChecks: CheckSnapshot[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
lastEvaluatedAt: number; // epoch ms
|
||||
}
|
||||
|
||||
export interface MonitorsSummary {
|
||||
|
||||
@@ -2,11 +2,15 @@ import { Schema, model } from "mongoose";
|
||||
import { QueueModes } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
|
||||
// Single source of truth for worker liveness: the TTL that GCs stale records and
|
||||
// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it.
|
||||
export const WORKER_TTL_SECONDS = 180;
|
||||
|
||||
const QueueWorkerSchema = new Schema<QueueWorkerDocument>(
|
||||
{
|
||||
_id: { type: String, required: true },
|
||||
mode: { type: String, enum: QueueModes, required: true },
|
||||
lastSeenAt: { type: Date, default: Date.now, expires: 30 },
|
||||
lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS },
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ const startApp = async () => {
|
||||
logger = new Logger({ envSettings });
|
||||
|
||||
// Initialize services (connects DB, creates repositories, injects settingsRepository)
|
||||
const services = await initializeServices({ logger, envSettings, settingsService, queueMode: envSettings.queueMode });
|
||||
const services = await initializeServices({ logger, envSettings, settingsService });
|
||||
|
||||
// If this is a worker instance, we're done. No need for express
|
||||
if (envSettings.queueMode === "worker") {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
|
||||
export interface ICheckEvaluator {
|
||||
evaluate(status: MonitorStatusResponse, check: Check): Promise<MonitorEvaluation>;
|
||||
}
|
||||
|
||||
export class CheckEvaluator implements ICheckEvaluator {
|
||||
constructor(
|
||||
private statusService: IStatusService,
|
||||
private monitorStatusPolicy: IMonitorStatusPolicy
|
||||
) {}
|
||||
evaluate = async (status: MonitorStatusResponse, check: Check) => {
|
||||
// ****************************
|
||||
// Step 3: Evaluate and return result to reactors
|
||||
// ****************************
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
|
||||
|
||||
// Step 5. Get decisions and create an evaluation obj
|
||||
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
|
||||
const evaluation: MonitorEvaluation = {
|
||||
monitor: statusChangeResult.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: statusChangeResult,
|
||||
decision,
|
||||
};
|
||||
return evaluation;
|
||||
};
|
||||
}
|
||||
@@ -1,17 +1,14 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { INetworkService } from "@/service/networkService.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { IBufferService } from "@/service/bufferService.js";
|
||||
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
|
||||
import { ICheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
|
||||
const SERVICE_NAME = "CheckPipeline";
|
||||
|
||||
@@ -21,83 +18,16 @@ export interface ICheckPipeline {
|
||||
|
||||
export class CheckPipeline implements ICheckPipeline {
|
||||
constructor(
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
|
||||
private checkService: ICheckService,
|
||||
private networkService: INetworkService,
|
||||
private bufferService: IBufferService,
|
||||
private monitorStatusPolicy: IMonitorStatusPolicy,
|
||||
private statusService: IStatusService,
|
||||
private logger: ILogger
|
||||
private checkProducer: ICheckProducer,
|
||||
private checkEvaluator: ICheckEvaluator
|
||||
) {}
|
||||
|
||||
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
|
||||
const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
|
||||
const now = new Date();
|
||||
return maintenanceWindows.some((window) => isWindowActive(window, now));
|
||||
}
|
||||
|
||||
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
|
||||
const monitorId = monitor.id;
|
||||
if (!monitorId) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getMonitorJob" });
|
||||
const checkResult = await this.checkProducer.produce(monitor);
|
||||
if (!checkResult) {
|
||||
return null; // skipped due to maintenance / geo guards / other guards
|
||||
}
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
// ****************************
|
||||
|
||||
// Step 1a: Maintenance window gate - skip if in maintenance
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runCheckPipeline",
|
||||
});
|
||||
if (monitor.status !== "maintenance") {
|
||||
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 1b: Acquire status
|
||||
const status = await this.networkService.requestStatus(monitor);
|
||||
if (!status) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
// ****************************
|
||||
|
||||
// Step 2a: Create & record a check, return null if fail
|
||||
const check = this.checkService.buildCheck(status);
|
||||
if (!check) {
|
||||
this.logger.warn({
|
||||
message: `No check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runCHeckPipeline",
|
||||
details: { code: status.code, message: status.message },
|
||||
});
|
||||
return null;
|
||||
}
|
||||
// Step 2b: Add to buffer
|
||||
this.bufferService.addToBuffer(check);
|
||||
|
||||
// ****************************
|
||||
// Step 3: Evaluate and return result to reactors
|
||||
// ****************************
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
|
||||
|
||||
// Step 5. Get decisions and create an evaluation obj
|
||||
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
|
||||
const evaluation: MonitorEvaluation = {
|
||||
monitor: statusChangeResult.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: statusChangeResult,
|
||||
decision,
|
||||
};
|
||||
const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check);
|
||||
return evaluation;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IBufferService } from "@/service/bufferService.js";
|
||||
import { INetworkService } from "@/service/networkService.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
|
||||
export interface ICheckProducer {
|
||||
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "CheckProducer";
|
||||
|
||||
export class CheckProducer implements ICheckProducer {
|
||||
constructor(
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
|
||||
private checkService: ICheckService,
|
||||
private networkService: INetworkService,
|
||||
private bufferService: IBufferService,
|
||||
private logger: ILogger
|
||||
) {}
|
||||
|
||||
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
|
||||
const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
|
||||
const now = new Date();
|
||||
return windows.some((w) => isWindowActive(w, now));
|
||||
}
|
||||
|
||||
produce = async (monitor: Monitor) => {
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
|
||||
}
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
// ****************************
|
||||
|
||||
// Step 1a: Maintenance window gate - skip if in maintenance
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
});
|
||||
if (monitor.status !== "maintenance") {
|
||||
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 1b: Acquire status
|
||||
const status = await this.networkService.requestStatus(monitor);
|
||||
if (!status) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
// ****************************
|
||||
|
||||
// Step 2a: Create & record a check, return null if fail
|
||||
const check = this.checkService.toCheck(status);
|
||||
if (!check) {
|
||||
this.logger.warn({
|
||||
message: `No check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
details: { code: status.code, message: status.message },
|
||||
});
|
||||
return null;
|
||||
}
|
||||
// Step 2b: Add to buffer
|
||||
this.bufferService.addToBuffer(check);
|
||||
return { status, check };
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js";
|
||||
import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { ICheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
const POLL_MS = 250;
|
||||
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
|
||||
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
|
||||
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
|
||||
const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
||||
const CONCURRENCY: Record<JobType, number> = {
|
||||
check: 500,
|
||||
"geo-check": 20,
|
||||
evaluate: 20,
|
||||
"cleanup-orphaned": 1,
|
||||
"cleanup-retention": 1,
|
||||
};
|
||||
|
||||
export class DBQueueWorker implements IWorker {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
private stopped = false;
|
||||
private timers = new Map<JobType | "heartbeat", NodeJS.Timeout>();
|
||||
private inFlight: Record<JobType, number> = {
|
||||
check: 0,
|
||||
"geo-check": 0,
|
||||
evaluate: 0,
|
||||
"cleanup-orphaned": 0,
|
||||
"cleanup-retention": 0,
|
||||
};
|
||||
|
||||
constructor(
|
||||
private logger: ILogger,
|
||||
private jobsRepository: IJobsRepository,
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private checksRepository: IChecksRepository,
|
||||
private checkService: ICheckService, // toStatusResponse + toLastEvaluatedAt
|
||||
private checkProducer: ICheckProducer,
|
||||
private checkEvaluator: ICheckEvaluator,
|
||||
private geoCheckPipeline: ICheckPipeline,
|
||||
private dispatcher: IReactorDispatcher,
|
||||
private helper: IWorkerHelper,
|
||||
private queueWorkersRepository: IQueueWorkersRepository,
|
||||
private queueMode: QueueMode,
|
||||
private queuePrimaryProcesses: boolean,
|
||||
private readonly workerId: string // same id MongoJobsRepository stamps into lockedBy
|
||||
) {}
|
||||
|
||||
get serviceName() {
|
||||
return DBQueueWorker.SERVICE_NAME;
|
||||
}
|
||||
|
||||
static async create(
|
||||
logger: ILogger,
|
||||
jobsRepository: IJobsRepository,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
checksRepository: IChecksRepository,
|
||||
checkService: ICheckService,
|
||||
checkProducer: ICheckProducer,
|
||||
checkEvaluator: ICheckEvaluator,
|
||||
geoCheckPipeline: ICheckPipeline,
|
||||
dispatcher: IReactorDispatcher,
|
||||
helper: IWorkerHelper,
|
||||
queueWorkersRepository: IQueueWorkersRepository,
|
||||
queueMode: QueueMode,
|
||||
queuePrimaryProcesses: boolean,
|
||||
workerId: string
|
||||
): Promise<DBQueueWorker> {
|
||||
const instance = new DBQueueWorker(
|
||||
logger,
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
dispatcher,
|
||||
helper,
|
||||
queueWorkersRepository,
|
||||
queueMode,
|
||||
queuePrimaryProcesses,
|
||||
workerId
|
||||
);
|
||||
await instance.init();
|
||||
return instance;
|
||||
}
|
||||
|
||||
// Helpers, builds job seeds
|
||||
private toCheckJob = (monitor: Monitor, now: number): JobSeed => ({
|
||||
id: jobId("check", monitor.id),
|
||||
type: "check",
|
||||
refId: monitor.id,
|
||||
isActive: monitor.isActive,
|
||||
nextScheduledAt: now + Math.floor(Math.random() * monitor.interval), // Avoid herd
|
||||
intervalMs: monitor.interval,
|
||||
});
|
||||
|
||||
private toGeoCheckJob = (monitor: Monitor, now: number): JobSeed => ({
|
||||
...this.toCheckJob(monitor, now),
|
||||
id: jobId("geo-check", monitor.id),
|
||||
type: "geo-check",
|
||||
intervalMs: monitor.geoCheckInterval!,
|
||||
});
|
||||
|
||||
private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({
|
||||
id: jobId(type, null),
|
||||
type,
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: now,
|
||||
intervalMs: CLEANUP_INTERVAL_MS,
|
||||
});
|
||||
|
||||
private toSummary = (job: Job): WorkerJobSummary => ({
|
||||
monitorId: job.refId ?? job.id,
|
||||
monitorUrl: null, // Jobs don't have URL
|
||||
monitorType: job.type,
|
||||
monitorInterval: job.intervalMs,
|
||||
monitorGeoInterval: null,
|
||||
monitorActive: job.isActive,
|
||||
active: job.isActive,
|
||||
lockedBy: job.lockedBy,
|
||||
lockedUntil: job.lockedUntil,
|
||||
lockedAt: null, // no lockedAt
|
||||
runCount: job.runCount,
|
||||
failCount: job.failCount,
|
||||
failReason: job.lastFailReason,
|
||||
lastRunAt: null, // not tracked
|
||||
lastFinishedAt: job.lastFinishedAt,
|
||||
lastRunTook: null, // not tracked
|
||||
lastFailedAt: null, // not tracked
|
||||
repeat: job.intervalMs,
|
||||
});
|
||||
|
||||
// ********************
|
||||
// Stage 1: This can eventually be offloaded to a separate service
|
||||
// ********************
|
||||
|
||||
private runCheck = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
const produced = await this.checkProducer.produce(monitor);
|
||||
if (produced) {
|
||||
await this.jobsRepository.upsertEvaluate(monitor.id, Date.now());
|
||||
}
|
||||
};
|
||||
|
||||
// ********************
|
||||
// Stage 2: Evaluator loop stays here
|
||||
// ********************
|
||||
|
||||
private runEvaluate = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
|
||||
for (const check of checks) {
|
||||
const status = this.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.checkEvaluator.evaluate(status, check);
|
||||
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
|
||||
}
|
||||
};
|
||||
|
||||
private runGeoCheck = async (job: Job) => {
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId!]);
|
||||
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
|
||||
};
|
||||
|
||||
// Extend the lock while a job runs so a slow-but-alive job is never reclaimed.
|
||||
// Returns false once we no longer hold the lock, so the renewal timer can stop.
|
||||
private renewLock = async (job: Job): Promise<boolean> => {
|
||||
try {
|
||||
const held = await this.jobsRepository.renewLocks([job.id], Date.now());
|
||||
if (held === 0) {
|
||||
this.logger.warn({ message: `Lost lock on job ${job.id}, stopping renewal`, service: SERVICE_NAME, method: "renewLock" });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
// Transient failure, keep renewing; a missed renewal is absorbed by the 1/3 margin
|
||||
this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "renewLock" });
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
private runJob = async (job: Job) => {
|
||||
const renewTimer = setInterval(async () => {
|
||||
const renewLock = await this.renewLock(job);
|
||||
if (!renewLock) clearInterval(renewTimer);
|
||||
}, LOCK_RENEW_MS);
|
||||
try {
|
||||
switch (job.type) {
|
||||
case "check":
|
||||
await this.runCheck(job);
|
||||
break;
|
||||
case "evaluate":
|
||||
await this.runEvaluate(job);
|
||||
break;
|
||||
case "geo-check":
|
||||
await this.runGeoCheck(job);
|
||||
break;
|
||||
case "cleanup-orphaned":
|
||||
await this.helper.getCleanupOrphanedJob()(); // Get job and execute
|
||||
break;
|
||||
case "cleanup-retention":
|
||||
await this.helper.getCleanupRetentionJob()(); // Get job and execute
|
||||
break;
|
||||
}
|
||||
|
||||
if (job.intervalMs === null) {
|
||||
// One-shot job
|
||||
await this.jobsRepository.recordOneShot(job.id, Date.now());
|
||||
} else {
|
||||
await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now());
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock
|
||||
this.logger.warn({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: `runJob:${job.type}`,
|
||||
});
|
||||
} finally {
|
||||
clearInterval(renewTimer);
|
||||
}
|
||||
};
|
||||
|
||||
private startLoop = (type: JobType) => {
|
||||
const tick = async () => {
|
||||
if (this.stopped) return;
|
||||
try {
|
||||
// Claim a batch sized to the free capacity
|
||||
const capacity = CONCURRENCY[type] - this.inFlight[type];
|
||||
const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now());
|
||||
for (const job of jobs) {
|
||||
this.inFlight[type]++; // stay within concurrency limits
|
||||
this.runJob(job).finally(() => {
|
||||
this.inFlight[type]--; // job done, free the slot
|
||||
});
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: `loop:${type}`,
|
||||
});
|
||||
}
|
||||
if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down
|
||||
};
|
||||
tick();
|
||||
};
|
||||
|
||||
// Seed queue
|
||||
private reconcile = async () => {
|
||||
const now = Date.now();
|
||||
const monitors = (await this.monitorsRepository.findAll()) ?? [];
|
||||
for (const monitor of monitors) {
|
||||
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
|
||||
}
|
||||
}
|
||||
// Both cleanup jobs run immediately on every startup, then once a day
|
||||
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now));
|
||||
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now));
|
||||
};
|
||||
|
||||
// Register worker
|
||||
private heartbeat = async () => {
|
||||
try {
|
||||
await this.queueWorkersRepository.upsert(this.workerId, this.queueMode);
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: "heartbeat",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
init = async () => {
|
||||
this.stopped = false;
|
||||
|
||||
await this.heartbeat();
|
||||
this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS));
|
||||
|
||||
if (this.queueMode === "primary") {
|
||||
await this.reconcile();
|
||||
}
|
||||
|
||||
if (this.queuePrimaryProcesses === true || this.queueMode === "worker") {
|
||||
for (const type of Object.keys(this.inFlight) as JobType[]) {
|
||||
this.startLoop(type);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
addJob = async (_monitorId: string, monitor: Monitor) => {
|
||||
const now = Date.now();
|
||||
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
|
||||
};
|
||||
deleteJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.deleteById(monitor.id);
|
||||
};
|
||||
pauseJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.setActiveById(monitor.id, false);
|
||||
};
|
||||
resumeJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.setActiveById(monitor.id, true);
|
||||
};
|
||||
updateJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval);
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now()));
|
||||
} else {
|
||||
await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check");
|
||||
}
|
||||
};
|
||||
shutdown = async () => {
|
||||
this.stopped = true;
|
||||
// Stop all timers
|
||||
for (const timer of this.timers.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
this.timers.clear(); // Clear the map out
|
||||
// Remove workers from registry
|
||||
await this.queueWorkersRepository.deleteById(this.workerId);
|
||||
};
|
||||
getMetrics = async (): Promise<WorkerMetrics> => {
|
||||
const rows = await this.jobsRepository.findAll();
|
||||
const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS);
|
||||
const now = Date.now();
|
||||
return rows.reduce<WorkerMetrics>(
|
||||
(acc, job) => {
|
||||
acc.jobs++;
|
||||
acc.totalRuns += job.runCount;
|
||||
acc.totalFailures += job.failCount;
|
||||
if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++;
|
||||
if (job.failCount > 0) {
|
||||
acc.failingJobs++;
|
||||
acc.jobsWithFailures.push({
|
||||
monitorId: job.refId ?? job.id,
|
||||
monitorUrl: null,
|
||||
monitorType: job.type,
|
||||
failedAt: job.lastFinishedAt,
|
||||
failCount: job.failCount,
|
||||
failReason: job.lastFailReason,
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
jobs: 0,
|
||||
activeJobs: 0,
|
||||
failingJobs: 0,
|
||||
jobsWithFailures: [],
|
||||
totalRuns: 0,
|
||||
totalFailures: 0,
|
||||
workers,
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
getJobs = async (pagination: WorkerJobsPagination) => {
|
||||
const { jobs, count } = await this.jobsRepository.findPage(pagination);
|
||||
return { jobs: jobs.map(this.toSummary), count };
|
||||
};
|
||||
flushQueues = async () => {
|
||||
await this.shutdown();
|
||||
const ok = await this.init();
|
||||
return { success: ok };
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.i
|
||||
import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
|
||||
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
@@ -42,6 +43,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
private checkService: ICheckService;
|
||||
private settingsService: ISettingsService;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private jobsRepository: IJobsRepository;
|
||||
private teamsRepository: ITeamsRepository;
|
||||
private monitorStatsRepository: IMonitorStatsRepository;
|
||||
private checksRepository: IChecksRepository;
|
||||
@@ -56,6 +58,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
checkService: ICheckService,
|
||||
settingsService: ISettingsService,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
jobsRepository: IJobsRepository,
|
||||
teamsRepository: ITeamsRepository,
|
||||
monitorStatsRepository: IMonitorStatsRepository,
|
||||
checksRepository: IChecksRepository,
|
||||
@@ -69,6 +72,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
this.checkService = checkService;
|
||||
this.settingsService = settingsService;
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.jobsRepository = jobsRepository;
|
||||
this.teamsRepository = teamsRepository;
|
||||
this.monitorStatsRepository = monitorStatsRepository;
|
||||
this.checksRepository = checksRepository;
|
||||
@@ -190,6 +194,16 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
});
|
||||
}
|
||||
|
||||
// Remove orphaned jobs
|
||||
const deletedJobsCount = await this.jobsRepository.deleteByMonitorIdsNotIn(allMonitorIds);
|
||||
if (deletedJobsCount > 0) {
|
||||
this.logger.info({
|
||||
message: `Deleted ${deletedJobsCount} orphaned jobs`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getCleanupOrphanedJob",
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.info({
|
||||
message: "Cleanup of orphaned data completed",
|
||||
service: SERVICE_NAME,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
|
||||
export type MonitorEvaluation = {
|
||||
monitor: Monitor;
|
||||
|
||||
@@ -1,427 +0,0 @@
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { MongoStore, Scheduler, IJob, AddJobInput } from "less-simple-scheduler";
|
||||
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { type WorkerMetrics, type WorkerJobsPagination, type WorkerJobsPage, IWorker } from "@/worker/worker.interface.js";
|
||||
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
|
||||
// How long after its last heartbeat a worker is still considered alive. Must be longer than the scheduler's heartbeat
|
||||
const WORKER_TTL_MS = 30_000;
|
||||
|
||||
// For discriminating job types
|
||||
type MonitorJob = Omit<IJob, "data"> & { data: Monitor };
|
||||
const MONITOR_TEMPLATES = new Set<string>(["monitor-job", "geo-check-job"]);
|
||||
const isMonitorJob = (job: IJob): job is MonitorJob => MONITOR_TEMPLATES.has(job.template);
|
||||
|
||||
export class LessSimpleWorker implements IWorker {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private logger: ILogger;
|
||||
private helper: IWorkerHelper;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private workersRepository: IQueueWorkersRepository;
|
||||
private readonly scheduler: Scheduler;
|
||||
private queueMode: QueueMode;
|
||||
private heartbeatListener: ((workerId: string) => void) | null = null;
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
helper: IWorkerHelper,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
workersRepository: IQueueWorkersRepository,
|
||||
scheduler: Scheduler,
|
||||
queueMode: QueueMode
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.helper = helper;
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.workersRepository = workersRepository;
|
||||
this.scheduler = scheduler;
|
||||
this.queueMode = queueMode;
|
||||
this.registerListeners();
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return LessSimpleWorker.SERVICE_NAME;
|
||||
}
|
||||
|
||||
private registerListeners = () => {
|
||||
this.scheduler.on("scheduler:start", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler started",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:stop", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler stopped",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:error", (workerId, error) => {
|
||||
this.logger.error({
|
||||
message: `${workerId} error: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:locked", (workerId, job) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} locked`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:abort", (workerId, job, reason) => {
|
||||
this.logger.warn({
|
||||
message: `${workerId} ${job.id} aborted: ${reason}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:attempt", (workerId, job, attempt) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} attempt ${attempt}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:complete", (workerId, job) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} completed successfully`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:exhausted", (workerId, job, error) => {
|
||||
this.logger.error({
|
||||
message: `${workerId} ${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:fail", (workerId, job, error, attempt) => {
|
||||
this.logger.warn({
|
||||
message: `${workerId} ${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:start", (workerId, job) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} started`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
static async create(
|
||||
logger: ILogger,
|
||||
helper: IWorkerHelper,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
workersRepository: IQueueWorkersRepository,
|
||||
envSettings: EnvConfig,
|
||||
queueMode: QueueMode
|
||||
) {
|
||||
const store = new MongoStore({
|
||||
url: envSettings.dbConnectionString ?? "mongodb://localhost:27017/uptime_db",
|
||||
});
|
||||
const scheduler = new Scheduler(store, {
|
||||
concurrency: envSettings.queueMode === "primary" ? 0 : 50,
|
||||
processEvery: 100,
|
||||
});
|
||||
const instance = new LessSimpleWorker(logger, helper, monitorsRepository, workersRepository, scheduler, queueMode);
|
||||
await instance.init(queueMode);
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
private startHeartbeat = (queueMode: QueueMode) => {
|
||||
const beat = (
|
||||
workerId: string // This is the heartbeat function that registers workers
|
||||
) =>
|
||||
this.workersRepository.upsert(workerId, queueMode).catch((error: unknown) => {
|
||||
this.logger.warn({
|
||||
message: `Worker heartbeat failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "startHeartbeat",
|
||||
});
|
||||
});
|
||||
beat(this.scheduler.workerId); // Beat immediately to register
|
||||
if (this.heartbeatListener) {
|
||||
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); // remove old listeners
|
||||
}
|
||||
this.heartbeatListener = beat; /// Keep a reference to the listener
|
||||
this.scheduler.on("scheduler:heartbeat", beat); // beat on every heartbeat
|
||||
};
|
||||
|
||||
init = async (queueMode: QueueMode = "primary") => {
|
||||
try {
|
||||
await this.scheduler.start();
|
||||
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
|
||||
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
|
||||
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
|
||||
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
|
||||
|
||||
// Register in the worker registry
|
||||
this.startHeartbeat(queueMode);
|
||||
|
||||
// If this is a worker, return early - primary will populate queue
|
||||
if (queueMode === "worker") {
|
||||
this.logger.info({
|
||||
message: `Worker queue: ${this.scheduler.workerId} initialized`,
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
const monitors = await this.monitorsRepository.findAll();
|
||||
if (!monitors) {
|
||||
return true;
|
||||
}
|
||||
const jobInputs = monitors.flatMap((monitor) => this.buildJobInputs(monitor));
|
||||
if (jobInputs.length > 0) {
|
||||
await this.scheduler.addJobs(jobInputs);
|
||||
}
|
||||
|
||||
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, upsert: true });
|
||||
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000, upsert: true });
|
||||
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: `Failed to initialize LessSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
});
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Builds the scheduler inputs for a monitor: always a monitor-job, plus a
|
||||
// geo-check job when the monitor supports and enables geo checks.
|
||||
private buildJobInputs = (monitor: Monitor): AddJobInput[] => {
|
||||
const inputs: AddJobInput[] = [
|
||||
{
|
||||
id: monitor.id,
|
||||
template: "monitor-job",
|
||||
repeat: monitor.interval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
upsert: true,
|
||||
jitter: true,
|
||||
},
|
||||
];
|
||||
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
inputs.push({
|
||||
id: `${monitor.id}-geo`,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
upsert: true,
|
||||
jitter: true,
|
||||
});
|
||||
}
|
||||
|
||||
return inputs;
|
||||
};
|
||||
|
||||
addJob = async (_monitorId: string, monitor: Monitor) => {
|
||||
await this.scheduler.addJobs(this.buildJobInputs(monitor));
|
||||
};
|
||||
|
||||
deleteJob = async (monitor: Monitor) => {
|
||||
this.scheduler.removeJob(monitor.id);
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
|
||||
};
|
||||
|
||||
pauseJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.pauseJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to pause monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to pause geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
|
||||
this.logger.debug({
|
||||
message: `Paused monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
};
|
||||
|
||||
resumeJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.resumeJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to resume monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to resume geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
this.logger.debug({
|
||||
message: `Resumed monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
};
|
||||
|
||||
private syncGeoJob = async (monitor: Monitor) => {
|
||||
const geoJobId = `${monitor.id}-geo`;
|
||||
const existingGeoJob = await this.scheduler.getJob(geoJobId);
|
||||
|
||||
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
|
||||
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
|
||||
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
|
||||
return;
|
||||
}
|
||||
|
||||
// If the job exists, update it
|
||||
if (existingGeoJob) {
|
||||
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, create it
|
||||
this.scheduler.addJob({
|
||||
id: geoJobId,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
upsert: true,
|
||||
});
|
||||
};
|
||||
|
||||
updateJob = async (monitor: Monitor) => {
|
||||
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
|
||||
await this.syncGeoJob(monitor);
|
||||
};
|
||||
|
||||
shutdown = async () => {
|
||||
// Stop beating before we remove this worker, so a heartbeat firing between
|
||||
// the remove and the scheduler stopping can't re-insert it.
|
||||
if (this.heartbeatListener) {
|
||||
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener);
|
||||
this.heartbeatListener = null;
|
||||
}
|
||||
await this.workersRepository.deleteById(this.scheduler.workerId).catch((error: unknown) => {
|
||||
this.logger.warn({
|
||||
message: `Failed to deregister worker on shutdown: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "shutdown",
|
||||
});
|
||||
});
|
||||
this.scheduler.stop();
|
||||
};
|
||||
|
||||
getMetrics = async (): Promise<WorkerMetrics> => {
|
||||
const stats = await this.scheduler.getStats();
|
||||
|
||||
const jobsWithFailures = stats.jobsWithFailures.map((failure) => {
|
||||
// `data` is untyped at the store level: for monitor jobs it's the Monitor,
|
||||
// for system jobs (e.g. cleanup) it has no url/type => null
|
||||
const monitor = failure.data as Monitor | undefined;
|
||||
return {
|
||||
monitorId: failure.id,
|
||||
monitorUrl: monitor?.url ?? null,
|
||||
monitorType: monitor?.type ?? null,
|
||||
failedAt: failure.failedAt,
|
||||
failCount: failure.failCount,
|
||||
failReason: failure.failReason,
|
||||
};
|
||||
});
|
||||
|
||||
const workers = await this.workersRepository.findRecent(WORKER_TTL_MS);
|
||||
return {
|
||||
jobs: stats.jobs,
|
||||
activeJobs: stats.activeJobs,
|
||||
failingJobs: stats.failingJobs,
|
||||
totalRuns: stats.totalRuns,
|
||||
totalFailures: stats.totalFailures,
|
||||
jobsWithFailures,
|
||||
workers,
|
||||
};
|
||||
};
|
||||
|
||||
getJobs = async ({ page = 0, rowsPerPage = 0 }: WorkerJobsPagination): Promise<WorkerJobsPage> => {
|
||||
const listOptions = rowsPerPage > 0 ? { skip: Math.max(page, 0) * rowsPerPage, limit: rowsPerPage } : undefined;
|
||||
const [jobs, count] = await Promise.all([this.scheduler.getJobs(listOptions), this.scheduler.countJobs()]);
|
||||
const mapped = jobs.map((job) => {
|
||||
const monitor = isMonitorJob(job) ? job.data : undefined;
|
||||
|
||||
return {
|
||||
monitorId: job.id,
|
||||
monitorUrl: monitor?.url || null,
|
||||
monitorType: monitor?.type || null,
|
||||
monitorInterval: monitor?.interval || null,
|
||||
monitorGeoInterval: monitor?.geoCheckInterval || null,
|
||||
monitorActive: monitor?.isActive ?? null,
|
||||
active: job.active,
|
||||
repeat: job.repeat ?? null,
|
||||
lockedBy: job.lockedBy ?? null,
|
||||
lockedUntil: job.lockedUntil ?? null,
|
||||
lockedAt: job.lockedAt ?? null,
|
||||
runCount: job.runCount ?? 0,
|
||||
failCount: job.failCount ?? 0,
|
||||
failReason: job.lastError ?? null,
|
||||
lastRunAt: job.lastStartedAt ?? null,
|
||||
lastFinishedAt: job.lastFinishedAt ?? null,
|
||||
lastRunTook: job.lockedAt || !job.lastFinishedAt || !job.lastStartedAt ? null : job.lastFinishedAt - job.lastStartedAt,
|
||||
lastFailedAt: job.lastFailedAt ?? null,
|
||||
};
|
||||
});
|
||||
|
||||
return { jobs: mapped, count };
|
||||
};
|
||||
|
||||
flushQueues = async () => {
|
||||
if (this.queueMode !== "primary") {
|
||||
// Don't allow workers to flush, though this isn't currently reachable since workers have no API endpoints
|
||||
this.logger.warn({ message: "Ignoring flush on non-primary worker", service: SERVICE_NAME, method: "flushQueues" });
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
const flushRes = await this.scheduler.flushJobs(); // Less Simple Queue must flush before stopping as it needs DB
|
||||
const stopRes = await this.scheduler.stop(); // Destructive, other workers will abort their jobs
|
||||
const initRes = await this.init();
|
||||
return {
|
||||
success: Boolean(stopRes && flushRes && initRes),
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,384 +0,0 @@
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import Scheduler from "super-simple-scheduler";
|
||||
import { Monitor, MonitorType, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { hostname } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { IWorker, WorkerJobSummary } from "@/worker/worker.interface.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
|
||||
export class SuperSimpleQueue implements IWorker {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private logger: ILogger;
|
||||
private helper: IWorkerHelper;
|
||||
private workerId: string;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private readonly scheduler: Scheduler;
|
||||
|
||||
constructor(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
|
||||
this.logger = logger;
|
||||
this.helper = helper;
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.scheduler = scheduler;
|
||||
this.registerListeners();
|
||||
this.workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return SuperSimpleQueue.SERVICE_NAME;
|
||||
}
|
||||
|
||||
private registerListeners = () => {
|
||||
this.scheduler.on("scheduler:start", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler started",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:stop", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler stopped",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:error", (error) => {
|
||||
this.logger.error({
|
||||
message: `Scheduler error: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:abort", (job, reason) => {
|
||||
this.logger.warn({
|
||||
message: `${job.id} aborted: ${reason}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:attempt", (job, attempt) => {
|
||||
this.logger.debug({
|
||||
message: `${job.id} attempt ${attempt}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:complete", (job) => {
|
||||
this.logger.debug({
|
||||
message: `${job.id} completed successfully`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:exhausted", (job, error) => {
|
||||
this.logger.error({
|
||||
message: `${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:fail", (job, error, attempt) => {
|
||||
this.logger.warn({
|
||||
message: `${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:start", (job) => {
|
||||
this.logger.debug({
|
||||
message: `${job.id} started`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
static async create(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository) {
|
||||
const scheduler = new Scheduler({
|
||||
concurrency: 50,
|
||||
// storeType: "mongo",
|
||||
// storeType: "redis",
|
||||
// dbUri: envSettings.dbConnectionString,
|
||||
});
|
||||
const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
|
||||
await instance.init();
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
init = async () => {
|
||||
try {
|
||||
this.scheduler.start();
|
||||
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
|
||||
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
|
||||
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
|
||||
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
|
||||
const monitors = await this.monitorsRepository.findAll();
|
||||
if (!monitors) {
|
||||
return true;
|
||||
}
|
||||
for (const monitor of monitors) {
|
||||
const randomOffset = Math.floor(Math.random() * 100);
|
||||
setTimeout(() => {
|
||||
this.addJob(monitor.id, monitor);
|
||||
}, randomOffset);
|
||||
}
|
||||
|
||||
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
|
||||
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 });
|
||||
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: `Failed to initialize SuperSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
});
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
addJob = async (monitorId: string, monitor: Monitor) => {
|
||||
this.scheduler.addJob({
|
||||
id: monitorId,
|
||||
template: "monitor-job",
|
||||
repeat: monitor.interval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
});
|
||||
|
||||
// Return early if we don't need geo checks
|
||||
if (!supportsGeoCheck(monitor.type)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Add geo check job if enabled for HTTP monitors
|
||||
if (monitor.geoCheckEnabled) {
|
||||
this.scheduler.addJob({
|
||||
id: `${monitorId}-geo`,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
deleteJob = async (monitor: Monitor) => {
|
||||
this.scheduler.removeJob(monitor.id);
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
|
||||
};
|
||||
|
||||
pauseJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.pauseJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to pause monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to pause geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
|
||||
this.logger.debug({
|
||||
message: `Paused monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
};
|
||||
|
||||
resumeJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.resumeJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to resume monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to resume geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
this.logger.debug({
|
||||
message: `Resumed monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
};
|
||||
|
||||
private syncGeoJob = async (monitor: Monitor) => {
|
||||
const geoJobId = `${monitor.id}-geo`;
|
||||
const existingGeoJob = await this.scheduler.getJob(geoJobId);
|
||||
|
||||
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
|
||||
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
|
||||
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
|
||||
return;
|
||||
}
|
||||
|
||||
// If the job exists, update it
|
||||
if (existingGeoJob) {
|
||||
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, create it
|
||||
this.scheduler.addJob({
|
||||
id: geoJobId,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
});
|
||||
};
|
||||
|
||||
updateJob = async (monitor: Monitor) => {
|
||||
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
|
||||
await this.syncGeoJob(monitor);
|
||||
};
|
||||
|
||||
shutdown = async () => {
|
||||
this.scheduler.stop();
|
||||
};
|
||||
|
||||
getMetrics = async () => {
|
||||
const jobs = await this.scheduler.getJobs();
|
||||
const metrics: {
|
||||
jobs: number;
|
||||
activeJobs: number;
|
||||
failingJobs: number;
|
||||
jobsWithFailures: Array<{
|
||||
monitorId: string | number;
|
||||
monitorUrl: string;
|
||||
monitorType: MonitorType;
|
||||
failedAt: number | null;
|
||||
failCount: number;
|
||||
failReason: string | null;
|
||||
}>;
|
||||
totalRuns: number;
|
||||
totalFailures: number;
|
||||
workers: Array<{
|
||||
workerId: string;
|
||||
mode: QueueMode;
|
||||
lastSeenAt: number;
|
||||
}>;
|
||||
} = jobs.reduce(
|
||||
(acc, job) => {
|
||||
const runCount = job.runCount ?? 0;
|
||||
const failCount = job.failCount ?? 0;
|
||||
const lastFailedAt = job.lastFailedAt ?? 0;
|
||||
const lastRunAt = job.lastRunAt ?? 0;
|
||||
acc.totalRuns += runCount;
|
||||
acc.totalFailures += failCount;
|
||||
acc.jobs++;
|
||||
if (failCount > 0 && lastFailedAt >= lastRunAt) {
|
||||
acc.failingJobs++;
|
||||
}
|
||||
|
||||
if (job.lockedAt) {
|
||||
acc.activeJobs++;
|
||||
}
|
||||
|
||||
if (failCount > 0) {
|
||||
acc.jobsWithFailures.push({
|
||||
monitorId: job.id,
|
||||
monitorUrl: job.data?.url || null,
|
||||
monitorType: job.data?.type || null,
|
||||
failedAt: job.lastFailedAt ?? null,
|
||||
failCount,
|
||||
failReason: job.lastFailReason ?? null,
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
jobs: 0,
|
||||
activeJobs: 0,
|
||||
failingJobs: 0,
|
||||
jobsWithFailures: [] as Array<{
|
||||
monitorId: string | number;
|
||||
monitorUrl: string;
|
||||
monitorType: MonitorType;
|
||||
failedAt: number | null;
|
||||
failCount: number;
|
||||
failReason: string | null;
|
||||
}>,
|
||||
totalRuns: 0,
|
||||
totalFailures: 0,
|
||||
workers: [
|
||||
{
|
||||
workerId: this.workerId,
|
||||
mode: "primary",
|
||||
lastSeenAt: Date.now(),
|
||||
},
|
||||
],
|
||||
}
|
||||
);
|
||||
return metrics;
|
||||
};
|
||||
|
||||
getJobs = async ({ page = 0, rowsPerPage = 0 }: { page?: number; rowsPerPage?: number }): Promise<{ jobs: WorkerJobSummary[]; count: number }> => {
|
||||
const jobs = await this.scheduler.getJobs();
|
||||
const mapped = jobs.map((job) => {
|
||||
return {
|
||||
monitorId: job.id,
|
||||
monitorUrl: job.data?.url || null,
|
||||
monitorType: job.data?.type || null,
|
||||
monitorInterval: job.data?.interval || null,
|
||||
monitorGeoInterval: job.data?.geoCheckInterval || null,
|
||||
monitorActive: job.data?.isActive ?? null,
|
||||
active: job.active,
|
||||
repeat: job.repeat ?? null,
|
||||
lockedBy: job.lockedAt ? this.workerId : null,
|
||||
lockedUntil: null,
|
||||
lockedAt: job.lockedAt ?? null,
|
||||
runCount: job.runCount ?? 0,
|
||||
failCount: job.failCount ?? 0,
|
||||
failReason: job.lastFailReason ?? null,
|
||||
lastRunAt: job.lastRunAt ?? null,
|
||||
lastFinishedAt: job.lastFinishedAt ?? null,
|
||||
lastRunTook: job.lockedAt ? null : (job.lastFinishedAt ?? 0) - (job.lastRunAt ?? 0),
|
||||
lastFailedAt: job.lastFailedAt ?? null,
|
||||
};
|
||||
});
|
||||
|
||||
const start = Math.max(page, 0) * rowsPerPage;
|
||||
return {
|
||||
jobs: rowsPerPage > 0 ? mapped.slice(start, start + rowsPerPage) : mapped,
|
||||
count: mapped.length,
|
||||
};
|
||||
};
|
||||
|
||||
flushQueues = async () => {
|
||||
const stopRes = await this.scheduler.stop();
|
||||
const flushRes = await this.scheduler.flushJobs();
|
||||
const initRes = await this.init();
|
||||
return {
|
||||
success: Boolean(stopRes && flushRes && initRes),
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -2,6 +2,8 @@ import { jest } from "@jest/globals";
|
||||
import { WorkerHelper } from "../../src/worker/worker.helper.ts";
|
||||
import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts";
|
||||
import { CheckProducer } from "../../src/worker/worker.check-producer.ts";
|
||||
import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts";
|
||||
import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts";
|
||||
import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts";
|
||||
import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts";
|
||||
@@ -131,7 +133,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
}),
|
||||
};
|
||||
const checkService = {
|
||||
buildCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => {
|
||||
toCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => {
|
||||
return makeCheck(response.status, response.code);
|
||||
}),
|
||||
};
|
||||
@@ -149,16 +151,16 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
|
||||
const geoChecksService = { buildGeoCheck: jest.fn() };
|
||||
|
||||
const checkPipeline = new CheckPipeline(
|
||||
const checkProducer = new CheckProducer(
|
||||
monitorsRepo as any,
|
||||
maintenanceWindowsRepo as any,
|
||||
checkService as any,
|
||||
networkService as any,
|
||||
bufferStub as any,
|
||||
new MonitorStatusPolicy(),
|
||||
statusService as any,
|
||||
logger
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
|
||||
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger);
|
||||
|
||||
const notificationReactor = new NotificationReactor(notificationsService as any);
|
||||
@@ -170,6 +172,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
checkService as any,
|
||||
{ getDBSettings: jest.fn() } as any,
|
||||
monitorsRepo as any,
|
||||
{ deleteByMonitorIdsNotIn: jest.fn() } as any,
|
||||
{ findAllTeamIds: jest.fn() } as any,
|
||||
createStubMonitorStatsRepo() as any,
|
||||
createStubChecksRepo() as any,
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import { MonitorModel } from "../../src/domain/monitors/monitor.model.ts";
|
||||
import { backfillMonitorLastEvaluatedAt } from "../../src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// Legacy monitors are inserted through the raw driver, NOT MonitorModel.create — the
|
||||
// schema default (lastEvaluatedAt: 0) would otherwise stamp the field and the migration's
|
||||
// { $exists: false } filter would skip it, defeating the test.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await MonitorModel.collection.deleteMany({});
|
||||
});
|
||||
|
||||
describe("0008_backfillMonitorLastEvaluatedAt", () => {
|
||||
it("stamps the upgrade moment (not 0) on monitors missing lastEvaluatedAt", async () => {
|
||||
await MonitorModel.collection.insertOne({ name: "legacy" }); // pre-upgrade doc, no lastEvaluatedAt
|
||||
|
||||
const before = Date.now();
|
||||
await backfillMonitorLastEvaluatedAt();
|
||||
|
||||
const doc = await MonitorModel.collection.findOne({ name: "legacy" });
|
||||
// Must be a recent timestamp — backfilling 0 would replay the whole check history on the first evaluate.
|
||||
expect(doc?.lastEvaluatedAt).not.toBe(0);
|
||||
expect(doc?.lastEvaluatedAt).toBeGreaterThanOrEqual(before);
|
||||
expect(doc?.lastEvaluatedAt).toBeLessThanOrEqual(Date.now());
|
||||
});
|
||||
|
||||
it("leaves monitors that already have lastEvaluatedAt untouched (idempotent)", async () => {
|
||||
await MonitorModel.collection.insertOne({ name: "current", lastEvaluatedAt: 12345 });
|
||||
|
||||
await backfillMonitorLastEvaluatedAt();
|
||||
|
||||
const doc = await MonitorModel.collection.findOne({ name: "current" });
|
||||
expect(doc?.lastEvaluatedAt).toBe(12345);
|
||||
});
|
||||
|
||||
it("drops the legacy jobs collection left behind by the old scheduler", async () => {
|
||||
const db = mongoose.connection.db!;
|
||||
await db.collection("jobs").insertOne({ id: "legacy-job", template: "monitor-job" }); // old-scheduler row
|
||||
|
||||
await backfillMonitorLastEvaluatedAt();
|
||||
|
||||
const exists = await db.listCollections({ name: "jobs" }).toArray();
|
||||
expect(exists).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not throw when there is no jobs collection to drop", async () => {
|
||||
await expect(backfillMonitorLastEvaluatedAt()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,461 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
||||
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
|
||||
import { BACKOFF_MS, LOCK_MS, type Job, type JobType } from "../../src/domain/jobs/job.type.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
|
||||
// single-document atomicity on findOneAndUpdate, so it can only be exercised against a
|
||||
// live engine. We boot a throwaway in-process mongod for this suite.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await JobModel.init(); // build the indexes the claim scan relies on
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await JobModel.deleteMany({});
|
||||
});
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const NOW = 1_000_000;
|
||||
|
||||
// Insert a job row directly, full control over every field. Defaults describe a due,
|
||||
// unlocked, active check job.
|
||||
const seedJob = (overrides: Partial<JobDocument> = {}): Promise<JobDocument> =>
|
||||
JobModel.create({
|
||||
_id: overrides._id ?? `check:${overrides.refId ?? "mon-1"}`,
|
||||
type: "check" as JobType,
|
||||
refId: "mon-1",
|
||||
isActive: true,
|
||||
nextScheduledAt: NOW,
|
||||
intervalMs: 60_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const readRow = (id: string) => JobModel.findById(id).lean<JobDocument>();
|
||||
|
||||
// record*/renewLocks are fenced on the repo's private workerId, so a pre-seeded "held"
|
||||
// lock must be stamped with the same id the repo will check against.
|
||||
const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string }).workerId;
|
||||
|
||||
const WORKER_ID = "worker-under-test";
|
||||
|
||||
describe("MongoJobsRepository", () => {
|
||||
let repo: MongoJobsRepository;
|
||||
|
||||
beforeEach(() => {
|
||||
repo = new MongoJobsRepository(WORKER_ID);
|
||||
});
|
||||
|
||||
// ── claimDue ───────────────────────────────────────────────────────────────
|
||||
describe("claimDue", () => {
|
||||
it("claims a due, unlocked job and leases it", async () => {
|
||||
await seedJob();
|
||||
|
||||
const claimed = await repo.claimDue("check", NOW);
|
||||
|
||||
expect(claimed?.id).toBe("check:mon-1");
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).not.toBeNull();
|
||||
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
});
|
||||
|
||||
it("returns null when nothing is due yet", async () => {
|
||||
await seedJob({ nextScheduledAt: NOW + 1 });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("skips inactive (paused) jobs", async () => {
|
||||
await seedJob({ isActive: false });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("only claims the requested type", async () => {
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not claim a job held by a live lease", async () => {
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("reclaims a job whose lease has expired (crash recovery)", async () => {
|
||||
await seedJob({ lockedBy: "dead-worker", lockedUntil: NOW - 1 });
|
||||
|
||||
const claimed = await repo.claimDue("check", NOW);
|
||||
|
||||
expect(claimed?.id).toBe("check:mon-1");
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).not.toBe("dead-worker");
|
||||
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
});
|
||||
|
||||
it("returns the oldest-due job first", async () => {
|
||||
await seedJob({ _id: "check:newer", refId: "newer", nextScheduledAt: NOW });
|
||||
await seedJob({ _id: "check:older", refId: "older", nextScheduledAt: NOW - 5_000 });
|
||||
|
||||
const claimed = await repo.claimDue("check", NOW);
|
||||
expect(claimed?.id).toBe("check:older");
|
||||
});
|
||||
|
||||
it("lets exactly one of many concurrent claimers win a single due job", async () => {
|
||||
await seedJob();
|
||||
|
||||
// Distinct repo instances = distinct workerIds = realistic competing workers.
|
||||
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
||||
const results = await Promise.all(workers.map((w) => w.claimDue("check", NOW)));
|
||||
|
||||
const winners = results.filter((r): r is Job => r !== null);
|
||||
expect(winners).toHaveLength(1);
|
||||
expect(winners[0].id).toBe("check:mon-1");
|
||||
});
|
||||
|
||||
it("hands two concurrent claimers two distinct due jobs (no double-claim)", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a" });
|
||||
await seedJob({ _id: "check:b", refId: "b" });
|
||||
|
||||
const [w1, w2] = [new MongoJobsRepository("worker-1"), new MongoJobsRepository("worker-2")];
|
||||
const [r1, r2] = await Promise.all([w1.claimDue("check", NOW), w2.claimDue("check", NOW)]);
|
||||
|
||||
expect(r1).not.toBeNull();
|
||||
expect(r2).not.toBeNull();
|
||||
expect(r1?.id).not.toBe(r2?.id);
|
||||
expect([r1?.id, r2?.id].sort()).toEqual(["check:a", "check:b"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordSuccess (fixed-rate reschedule) ────────────────────────────────────
|
||||
describe("recordSuccess", () => {
|
||||
it("reschedules fixed-rate from the scheduled tick and releases the lease", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, runCount: 4 });
|
||||
|
||||
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_500);
|
||||
|
||||
expect(ok).toBe(true);
|
||||
const row = await readRow("check:mon-1");
|
||||
// next tick measured from the scheduled time (NOW), not completion time
|
||||
expect(row?.nextScheduledAt).toBe(NOW + 60_000);
|
||||
expect(row?.lockedBy).toBeNull();
|
||||
expect(row?.lockedUntil).toBeNull();
|
||||
expect(row?.lastFinishedAt).toBe(NOW + 1_500);
|
||||
expect(row?.runCount).toBe(5);
|
||||
});
|
||||
|
||||
it("skips missed ticks when the worker has fallen behind, with jitter to avoid a herd", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
// scheduled NOW, interval 60s, but completion is 200s late → NOW+60s is already past
|
||||
const now = NOW + 200_000;
|
||||
await repo.recordSuccess("check:mon-1", NOW, 60_000, now);
|
||||
|
||||
const row = await readRow("check:mon-1");
|
||||
// no catch-up burst: rebased to one interval from now, plus up to one interval of jitter
|
||||
// so behind-jobs don't re-synchronize. Never earlier than now + interval (no over-checking).
|
||||
expect(row?.nextScheduledAt).toBeGreaterThanOrEqual(now + 60_000);
|
||||
expect(row?.nextScheduledAt).toBeLessThan(now + 120_000);
|
||||
});
|
||||
|
||||
it("returns false for an unknown id", async () => {
|
||||
expect(await repo.recordSuccess("missing", NOW, 60_000, NOW)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not release a lock owned by another worker (fenced no-op)", async () => {
|
||||
// A worker whose lease expired and was reclaimed must not clobber the new owner's lock.
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7 });
|
||||
|
||||
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000);
|
||||
|
||||
expect(ok).toBe(false);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).toBe("other-worker"); // lock untouched
|
||||
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
expect(row?.runCount).toBe(7); // not double-counted
|
||||
});
|
||||
});
|
||||
|
||||
// ── renewLocks (lease renewal) ───────────────────────────────────────────────
|
||||
describe("renewLocks", () => {
|
||||
it("extends the lease on a lock this worker holds", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000);
|
||||
|
||||
expect(renewed).toBe(1);
|
||||
expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS);
|
||||
});
|
||||
|
||||
it("does not renew a lock held by another worker (fenced)", async () => {
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000);
|
||||
|
||||
expect(renewed).toBe(0);
|
||||
expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + LOCK_MS); // unchanged
|
||||
});
|
||||
|
||||
it("renews only the locks this worker holds out of a mixed batch", async () => {
|
||||
await seedJob({ _id: "check:mine", refId: "mine", lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
await seedJob({ _id: "check:theirs", refId: "theirs", lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
const renewed = await repo.renewLocks(["check:mine", "check:theirs"], NOW + 1_000);
|
||||
|
||||
expect(renewed).toBe(1);
|
||||
expect((await readRow("check:mine"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS);
|
||||
expect((await readRow("check:theirs"))?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
});
|
||||
|
||||
it("returns 0 for an empty id list without touching the db", async () => {
|
||||
expect(await repo.renewLocks([], NOW)).toBe(0);
|
||||
});
|
||||
|
||||
// Regression: a renewed lock keeps a slow-but-alive job from being reclaimed once its
|
||||
// original lease would have expired — preventing the duplicate-execution path.
|
||||
it("keeps a job from being reclaimed after the original lease window passes", async () => {
|
||||
await seedJob();
|
||||
await repo.claimDue("check", NOW); // this worker now holds the lease until NOW + LOCK_MS
|
||||
|
||||
// just before expiry, the still-running job renews its own lock
|
||||
const renewAt = NOW + LOCK_MS - 1_000;
|
||||
expect(await repo.renewLocks(["check:mon-1"], renewAt)).toBe(1);
|
||||
|
||||
// a second worker scans PAST the original lease end — without renewal it would reclaim
|
||||
const other = new MongoJobsRepository("competing-worker");
|
||||
expect(await other.claimDue("check", NOW + LOCK_MS + 1)).toBeNull();
|
||||
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).toBe(ownedBy(repo)); // still held by the original worker
|
||||
expect(row?.lockedUntil).toBe(renewAt + LOCK_MS);
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordFailure (backoff) ──────────────────────────────────────────────────
|
||||
describe("recordFailure", () => {
|
||||
it("backs off, records the reason, bumps failCount, releases the lease", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, failCount: 2 });
|
||||
|
||||
const ok = await repo.recordFailure("check:mon-1", new Error("connection refused"), NOW);
|
||||
|
||||
expect(ok).toBe(true);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW + BACKOFF_MS);
|
||||
expect(row?.lastFailReason).toBe("connection refused");
|
||||
expect(row?.failCount).toBe(3);
|
||||
expect(row?.lockedBy).toBeNull();
|
||||
expect(row?.lockedUntil).toBeNull();
|
||||
});
|
||||
|
||||
it("stringifies a non-Error reason", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
await repo.recordFailure("check:mon-1", "weird non-error", NOW);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lastFailReason).toBe("weird non-error");
|
||||
});
|
||||
|
||||
it("returns false for an unknown id", async () => {
|
||||
expect(await repo.recordFailure("missing", new Error("x"), NOW)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── upsertEvaluate (handoff + $min coalescing) ───────────────────────────────
|
||||
describe("upsertEvaluate", () => {
|
||||
it("creates a single evaluate row keyed by monitor", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.type).toBe("evaluate");
|
||||
expect(row?.refId).toBe("mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW);
|
||||
expect(row?.intervalMs).toBeNull();
|
||||
expect(row?.isActive).toBe(true);
|
||||
});
|
||||
|
||||
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
||||
|
||||
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].nextScheduledAt).toBe(NOW); // earliest pending wins
|
||||
});
|
||||
|
||||
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW);
|
||||
});
|
||||
|
||||
it("concurrent upserts for one monitor produce exactly one row", async () => {
|
||||
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
||||
await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW)));
|
||||
|
||||
const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" });
|
||||
expect(count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── monitor lifecycle CRUD ───────────────────────────────────────────────────
|
||||
describe("CRUD", () => {
|
||||
it("upsertJob inserts, then refreshes config without clobbering scheduling state", async () => {
|
||||
const job: Job = {
|
||||
id: "check:mon-1",
|
||||
type: "check",
|
||||
refId: "mon-1",
|
||||
isActive: true,
|
||||
nextScheduledAt: NOW,
|
||||
intervalMs: 60_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
};
|
||||
await repo.upsertJob(job);
|
||||
|
||||
// simulate a running job that has advanced + holds a lease
|
||||
await JobModel.updateOne({ _id: "check:mon-1" }, { $set: { nextScheduledAt: NOW + 5_000, lockedBy: "live", runCount: 7 } });
|
||||
|
||||
// a reconcile sweep re-upserts the same id with new config
|
||||
await repo.upsertJob({ ...job, intervalMs: 30_000, isActive: false });
|
||||
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.intervalMs).toBe(30_000); // config refreshed ($set)
|
||||
expect(row?.isActive).toBe(false);
|
||||
expect(row?.nextScheduledAt).toBe(NOW + 5_000); // scheduling untouched ($setOnInsert)
|
||||
expect(row?.lockedBy).toBe("live");
|
||||
expect(row?.runCount).toBe(7);
|
||||
});
|
||||
|
||||
it("upsertCleanupJob resets nextScheduledAt on an existing row so it is due again on restart", async () => {
|
||||
const job: Job = {
|
||||
id: "cleanup-orphaned",
|
||||
type: "cleanup-orphaned",
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: NOW,
|
||||
intervalMs: 86_400_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
};
|
||||
await repo.upsertCleanupJob(job);
|
||||
|
||||
// row has already advanced a day into the future after a previous run
|
||||
await JobModel.updateOne({ _id: "cleanup-orphaned" }, { $set: { nextScheduledAt: NOW + 86_400_000, runCount: 3 } });
|
||||
|
||||
// a restart reconcile re-upserts and pulls it due now
|
||||
await repo.upsertCleanupJob({ ...job, nextScheduledAt: NOW + 1_000 });
|
||||
|
||||
const row = await readRow("cleanup-orphaned");
|
||||
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // schedule reset ($set)
|
||||
expect(row?.runCount).toBe(3); // run history untouched ($setOnInsert)
|
||||
});
|
||||
|
||||
it("setActiveById flips check and geo rows together", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
|
||||
await repo.setActiveById("mon-1", false);
|
||||
|
||||
const rows = await JobModel.find({ refId: "mon-1" }).lean<JobDocument[]>();
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(rows.every((r) => r.isActive === false)).toBe(true);
|
||||
});
|
||||
|
||||
it("updateScheduleById changes the interval on a single typed row", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check", intervalMs: 60_000 });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check", intervalMs: 60_000 });
|
||||
|
||||
await repo.updateScheduleById("mon-1", "check", 15_000);
|
||||
|
||||
expect((await readRow("check:mon-1"))?.intervalMs).toBe(15_000);
|
||||
expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched
|
||||
});
|
||||
|
||||
it("deleteById drops every row for the monitor", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
|
||||
await repo.deleteById("mon-1");
|
||||
|
||||
expect(await JobModel.countDocuments({ refId: "mon-1" })).toBe(0);
|
||||
});
|
||||
|
||||
it("deleteByMonitorIdsNotIn drops jobs for missing monitors but keeps valid and global rows", async () => {
|
||||
await seedJob({ _id: "check:keep", refId: "keep", type: "check" });
|
||||
await seedJob({ _id: "geo:keep", refId: "keep", type: "geo-check" });
|
||||
await seedJob({ _id: "check:gone", refId: "gone", type: "check" });
|
||||
await seedJob({ _id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" });
|
||||
await seedJob({ _id: "cleanup-retention", refId: null, type: "cleanup-retention" });
|
||||
|
||||
const deleted = await repo.deleteByMonitorIdsNotIn(["keep"]);
|
||||
|
||||
expect(deleted).toBe(1); // only check:gone
|
||||
const remaining = (await JobModel.find().lean<JobDocument[]>()).map((d) => d._id).sort();
|
||||
expect(remaining).toEqual(["check:keep", "cleanup-orphaned", "cleanup-retention", "geo:keep"]);
|
||||
});
|
||||
|
||||
it("findById returns all of a monitor's rows", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
|
||||
const jobs = await repo.findById("mon-1");
|
||||
expect(jobs.map((j) => j.id).sort()).toEqual(["check:mon-1", "geo:mon-1"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── observability ────────────────────────────────────────────────────────────
|
||||
describe("observability", () => {
|
||||
it("findPage returns a page plus the total count", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a", nextScheduledAt: NOW + 1 });
|
||||
await seedJob({ _id: "check:b", refId: "b", nextScheduledAt: NOW + 2 });
|
||||
await seedJob({ _id: "check:c", refId: "c", nextScheduledAt: NOW + 3 });
|
||||
|
||||
const { jobs, count } = await repo.findPage({ page: 0, rowsPerPage: 2 });
|
||||
expect(count).toBe(3);
|
||||
expect(jobs).toHaveLength(2);
|
||||
expect(jobs.map((j) => j.id)).toEqual(["check:a", "check:b"]); // sorted by nextScheduledAt
|
||||
});
|
||||
|
||||
it("findPage returns everything when rowsPerPage is 0", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a" });
|
||||
await seedJob({ _id: "check:b", refId: "b" });
|
||||
|
||||
const { jobs, count } = await repo.findPage({});
|
||||
expect(count).toBe(2);
|
||||
expect(jobs).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("findAll returns every row", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a" });
|
||||
await seedJob({ _id: "geo:b", refId: "b", type: "geo-check" });
|
||||
|
||||
expect(await repo.findAll()).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
@@ -30,23 +32,23 @@ const createCheckPipeline = (overrides?: Record<string, any>) => {
|
||||
statusService: {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
checkService: { buildCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) },
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
...overrides,
|
||||
};
|
||||
const pipeline = new CheckPipeline(
|
||||
const checkProducer = new CheckProducer(
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.maintenanceWindowsRepository as any,
|
||||
defaults.checkService as any,
|
||||
defaults.networkService as any,
|
||||
defaults.buffer as any,
|
||||
defaults.monitorStatusPolicy as any,
|
||||
defaults.statusService as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
|
||||
const pipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
return { pipeline, defaults };
|
||||
};
|
||||
|
||||
@@ -85,9 +87,9 @@ describe("CheckPipeline", () => {
|
||||
await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response");
|
||||
});
|
||||
|
||||
it("returns null and warns when buildCheck returns null", async () => {
|
||||
it("returns null and warns when toCheck returns null", async () => {
|
||||
const { pipeline, defaults } = createCheckPipeline({
|
||||
checkService: { buildCheck: jest.fn().mockReturnValue(null) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(null) },
|
||||
});
|
||||
|
||||
const result = await pipeline.run(makeMonitor());
|
||||
|
||||
@@ -4,6 +4,8 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts";
|
||||
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -53,12 +55,12 @@ describe("CheckService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildCheck ───────────────────────────────────────────────────────────
|
||||
// ── toCheck ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("buildCheck", () => {
|
||||
describe("toCheck", () => {
|
||||
it("builds a basic HTTP check with correct fields", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse());
|
||||
const check = service.toCheck(makeStatusResponse());
|
||||
|
||||
expect(check).toBeDefined();
|
||||
expect(check!.metadata).toEqual({ monitorId: "mon-1", teamId: "team-1", type: "http" });
|
||||
@@ -72,14 +74,14 @@ describe("CheckService", () => {
|
||||
|
||||
it("defaults responseTime to 0 when falsy", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ responseTime: undefined }));
|
||||
const check = service.toCheck(makeStatusResponse({ responseTime: undefined }));
|
||||
expect(check!.responseTime).toBe(0);
|
||||
});
|
||||
|
||||
it("includes timings when provided", () => {
|
||||
const timings = { start: 0, socket: 10, lookup: 20 };
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ timings } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ timings } as any));
|
||||
expect(check!.timings).toBe(timings);
|
||||
});
|
||||
|
||||
@@ -106,7 +108,7 @@ describe("CheckService", () => {
|
||||
} as any;
|
||||
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.accessibility).toBe(90);
|
||||
expect(check!.bestPractices).toBe(80);
|
||||
@@ -119,7 +121,7 @@ describe("CheckService", () => {
|
||||
|
||||
it("returns undefined when pagespeed payload is missing", () => {
|
||||
const { service, logger } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any));
|
||||
|
||||
expect(check).toBeUndefined();
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build check" }));
|
||||
@@ -127,7 +129,7 @@ describe("CheckService", () => {
|
||||
|
||||
it("defaults categories and audits to empty when lighthouseResult is missing", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any));
|
||||
|
||||
expect(check!.accessibility).toBe(0);
|
||||
expect(check!.bestPractices).toBe(0);
|
||||
@@ -146,7 +148,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.cls!.score).toBe("informative");
|
||||
});
|
||||
@@ -161,7 +163,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.cls!.score).toBeNull();
|
||||
});
|
||||
@@ -176,7 +178,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.si!.numericValue).toBeUndefined();
|
||||
});
|
||||
@@ -192,7 +194,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.cls).toBeUndefined();
|
||||
expect(check!.audits!.si).toBeUndefined();
|
||||
@@ -214,7 +216,7 @@ describe("CheckService", () => {
|
||||
capture: { screenshot: "base64data" },
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
|
||||
expect(check!.cpu).toEqual({ usage_percent: 0.5 });
|
||||
expect(check!.memory).toEqual({ usage_percent: 0.6 });
|
||||
@@ -230,7 +232,7 @@ describe("CheckService", () => {
|
||||
errors: [{ message: "timeout" }],
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
|
||||
expect(check!.errors).toEqual([{ message: "timeout" }]);
|
||||
});
|
||||
@@ -241,14 +243,14 @@ describe("CheckService", () => {
|
||||
errors: { errors: [{ message: "nested error" }] },
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
|
||||
expect(check!.errors).toEqual([{ message: "nested error" }]);
|
||||
});
|
||||
|
||||
it("handles undefined payload gracefully", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any));
|
||||
|
||||
expect(check).toBeDefined();
|
||||
expect(check!.cpu).toBeUndefined();
|
||||
@@ -259,6 +261,84 @@ describe("CheckService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── toStatusResponse round-trip (Phase 2 §5 guard) ─────────────────────────
|
||||
// The DB-queue split persists a Check, then the evaluator rebuilds the status response from it.
|
||||
// toStatusResponse is only correct because every field a downstream consumer reads survives
|
||||
// toCheck → toStatusResponse. These tests pin that: if a future change drops a consumed field
|
||||
// from the round-trip, they fail instead of silently regressing the evaluator/reactors.
|
||||
|
||||
describe("toStatusResponse round-trip", () => {
|
||||
const makeHardwareStatus = (overrides?: Partial<MonitorStatusResponse>): MonitorStatusResponse =>
|
||||
makeStatusResponse({
|
||||
type: "hardware",
|
||||
status: true,
|
||||
code: 200,
|
||||
message: "OK",
|
||||
responseTime: 42,
|
||||
payload: {
|
||||
data: {
|
||||
cpu: { usage_percent: 0.92 },
|
||||
memory: { usage_percent: 0.81 },
|
||||
disk: [{ device: "/dev/sda", usage_percent: 0.75 }],
|
||||
host: { os: "linux" },
|
||||
net: [{ name: "eth0" }],
|
||||
},
|
||||
} as HardwareStatusPayload,
|
||||
...overrides,
|
||||
} as any);
|
||||
|
||||
it("uptime: preserves every consumed scalar field", () => {
|
||||
const { service } = createService();
|
||||
const status = makeStatusResponse({ type: "http", status: false, code: 503, message: "Service Unavailable", responseTime: 87 });
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
expect(rebuilt.monitorId).toBe(status.monitorId);
|
||||
expect(rebuilt.teamId).toBe(status.teamId);
|
||||
expect(rebuilt.type).toBe(status.type);
|
||||
expect(rebuilt.status).toBe(status.status);
|
||||
expect(rebuilt.code).toBe(status.code);
|
||||
expect(rebuilt.message).toBe(status.message);
|
||||
expect(rebuilt.responseTime).toBe(status.responseTime);
|
||||
});
|
||||
|
||||
it("hardware: preserves payload.data metrics verbatim, same units (usage_percent stays a 0–1 decimal)", () => {
|
||||
const { service } = createService();
|
||||
const status = makeHardwareStatus();
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
const original = status.payload as HardwareStatusPayload;
|
||||
const roundtripped = rebuilt.payload as HardwareStatusPayload;
|
||||
expect(roundtripped.data).toEqual(original.data);
|
||||
expect(roundtripped.data.cpu!.usage_percent).toBe(0.92); // not rescaled to 92
|
||||
});
|
||||
|
||||
it("hardware: a real consumer (extractThresholdBreaches) behaves identically on raw vs round-tripped status", () => {
|
||||
const { service } = createService();
|
||||
const builder = new NotificationMessageBuilder();
|
||||
const monitor = {
|
||||
id: "mon-1",
|
||||
type: "hardware",
|
||||
cpuAlertThreshold: 90, // 0.92 → 92% breaches
|
||||
memoryAlertThreshold: 90, // 0.81 → 81% does not
|
||||
diskAlertThreshold: 90, // 0.75 → 75% does not
|
||||
tempAlertThreshold: 100,
|
||||
} as Monitor;
|
||||
const status = makeHardwareStatus();
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
const fromRaw = builder.extractThresholdBreaches(monitor, status as MonitorStatusResponse<HardwareStatusPayload>);
|
||||
const fromRebuilt = builder.extractThresholdBreaches(monitor, rebuilt as MonitorStatusResponse<HardwareStatusPayload>);
|
||||
|
||||
// sanity: the fixture actually triggers a breach, so the equality below is not vacuous
|
||||
expect(fromRaw).toHaveLength(1);
|
||||
expect(fromRaw[0].metric).toBe("cpu");
|
||||
expect(fromRebuilt).toEqual(fromRaw);
|
||||
});
|
||||
});
|
||||
|
||||
// ── createChecks ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("createChecks", () => {
|
||||
|
||||
@@ -0,0 +1,536 @@
|
||||
import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals";
|
||||
import { DBQueueWorker } from "../../../src/worker/worker.db-queue.ts";
|
||||
import { LOCK_MS } from "../../../src/domain/jobs/job.type.ts";
|
||||
import type { Job } from "../../../src/domain/jobs/job.type.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
import type { QueueMode } from "../../../src/domain/app-settings/app-settings.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Notes ──────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
|
||||
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
|
||||
// renews their locks, and registers itself in the worker heartbeat. The repository
|
||||
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
|
||||
// test only the orchestration around it, with the repository fully mocked.
|
||||
//
|
||||
// Fake timers are mandatory: init() starts self-re-arming setTimeout loops and a
|
||||
// heartbeat setInterval, so without them the loops would leak across tests. The
|
||||
// loop re-arms at POLL_MS (250ms) and the lock renews at LOCK_MS / 3 (20s).
|
||||
|
||||
const POLL_MS = 250;
|
||||
const LOCK_RENEW_MS = LOCK_MS / 3;
|
||||
|
||||
const deferred = <T>() => {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
isActive: true,
|
||||
geoCheckEnabled: false,
|
||||
geoCheckInterval: 300000,
|
||||
lastEvaluatedAt: 0,
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
id: "check:m1",
|
||||
type: "check",
|
||||
refId: "m1",
|
||||
isActive: true,
|
||||
nextScheduledAt: 1000,
|
||||
intervalMs: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
|
||||
const jobsRepository = {
|
||||
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
|
||||
upsertJob: jest.fn<any>().mockResolvedValue(true),
|
||||
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
|
||||
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
|
||||
recordSuccess: jest.fn<any>().mockResolvedValue(true),
|
||||
recordFailure: jest.fn<any>().mockResolvedValue(true),
|
||||
recordOneShot: jest.fn<any>().mockResolvedValue(true),
|
||||
renewLocks: jest.fn<any>().mockResolvedValue(1),
|
||||
deleteById: jest.fn<any>().mockResolvedValue(true),
|
||||
deleteByIdAndType: jest.fn<any>().mockResolvedValue(true),
|
||||
setActiveById: jest.fn<any>().mockResolvedValue(true),
|
||||
updateScheduleById: jest.fn<any>().mockResolvedValue(true),
|
||||
findAll: jest.fn<any>().mockResolvedValue([]),
|
||||
findPage: jest.fn<any>().mockResolvedValue({ jobs: [], count: 0 }),
|
||||
};
|
||||
const monitorsRepository = {
|
||||
findByIds: jest.fn<any>().mockResolvedValue([makeMonitor()]),
|
||||
findAll: jest.fn<any>().mockResolvedValue([]),
|
||||
updateById: jest.fn<any>().mockResolvedValue({}),
|
||||
};
|
||||
const checksRepository = {
|
||||
findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([]),
|
||||
};
|
||||
const checkService = {
|
||||
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
|
||||
toLastEvaluatedAt: jest.fn<any>().mockReturnValue(12345),
|
||||
};
|
||||
const checkProducer = {
|
||||
produce: jest.fn<any>().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }),
|
||||
};
|
||||
const checkEvaluator = {
|
||||
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), decision: {} }),
|
||||
};
|
||||
const geoCheckPipeline = { run: jest.fn<any>().mockResolvedValue(null) };
|
||||
const dispatcher = { dispatch: jest.fn<any>().mockResolvedValue(undefined) };
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const queueWorkersRepository = {
|
||||
upsert: jest.fn<any>().mockResolvedValue(undefined),
|
||||
deleteById: jest.fn<any>().mockResolvedValue(undefined),
|
||||
findRecent: jest.fn<any>().mockResolvedValue([]),
|
||||
};
|
||||
const logger = createMockLogger();
|
||||
|
||||
const mocks = {
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
dispatcher,
|
||||
helper,
|
||||
queueWorkersRepository,
|
||||
logger,
|
||||
...overrides?.mocks,
|
||||
};
|
||||
|
||||
const worker = new DBQueueWorker(
|
||||
mocks.logger as any,
|
||||
mocks.jobsRepository as any,
|
||||
mocks.monitorsRepository as any,
|
||||
mocks.checksRepository as any,
|
||||
mocks.checkService as any,
|
||||
mocks.checkProducer as any,
|
||||
mocks.checkEvaluator as any,
|
||||
mocks.geoCheckPipeline as any,
|
||||
mocks.dispatcher as any,
|
||||
mocks.helper as any,
|
||||
mocks.queueWorkersRepository as any,
|
||||
overrides?.queueMode ?? "worker",
|
||||
overrides?.queuePrimaryProcesses ?? true,
|
||||
"worker-1"
|
||||
);
|
||||
return { worker, mocks };
|
||||
};
|
||||
|
||||
describe("DBQueueWorker", () => {
|
||||
let active: DBQueueWorker | null = null;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (active) await active.shutdown();
|
||||
active = null;
|
||||
jest.clearAllTimers();
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
const start = async (opts?: Parameters<typeof createWorker>[0]) => {
|
||||
const ctx = createWorker(opts);
|
||||
active = ctx.worker;
|
||||
await ctx.worker.init();
|
||||
await jest.advanceTimersByTimeAsync(1); // flush the immediate tick + any job it launched
|
||||
return ctx;
|
||||
};
|
||||
|
||||
// ── init() wiring ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("init", () => {
|
||||
it("registers the worker in the heartbeat registry on startup", async () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker");
|
||||
});
|
||||
|
||||
it("primary mode reconciles: seeds a check row per monitor plus the two cleanup rows", async () => {
|
||||
const { mocks } = await start({
|
||||
queueMode: "primary",
|
||||
queuePrimaryProcesses: false, // isolate reconcile from the polling loops
|
||||
mocks: {
|
||||
monitorsRepository: { findByIds: jest.fn<any>(), findAll: jest.fn<any>().mockResolvedValue([makeMonitor()]), updateById: jest.fn<any>() },
|
||||
},
|
||||
});
|
||||
|
||||
const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(seededTypes).toContain("check");
|
||||
const cleanupTypes = mocks.jobsRepository.upsertCleanupJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(cleanupTypes).toContain("cleanup-orphaned");
|
||||
expect(cleanupTypes).toContain("cleanup-retention");
|
||||
});
|
||||
|
||||
it("primary mode also seeds a geo-check row for geo-enabled monitors", async () => {
|
||||
const geoMonitor = makeMonitor({ type: "http", geoCheckEnabled: true });
|
||||
const { mocks } = await start({
|
||||
queueMode: "primary",
|
||||
queuePrimaryProcesses: false,
|
||||
mocks: {
|
||||
monitorsRepository: { findByIds: jest.fn<any>(), findAll: jest.fn<any>().mockResolvedValue([geoMonitor]), updateById: jest.fn<any>() },
|
||||
},
|
||||
});
|
||||
|
||||
const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(seededTypes).toContain("geo-check");
|
||||
});
|
||||
|
||||
it("worker mode does not reconcile (never seeds jobs)", async () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
expect(mocks.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("starts a polling loop per job type, each claiming up to its concurrency cap", async () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
|
||||
const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
|
||||
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
|
||||
});
|
||||
|
||||
it("does not start polling loops when primary and queuePrimaryProcesses is false", async () => {
|
||||
const { mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false });
|
||||
expect(mocks.jobsRepository.claimDueBatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("create() constructs the worker and runs init", async () => {
|
||||
const { mocks } = createWorker(); // reuse the mock set; the un-init'd instance is discarded
|
||||
const worker = await DBQueueWorker.create(
|
||||
mocks.logger as any,
|
||||
mocks.jobsRepository as any,
|
||||
mocks.monitorsRepository as any,
|
||||
mocks.checksRepository as any,
|
||||
mocks.checkService as any,
|
||||
mocks.checkProducer as any,
|
||||
mocks.checkEvaluator as any,
|
||||
mocks.geoCheckPipeline as any,
|
||||
mocks.dispatcher as any,
|
||||
mocks.helper as any,
|
||||
mocks.queueWorkersRepository as any,
|
||||
"worker",
|
||||
true,
|
||||
"worker-1"
|
||||
);
|
||||
active = worker;
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker");
|
||||
});
|
||||
});
|
||||
|
||||
// ── runJob: the per-type execution paths ────────────────────────────────────
|
||||
|
||||
describe("runJob", () => {
|
||||
// Hand the "check" loop exactly one job, then nothing, so the assertion target runs once.
|
||||
const claimOnce = (job: Job) => {
|
||||
let handed = false;
|
||||
return jest.fn<any>(async (type: string) => {
|
||||
if (type === job.type && !handed) {
|
||||
handed = true;
|
||||
return [job];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
};
|
||||
|
||||
it("runs a check job and reschedules it on success", async () => {
|
||||
const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]);
|
||||
expect(mocks.checkProducer.produce).toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("m1", expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("records a failure (and not a success) when the job handler throws", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: failingProducer },
|
||||
});
|
||||
|
||||
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluate job dispatches each unevaluated check and advances lastEvaluatedAt", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||
});
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
|
||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
|
||||
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
|
||||
});
|
||||
|
||||
it("cleanup-orphaned job runs the helper's cleanup function", async () => {
|
||||
const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 });
|
||||
const cleanupFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(cleanupFn),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(cleanupFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cleanup-retention job runs the helper's retention function", async () => {
|
||||
const job = makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 });
|
||||
const retentionFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(retentionFn),
|
||||
};
|
||||
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(retentionFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
|
||||
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.geoCheckPipeline.run).toHaveBeenCalled();
|
||||
expect(mocks.dispatcher.dispatch).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
});
|
||||
|
||||
it("renews the lock while a slow job is still running", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer },
|
||||
});
|
||||
|
||||
// Job is parked on produce(); push past one renewal interval.
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledWith([job.id], expect.any(Number));
|
||||
|
||||
// Let it finish; success is still recorded.
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops renewing once the lock is lost to another worker", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const jobsRepository = {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
renewLocks: jest.fn<any>().mockResolvedValue(0),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository, checkProducer: slowProducer } });
|
||||
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); // lease lost → renewal timer stops
|
||||
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Lost lock") }));
|
||||
|
||||
// A second interval passes without another renewal attempt.
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1);
|
||||
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── monitor lifecycle delegation ────────────────────────────────────────────
|
||||
|
||||
describe("job lifecycle methods", () => {
|
||||
it("addJob seeds a check row, plus a geo row for geo-enabled monitors", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.addJob("m1", makeMonitor({ geoCheckEnabled: true }));
|
||||
|
||||
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(types).toContain("check");
|
||||
expect(types).toContain("geo-check");
|
||||
});
|
||||
|
||||
it("addJob seeds only a check row for non-geo monitors", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.addJob("m1", makeMonitor({ geoCheckEnabled: false }));
|
||||
|
||||
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(types).toEqual(["check"]);
|
||||
});
|
||||
|
||||
it("deleteJob removes every row for the monitor", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.deleteJob(makeMonitor());
|
||||
expect(mocks.jobsRepository.deleteById).toHaveBeenCalledWith("m1");
|
||||
});
|
||||
|
||||
it("pauseJob and resumeJob flip the active flag", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.pauseJob(makeMonitor());
|
||||
expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", false);
|
||||
await worker.resumeJob(makeMonitor());
|
||||
expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", true);
|
||||
});
|
||||
|
||||
it("updateJob reschedules the check row and drops the geo row when geo is disabled", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.updateJob(makeMonitor({ geoCheckEnabled: false, interval: 30000 }));
|
||||
expect(mocks.jobsRepository.updateScheduleById).toHaveBeenCalledWith("m1", "check", 30000);
|
||||
expect(mocks.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith("m1", "geo-check");
|
||||
});
|
||||
|
||||
it("updateJob upserts the geo row when geo is enabled", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.updateJob(makeMonitor({ geoCheckEnabled: true }));
|
||||
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(types).toContain("geo-check");
|
||||
expect(mocks.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── shutdown ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("shutdown", () => {
|
||||
it("deregisters the worker and stops the polling loops", async () => {
|
||||
const { worker, mocks } = createWorker({ queueMode: "worker" });
|
||||
await worker.init();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
const claimsAfterInit = mocks.jobsRepository.claimDueBatch.mock.calls.length;
|
||||
|
||||
await worker.shutdown();
|
||||
expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1");
|
||||
|
||||
// Loops must not re-arm after shutdown.
|
||||
await jest.advanceTimersByTimeAsync(POLL_MS * 4);
|
||||
expect(mocks.jobsRepository.claimDueBatch.mock.calls.length).toBe(claimsAfterInit);
|
||||
// already shut down; keep afterEach from double-shutting-down
|
||||
active = null;
|
||||
});
|
||||
});
|
||||
|
||||
// ── flushQueues ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("flushQueues", () => {
|
||||
it("restarts the worker and reports success", async () => {
|
||||
const { worker, mocks } = createWorker({ queueMode: "worker" });
|
||||
active = worker;
|
||||
await worker.init();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
const result = await worker.flushQueues();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(result).toEqual({ success: true });
|
||||
expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1"); // shutdown leg
|
||||
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledTimes(2); // init + re-init
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics", () => {
|
||||
it("aggregates run/fail counts, active leases, and failing jobs across all rows", async () => {
|
||||
const now = Date.now();
|
||||
const rows = [
|
||||
makeJob({ id: "a", runCount: 5, failCount: 0, lockedBy: "w", lockedUntil: now + 10000 }), // active lease
|
||||
makeJob({ id: "b", runCount: 2, failCount: 3, lastFailReason: "boom", lockedBy: null, lockedUntil: null }), // failing
|
||||
makeJob({ id: "c", runCount: 1, failCount: 0, lockedBy: "w", lockedUntil: now - 10000 }), // expired lease, not active
|
||||
];
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
mocks.jobsRepository.findAll.mockResolvedValue(rows);
|
||||
mocks.queueWorkersRepository.findRecent.mockResolvedValue([{ workerId: "w" }]);
|
||||
|
||||
const metrics = await worker.getMetrics();
|
||||
|
||||
expect(metrics.jobs).toBe(3);
|
||||
expect(metrics.totalRuns).toBe(8);
|
||||
expect(metrics.totalFailures).toBe(3);
|
||||
expect(metrics.activeJobs).toBe(1);
|
||||
expect(metrics.failingJobs).toBe(1);
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ failCount: 3, failReason: "boom" }));
|
||||
expect(metrics.workers).toEqual([{ workerId: "w" }]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getJobs ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getJobs", () => {
|
||||
it("maps rows to summaries and passes the total count through", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
mocks.jobsRepository.findPage.mockResolvedValue({
|
||||
jobs: [makeJob({ id: "check:m1", refId: "m1", type: "check", intervalMs: 60000 })],
|
||||
count: 1,
|
||||
});
|
||||
|
||||
const page = await worker.getJobs({ page: 0, rowsPerPage: 10 });
|
||||
|
||||
expect(page.count).toBe(1);
|
||||
expect(page.jobs).toHaveLength(1);
|
||||
expect(page.jobs[0]).toEqual(expect.objectContaining({ monitorId: "m1", monitorType: "check", monitorInterval: 60000, repeat: 60000 }));
|
||||
});
|
||||
|
||||
it("falls back to the job id for monitorId when refId is null (global jobs)", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
mocks.jobsRepository.findPage.mockResolvedValue({
|
||||
jobs: [makeJob({ id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" })],
|
||||
count: 1,
|
||||
});
|
||||
|
||||
const page = await worker.getJobs({ page: 0, rowsPerPage: 10 });
|
||||
expect(page.jobs[0].monitorId).toBe("cleanup-orphaned");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,791 +0,0 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
|
||||
// ── Mock Scheduler / MongoStore ──────────────────────────────────────────────
|
||||
|
||||
const createScheduler = () => {
|
||||
const listeners: Record<string, Function[]> = {};
|
||||
return {
|
||||
workerId: "host-1:1234:abcd",
|
||||
on: jest.fn((event: string, cb: Function) => {
|
||||
listeners[event] = listeners[event] || [];
|
||||
listeners[event].push(cb);
|
||||
}),
|
||||
off: jest.fn((event: string, cb: Function) => {
|
||||
listeners[event] = (listeners[event] || []).filter((fn) => fn !== cb);
|
||||
}),
|
||||
emit: (event: string, ...args: unknown[]) => {
|
||||
(listeners[event] || []).forEach((cb) => cb(...args));
|
||||
},
|
||||
start: jest.fn().mockResolvedValue(true),
|
||||
stop: jest.fn().mockResolvedValue(true),
|
||||
addTemplate: jest.fn(),
|
||||
addJob: jest.fn(),
|
||||
addJobs: jest.fn().mockResolvedValue({ inserted: 0, upserted: 0, modified: 0, failures: [] }),
|
||||
removeJob: jest.fn(),
|
||||
getJob: jest.fn().mockResolvedValue(null),
|
||||
getJobs: jest.fn().mockResolvedValue([]),
|
||||
countJobs: jest.fn().mockResolvedValue(0),
|
||||
getStats: jest.fn().mockResolvedValue({ jobs: 0, activeJobs: 0, failingJobs: 0, totalRuns: 0, totalFailures: 0, jobsWithFailures: [] }),
|
||||
pauseJob: jest.fn().mockResolvedValue(true),
|
||||
resumeJob: jest.fn().mockResolvedValue(true),
|
||||
updateJob: jest.fn(),
|
||||
flushJobs: jest.fn().mockResolvedValue(true),
|
||||
};
|
||||
};
|
||||
|
||||
const mockSchedulerInstance = createScheduler();
|
||||
const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance);
|
||||
const MockMongoStore = jest.fn();
|
||||
|
||||
jest.unstable_mockModule("less-simple-scheduler", () => ({
|
||||
Scheduler: MockScheduler,
|
||||
MongoStore: MockMongoStore,
|
||||
}));
|
||||
|
||||
const { LessSimpleWorker: LessSimpleQueue } = await import("../../../src/worker/worker.less-simple.ts");
|
||||
|
||||
const createQueueHelper = () => ({
|
||||
getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
});
|
||||
|
||||
const createMonitorsRepo = () => ({
|
||||
findAll: jest.fn().mockResolvedValue([]),
|
||||
});
|
||||
|
||||
const createWorkersRepo = () => ({
|
||||
upsert: jest.fn().mockResolvedValue(undefined),
|
||||
findRecent: jest.fn().mockResolvedValue([]),
|
||||
deleteById: jest.fn().mockResolvedValue(undefined),
|
||||
});
|
||||
|
||||
const createQueue = (overrides?: Record<string, unknown>) => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const workersRepository = createWorkersRepo();
|
||||
const scheduler = createScheduler();
|
||||
|
||||
const defaults = { logger, helper, monitorsRepository, workersRepository, scheduler, queueMode: "primary", ...overrides };
|
||||
|
||||
const queue = new LessSimpleQueue(
|
||||
defaults.logger as any,
|
||||
defaults.helper as any,
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.workersRepository as any,
|
||||
defaults.scheduler as any,
|
||||
defaults.queueMode as any
|
||||
);
|
||||
return { queue, ...defaults };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "mon-1",
|
||||
teamId: "team-1",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
isActive: true,
|
||||
geoCheckEnabled: false,
|
||||
geoCheckInterval: 300000,
|
||||
geoCheckLocations: [],
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("LessSimpleQueue", () => {
|
||||
describe("serviceName", () => {
|
||||
it("returns JobQueue", () => {
|
||||
const { queue } = createQueue();
|
||||
expect(queue.serviceName).toBe("JobQueue");
|
||||
});
|
||||
});
|
||||
|
||||
// ── init (primary) ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("init - primary mode", () => {
|
||||
it("starts scheduler, registers all templates, seeds monitors and system jobs", async () => {
|
||||
const { queue, scheduler, monitorsRepository } = createQueue();
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.start).toHaveBeenCalled();
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
|
||||
expect(monitorsRepository.findAll).toHaveBeenCalled();
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
|
||||
});
|
||||
|
||||
it("registers existing monitors in a single bulk write", async () => {
|
||||
const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })];
|
||||
const { queue, scheduler, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors);
|
||||
|
||||
await queue.init();
|
||||
|
||||
expect(scheduler.addJobs).toHaveBeenCalledTimes(1);
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ id: "m1", template: "monitor-job", jitter: true }),
|
||||
expect.objectContaining({ id: "m2", template: "monitor-job", jitter: true }),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it("returns true and skips system jobs when findAll returns null", async () => {
|
||||
const { queue, monitorsRepository, scheduler } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(null);
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
});
|
||||
|
||||
it("returns false and logs error on failure", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") }));
|
||||
});
|
||||
|
||||
it("logs error with String(error) for non-Error exceptions", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error");
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── init (worker) ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("init - worker mode", () => {
|
||||
it("starts scheduler and registers all templates", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
|
||||
const result = await queue.init("worker");
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.start).toHaveBeenCalled();
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
|
||||
});
|
||||
|
||||
it("does NOT seed monitor jobs (no findAll, primary populates the queue)", async () => {
|
||||
const { queue, monitorsRepository } = createQueue();
|
||||
|
||||
await queue.init("worker");
|
||||
|
||||
expect(monitorsRepository.findAll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does NOT seed cleanup system jobs", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
|
||||
await queue.init("worker");
|
||||
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs the initialized worker with its workerId", async () => {
|
||||
const { queue, logger } = createQueue();
|
||||
|
||||
await queue.init("worker");
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Worker queue: host-1:1234:abcd initialized" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── registerListeners ──────────────────────────────────────────────────────
|
||||
|
||||
describe("registerListeners (via constructor)", () => {
|
||||
it("logs on scheduler:start", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:start", "host-1:1234:abcd");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:stop", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:stop", "host-1:1234:abcd");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:error", "w1", new Error("boom"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: boom" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with non-Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:error", "w1", "string error");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: string error", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:locked", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:locked", "w1", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 locked" }));
|
||||
});
|
||||
|
||||
it("logs on job:abort", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:abort", "w1", { id: "m1" }, "timeout");
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 aborted: timeout" }));
|
||||
});
|
||||
|
||||
it("logs on job:attempt", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:attempt", "w1", { id: "m1" }, 2);
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 attempt 2" }));
|
||||
});
|
||||
|
||||
it("logs on job:complete", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:complete", "w1", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 completed successfully" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:exhausted", "w1", { id: "m1" }, new Error("gave up"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with non-Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:exhausted", "w1", { id: "m1" }, "gave up");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:fail", "w1", { id: "m1" }, new Error("oops"), 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops" }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with non-Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:fail", "w1", { id: "m1" }, "oops", 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:start", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:start", "w1", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 started" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── addJob ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("addJob", () => {
|
||||
it("adds monitor job to scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor());
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true })])
|
||||
);
|
||||
});
|
||||
|
||||
it("adds geo check job when geoCheckEnabled and type supports it", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }));
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 })])
|
||||
);
|
||||
});
|
||||
|
||||
it("skips geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })]));
|
||||
});
|
||||
|
||||
it("skips geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })]));
|
||||
});
|
||||
});
|
||||
|
||||
// ── deleteJob ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("deleteJob", () => {
|
||||
it("removes monitor job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1");
|
||||
});
|
||||
|
||||
it("removes geo job if it exists", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not remove geo job if it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
});
|
||||
|
||||
// ── pauseJob ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("pauseJob", () => {
|
||||
it("pauses monitor and geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.pauseJob(makeMonitor());
|
||||
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("does not pause geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.pauseJob(makeMonitor());
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("throws when scheduler.pauseJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor");
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo pauseJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── resumeJob ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("resumeJob", () => {
|
||||
it("resumes monitor and geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.resumeJob(makeMonitor());
|
||||
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("does not resume geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.resumeJob(makeMonitor());
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("throws when scheduler.resumeJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor");
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo resumeJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── updateJob / syncGeoJob ─────────────────────────────────────────────────
|
||||
|
||||
describe("updateJob", () => {
|
||||
it("updates monitor job with new interval", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ interval: 120000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor }));
|
||||
});
|
||||
|
||||
it("removes existing geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("removes existing geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("updates existing geo job when geoCheckEnabled", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 }));
|
||||
});
|
||||
|
||||
it("creates new geo job when enabled but no existing job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── shutdown ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("shutdown", () => {
|
||||
it("stops the scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.shutdown();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics", () => {
|
||||
it("returns empty metrics when no jobs", async () => {
|
||||
const { queue } = createQueue();
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics).toEqual({ jobs: 0, activeJobs: 0, failingJobs: 0, jobsWithFailures: [], totalRuns: 0, totalFailures: 0, workers: [] });
|
||||
});
|
||||
|
||||
it("maps the store's aggregate counters through", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getStats as jest.Mock).mockResolvedValue({
|
||||
jobs: 2,
|
||||
activeJobs: 1,
|
||||
failingJobs: 1,
|
||||
totalRuns: 15,
|
||||
totalFailures: 2,
|
||||
jobsWithFailures: [],
|
||||
});
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics).toMatchObject({ jobs: 2, activeJobs: 1, failingJobs: 1, totalRuns: 15, totalFailures: 2 });
|
||||
});
|
||||
|
||||
it("derives monitor fields for failing monitor jobs from job data", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getStats as jest.Mock).mockResolvedValue({
|
||||
jobs: 1,
|
||||
activeJobs: 0,
|
||||
failingJobs: 1,
|
||||
totalRuns: 10,
|
||||
totalFailures: 2,
|
||||
jobsWithFailures: [{ id: "m1", data: { url: "http://a.com", type: "http" }, failedAt: 200, failCount: 2, failReason: "timeout" }],
|
||||
});
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures).toEqual([
|
||||
{ monitorId: "m1", monitorUrl: "http://a.com", monitorType: "http", failedAt: 200, failCount: 2, failReason: "timeout" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("leaves monitor fields null for non-monitor (cleanup) failures", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getStats as jest.Mock).mockResolvedValue({
|
||||
jobs: 1,
|
||||
activeJobs: 0,
|
||||
failingJobs: 1,
|
||||
totalRuns: 1,
|
||||
totalFailures: 1,
|
||||
jobsWithFailures: [{ id: "cleanup-orphaned", data: undefined, failedAt: 100, failCount: 1, failReason: "boom" }],
|
||||
});
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures[0]).toEqual({
|
||||
monitorId: "cleanup-orphaned",
|
||||
monitorUrl: null,
|
||||
monitorType: null,
|
||||
failedAt: 100,
|
||||
failCount: 1,
|
||||
failReason: "boom",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics: worker registry ──────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics - worker registry", () => {
|
||||
it("returns the alive workers from the registry", async () => {
|
||||
const { queue, workersRepository } = createQueue();
|
||||
const alive = [
|
||||
{ workerId: "host:1:aaa", mode: "primary", lastSeenAt: 1000 },
|
||||
{ workerId: "host:2:bbb", mode: "worker", lastSeenAt: 2000 },
|
||||
];
|
||||
(workersRepository.findRecent as jest.Mock).mockResolvedValue(alive);
|
||||
|
||||
const metrics = await queue.getMetrics();
|
||||
|
||||
expect(workersRepository.findRecent).toHaveBeenCalledWith(30000);
|
||||
expect(metrics.workers).toEqual(alive);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getJobs ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getJobs", () => {
|
||||
it("maps scheduler jobs to summaries", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{
|
||||
id: "m1",
|
||||
template: "monitor-job",
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
lastError: "timeout",
|
||||
lastStartedAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastFailedAt: 900,
|
||||
data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true },
|
||||
},
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]).toEqual({
|
||||
monitorId: "m1",
|
||||
monitorUrl: "http://a.com",
|
||||
monitorType: "http",
|
||||
monitorInterval: 60000,
|
||||
monitorGeoInterval: 300000,
|
||||
monitorActive: true,
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
failReason: "timeout",
|
||||
lastRunAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastRunTook: 100,
|
||||
lastFailedAt: 900,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null lastRunTook when job is locked", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{
|
||||
id: "m1",
|
||||
template: "monitor-job",
|
||||
active: true,
|
||||
lockedAt: 999,
|
||||
runCount: 1,
|
||||
lastStartedAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
data: { url: "a", type: "http", interval: 60000 },
|
||||
},
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].lastRunTook).toBeNull();
|
||||
});
|
||||
|
||||
it("leaves monitor fields null for non-monitor jobs", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, data: undefined }]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].monitorUrl).toBeNull();
|
||||
expect(jobs[0].monitorType).toBeNull();
|
||||
expect(jobs[0].monitorInterval).toBeNull();
|
||||
expect(jobs[0].monitorGeoInterval).toBeNull();
|
||||
expect(jobs[0].monitorActive).toBeNull();
|
||||
});
|
||||
|
||||
it("pushes pagination to the scheduler and returns its total count", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", template: "monitor-job", active: true, data: { url: "a", type: "http", interval: 60000 } },
|
||||
]);
|
||||
(scheduler.countJobs as jest.Mock).mockResolvedValue(42);
|
||||
|
||||
const { jobs, count } = await queue.getJobs({ page: 2, rowsPerPage: 10 });
|
||||
|
||||
expect(scheduler.getJobs).toHaveBeenCalledWith({ skip: 20, limit: 10 });
|
||||
expect(count).toBe(42);
|
||||
expect(jobs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("requests all jobs when rowsPerPage is omitted", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.countJobs as jest.Mock).mockResolvedValue(7);
|
||||
|
||||
const { count } = await queue.getJobs({});
|
||||
|
||||
expect(scheduler.getJobs).toHaveBeenCalledWith(undefined);
|
||||
expect(count).toBe(7);
|
||||
});
|
||||
});
|
||||
|
||||
// ── flushQueues ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("flushQueues", () => {
|
||||
it("stops, flushes, and reinitializes", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const result = await queue.flushQueues();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
expect(scheduler.flushJobs).toHaveBeenCalled();
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("flushes before stopping, so the store is still connected for removeAll", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const order: string[] = [];
|
||||
(scheduler.flushJobs as jest.Mock).mockImplementation(async () => {
|
||||
order.push("flush");
|
||||
return true;
|
||||
});
|
||||
(scheduler.stop as jest.Mock).mockImplementation(async () => {
|
||||
order.push("stop");
|
||||
return true;
|
||||
});
|
||||
await queue.flushQueues();
|
||||
expect(order).toEqual(["flush", "stop"]);
|
||||
});
|
||||
|
||||
it("refuses to flush on a non-primary worker without touching the scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue({ queueMode: "worker" });
|
||||
const result = await queue.flushQueues();
|
||||
expect(result.success).toBe(false);
|
||||
expect(scheduler.flushJobs).not.toHaveBeenCalled();
|
||||
expect(scheduler.stop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns false when flush fails", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.flushJobs as jest.Mock).mockResolvedValue(false);
|
||||
const result = await queue.flushQueues();
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── static create ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("static create", () => {
|
||||
const envSettings = { dbConnectionString: "mongodb://localhost:27017/test_db" } as any;
|
||||
|
||||
it("creates a MongoStore + Scheduler and inits in primary mode", async () => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const workersRepository = createWorkersRepo();
|
||||
|
||||
const instance = await LessSimpleQueue.create(
|
||||
logger as any,
|
||||
helper as any,
|
||||
monitorsRepository as any,
|
||||
workersRepository as any,
|
||||
envSettings,
|
||||
"primary"
|
||||
);
|
||||
|
||||
expect(MockMongoStore).toHaveBeenCalledWith(expect.objectContaining({ url: "mongodb://localhost:27017/test_db" }));
|
||||
expect(MockScheduler).toHaveBeenCalled();
|
||||
expect(instance).toBeInstanceOf(LessSimpleQueue);
|
||||
expect(mockSchedulerInstance.start).toHaveBeenCalled();
|
||||
expect(monitorsRepository.findAll).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("inits in worker mode without seeding monitors", async () => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const workersRepository = createWorkersRepo();
|
||||
|
||||
const instance = await LessSimpleQueue.create(
|
||||
logger as any,
|
||||
helper as any,
|
||||
monitorsRepository as any,
|
||||
workersRepository as any,
|
||||
envSettings,
|
||||
"worker"
|
||||
);
|
||||
|
||||
expect(instance).toBeInstanceOf(LessSimpleQueue);
|
||||
expect(monitorsRepository.findAll).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── worker heartbeat / deregistration ────────────────────────────────────────
|
||||
|
||||
describe("worker heartbeat", () => {
|
||||
it("writes a heartbeat on init with the workerId and mode (primary)", async () => {
|
||||
const { queue, workersRepository } = createQueue();
|
||||
await queue.init("primary");
|
||||
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "primary");
|
||||
});
|
||||
|
||||
it("writes a heartbeat on init with the workerId and mode (worker)", async () => {
|
||||
const { queue, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker");
|
||||
});
|
||||
|
||||
it("beats again on each scheduler:heartbeat event", async () => {
|
||||
const { queue, scheduler, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
(workersRepository.upsert as jest.Mock).mockClear();
|
||||
|
||||
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
|
||||
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
|
||||
|
||||
expect(workersRepository.upsert).toHaveBeenCalledTimes(2);
|
||||
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker");
|
||||
});
|
||||
|
||||
it("stops beating after shutdown", async () => {
|
||||
const { queue, scheduler, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
await queue.shutdown();
|
||||
(workersRepository.upsert as jest.Mock).mockClear();
|
||||
|
||||
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
|
||||
|
||||
expect(workersRepository.upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("deregisters the worker and stops the scheduler on shutdown", async () => {
|
||||
const { queue, scheduler, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
await queue.shutdown();
|
||||
expect(workersRepository.deleteById).toHaveBeenCalledWith("host-1:1234:abcd");
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,712 +0,0 @@
|
||||
import { describe, expect, it, jest, beforeEach } from "@jest/globals";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
|
||||
// ── Mock Scheduler ───────────────────────────────────────────────────────────
|
||||
|
||||
const createScheduler = () => {
|
||||
const listeners: Record<string, Function[]> = {};
|
||||
return {
|
||||
on: jest.fn((event: string, cb: Function) => {
|
||||
listeners[event] = listeners[event] || [];
|
||||
listeners[event].push(cb);
|
||||
}),
|
||||
emit: (event: string, ...args: unknown[]) => {
|
||||
(listeners[event] || []).forEach((cb) => cb(...args));
|
||||
},
|
||||
start: jest.fn(),
|
||||
stop: jest.fn().mockResolvedValue(true),
|
||||
addTemplate: jest.fn(),
|
||||
addJob: jest.fn(),
|
||||
removeJob: jest.fn(),
|
||||
getJob: jest.fn().mockResolvedValue(null),
|
||||
getJobs: jest.fn().mockResolvedValue([]),
|
||||
pauseJob: jest.fn().mockResolvedValue(true),
|
||||
resumeJob: jest.fn().mockResolvedValue(true),
|
||||
updateJob: jest.fn(),
|
||||
flushJobs: jest.fn().mockResolvedValue(true),
|
||||
};
|
||||
};
|
||||
|
||||
const mockSchedulerInstance = createScheduler();
|
||||
const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance);
|
||||
|
||||
jest.unstable_mockModule("super-simple-scheduler", () => ({
|
||||
default: MockScheduler,
|
||||
}));
|
||||
|
||||
const { SuperSimpleQueue } = await import("../../../src/worker/worker.super-simple.ts");
|
||||
|
||||
const createQueueHelper = () => ({
|
||||
getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
});
|
||||
|
||||
const createMonitorsRepo = () => ({
|
||||
findAll: jest.fn().mockResolvedValue([]),
|
||||
});
|
||||
|
||||
const createQueue = (overrides?: Record<string, unknown>) => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const scheduler = createScheduler();
|
||||
|
||||
const defaults = { logger, helper, monitorsRepository, scheduler, ...overrides };
|
||||
|
||||
const queue = new SuperSimpleQueue(defaults.logger as any, defaults.helper as any, defaults.monitorsRepository as any, defaults.scheduler as any);
|
||||
return { queue, ...defaults };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "mon-1",
|
||||
teamId: "team-1",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
isActive: true,
|
||||
geoCheckEnabled: false,
|
||||
geoCheckInterval: 300000,
|
||||
geoCheckLocations: [],
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("SuperSimpleQueue", () => {
|
||||
describe("serviceName", () => {
|
||||
it("returns JobQueue", () => {
|
||||
const { queue } = createQueue();
|
||||
expect(queue.serviceName).toBe("JobQueue");
|
||||
});
|
||||
});
|
||||
|
||||
// ── init ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("init", () => {
|
||||
it("registers listeners, starts scheduler, adds templates and system jobs", async () => {
|
||||
const { queue, scheduler, helper } = createQueue();
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.start).toHaveBeenCalled();
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
|
||||
});
|
||||
|
||||
it("schedules existing monitors with staggered offsets", async () => {
|
||||
jest.useFakeTimers();
|
||||
const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })];
|
||||
const { queue, scheduler, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors);
|
||||
|
||||
await queue.init();
|
||||
jest.runAllTimers();
|
||||
|
||||
// Each monitor gets addJob called (once for the monitor itself via addJob method)
|
||||
// The init schedules via setTimeout → addJob, plus the two system jobs
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m1", template: "monitor-job" }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m2", template: "monitor-job" }));
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it("returns true when findAll returns null", async () => {
|
||||
const { queue, monitorsRepository, scheduler } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(null);
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
// System jobs should not be added when monitors is null (early return)
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
});
|
||||
|
||||
it("returns false and logs error on failure", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") }));
|
||||
});
|
||||
|
||||
it("logs error with String(error) for non-Error exceptions", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error");
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── registerListeners ────────────────────────────────────────────────────
|
||||
|
||||
describe("registerListeners (via init)", () => {
|
||||
it("logs on scheduler:start", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:start");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:stop", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:stop");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:error", new Error("boom"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: boom" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with non-Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:error", "string error");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: string error", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:abort", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:abort", { id: "m1" }, "timeout");
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 aborted: timeout" }));
|
||||
});
|
||||
|
||||
it("logs on job:attempt", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:attempt", { id: "m1" }, 2);
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 attempt 2" }));
|
||||
});
|
||||
|
||||
it("logs on job:complete", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:complete", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 completed successfully" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:exhausted", { id: "m1" }, new Error("gave up"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with non-Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:exhausted", { id: "m1" }, "gave up");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:fail", { id: "m1" }, new Error("oops"), 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops" }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with non-Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:fail", { id: "m1" }, "oops", 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:start", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:start", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 started" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── addJob ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("addJob", () => {
|
||||
it("adds monitor job to scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor());
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true }));
|
||||
});
|
||||
|
||||
it("adds geo check job when geoCheckEnabled and type supports it", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 }));
|
||||
});
|
||||
|
||||
it("skips geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" }));
|
||||
});
|
||||
|
||||
it("skips geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── deleteJob ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("deleteJob", () => {
|
||||
it("removes monitor job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1");
|
||||
});
|
||||
|
||||
it("removes geo job if it exists", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not remove geo job if it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
});
|
||||
|
||||
// ── pauseJob ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("pauseJob", () => {
|
||||
it("pauses monitor and pauses geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.pauseJob(makeMonitor());
|
||||
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("refreshes scheduler data with current monitor on pause", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ isActive: false, status: "paused" });
|
||||
|
||||
await queue.pauseJob(monitor);
|
||||
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor });
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor });
|
||||
});
|
||||
|
||||
it("does not pause geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.pauseJob(makeMonitor());
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not refresh geo data when geo job does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.pauseJob(makeMonitor());
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() }));
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything());
|
||||
});
|
||||
|
||||
it("throws when scheduler.pauseJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor");
|
||||
});
|
||||
|
||||
it("does not refresh data when main pauseJob throws", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow();
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo pauseJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── resumeJob ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("resumeJob", () => {
|
||||
it("resumes monitor and geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.resumeJob(makeMonitor());
|
||||
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("refreshes scheduler data with current monitor on resume", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ isActive: true, status: "initializing" });
|
||||
|
||||
await queue.resumeJob(monitor);
|
||||
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor });
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor });
|
||||
});
|
||||
|
||||
it("does not resume geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.resumeJob(makeMonitor());
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not refresh geo data when geo job does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.resumeJob(makeMonitor());
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() }));
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything());
|
||||
});
|
||||
|
||||
it("throws when scheduler.resumeJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor");
|
||||
});
|
||||
|
||||
it("does not refresh data when main resumeJob throws", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow();
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo resumeJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("preserves geo job through a pause/resume cycle", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
|
||||
await queue.pauseJob(makeMonitor());
|
||||
await queue.resumeJob(makeMonitor());
|
||||
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
});
|
||||
|
||||
// ── updateJob ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("updateJob", () => {
|
||||
it("updates monitor job and syncs geo job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ interval: 120000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── syncGeoJob (via updateJob) ───────────────────────────────────────────
|
||||
|
||||
describe("syncGeoJob (via updateJob)", () => {
|
||||
it("removes existing geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("removes existing geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not remove when geo disabled and no existing geo job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: false }));
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("updates existing geo job when geoCheckEnabled", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 }));
|
||||
});
|
||||
|
||||
it("creates new geo job when enabled but no existing job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── shutdown ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("shutdown", () => {
|
||||
it("stops the scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.shutdown();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics", () => {
|
||||
it("returns empty job metrics plus this process as the sole worker when no jobs", async () => {
|
||||
const { queue } = createQueue();
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics).toEqual({
|
||||
jobs: 0,
|
||||
activeJobs: 0,
|
||||
failingJobs: 0,
|
||||
jobsWithFailures: [],
|
||||
totalRuns: 0,
|
||||
totalFailures: 0,
|
||||
workers: [
|
||||
{
|
||||
// hostname:pid:uuid, matching less-simple-scheduler's workerId format
|
||||
workerId: expect.stringMatching(/^.+:\d+:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/),
|
||||
mode: "primary",
|
||||
lastSeenAt: expect.any(Number),
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("aggregates metrics from jobs", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", runCount: 10, failCount: 2, lastFailedAt: 200, lastRunAt: 100, lockedAt: null, data: { url: "http://a.com", type: "http" } },
|
||||
{ id: "m2", runCount: 5, failCount: 0, lastFailedAt: 0, lastRunAt: 50, lockedAt: 999, data: { url: "http://b.com", type: "ping" } },
|
||||
]);
|
||||
const metrics = await queue.getMetrics();
|
||||
|
||||
expect(metrics.jobs).toBe(2);
|
||||
expect(metrics.totalRuns).toBe(15);
|
||||
expect(metrics.totalFailures).toBe(2);
|
||||
expect(metrics.activeJobs).toBe(1);
|
||||
expect(metrics.failingJobs).toBe(1);
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ monitorId: "m1", failCount: 2, monitorUrl: "http://a.com" }));
|
||||
});
|
||||
|
||||
it("handles jobs with undefined fields", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", data: null }]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobs).toBe(1);
|
||||
expect(metrics.totalRuns).toBe(0);
|
||||
});
|
||||
|
||||
it("handles failing job with null data", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", runCount: 1, failCount: 1, lastFailedAt: 100, lastRunAt: 50, data: null, lastFailReason: "timeout" },
|
||||
]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
expect(metrics.jobsWithFailures[0].monitorUrl).toBeNull();
|
||||
expect(metrics.jobsWithFailures[0].monitorType).toBeNull();
|
||||
expect(metrics.jobsWithFailures[0].failReason).toBe("timeout");
|
||||
});
|
||||
|
||||
it("handles failing job with undefined lastFailedAt and lastFailReason", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", runCount: 1, failCount: 1, data: { url: "http://a.com", type: "http" } }]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures[0].failedAt).toBeNull();
|
||||
expect(metrics.jobsWithFailures[0].failReason).toBeNull();
|
||||
});
|
||||
|
||||
it("does not count as failing when lastFailedAt < lastRunAt", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", runCount: 10, failCount: 1, lastFailedAt: 50, lastRunAt: 100, data: { url: "a" } },
|
||||
]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.failingJobs).toBe(0);
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getJobs ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getJobs", () => {
|
||||
it("maps scheduler jobs to summaries", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{
|
||||
id: "m1",
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
lastFailReason: "timeout",
|
||||
lastRunAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastFailedAt: 900,
|
||||
data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true },
|
||||
},
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]).toEqual({
|
||||
monitorId: "m1",
|
||||
monitorUrl: "http://a.com",
|
||||
monitorType: "http",
|
||||
monitorInterval: 60000,
|
||||
monitorGeoInterval: 300000,
|
||||
monitorActive: true,
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
failReason: "timeout",
|
||||
lastRunAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastRunTook: 100,
|
||||
lastFailedAt: 900,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null lastRunTook when job is locked", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", active: true, lockedAt: 999, runCount: 1, data: { url: "a", type: "http", interval: 60000 } },
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].lastRunTook).toBeNull();
|
||||
});
|
||||
|
||||
it("handles jobs with missing data", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", active: true, data: null }]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].monitorUrl).toBeNull();
|
||||
expect(jobs[0].monitorType).toBeNull();
|
||||
expect(jobs[0].monitorInterval).toBeNull();
|
||||
expect(jobs[0].monitorGeoInterval).toBeNull();
|
||||
expect(jobs[0].monitorActive).toBeNull();
|
||||
expect(jobs[0].repeat).toBeNull();
|
||||
});
|
||||
|
||||
it("exposes monitor.isActive as monitorActive in the summary", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", active: true, data: { url: "a", type: "http", interval: 60000, isActive: true } },
|
||||
{ id: "m2", active: false, data: { url: "b", type: "http", interval: 60000, isActive: false } },
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].monitorActive).toBe(true);
|
||||
expect(jobs[1].monitorActive).toBe(false);
|
||||
});
|
||||
|
||||
it("exposes geo check interval and repeat", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", active: true, repeat: 120000, data: { url: "a", type: "http", interval: 60000, geoCheckInterval: 600000 } },
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].repeat).toBe(120000);
|
||||
expect(jobs[0].monitorGeoInterval).toBe(600000);
|
||||
});
|
||||
|
||||
it("paginates jobs and returns the total count", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue(
|
||||
Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } }))
|
||||
);
|
||||
|
||||
const { jobs, count } = await queue.getJobs({ page: 1, rowsPerPage: 10 });
|
||||
expect(count).toBe(25);
|
||||
expect(jobs).toHaveLength(10);
|
||||
expect(jobs[0].monitorId).toBe("m10");
|
||||
});
|
||||
|
||||
it("returns all jobs when rowsPerPage is omitted", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue(
|
||||
Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } }))
|
||||
);
|
||||
|
||||
const { jobs, count } = await queue.getJobs({});
|
||||
expect(count).toBe(25);
|
||||
expect(jobs).toHaveLength(25);
|
||||
});
|
||||
});
|
||||
|
||||
// ── flushQueues ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("flushQueues", () => {
|
||||
it("stops, flushes, and reinitializes", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const result = await queue.flushQueues();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
expect(scheduler.flushJobs).toHaveBeenCalled();
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false when any step fails", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.flushJobs as jest.Mock).mockResolvedValue(false);
|
||||
const result = await queue.flushQueues();
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── static create ────────────────────────────────────────────────────────
|
||||
|
||||
describe("static create", () => {
|
||||
it("creates a SuperSimpleQueue instance with a Scheduler and calls init", async () => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
|
||||
const instance = await SuperSimpleQueue.create(logger as any, helper as any, monitorsRepository as any);
|
||||
|
||||
expect(MockScheduler).toHaveBeenCalled();
|
||||
expect(instance).toBeInstanceOf(SuperSimpleQueue);
|
||||
// init was called, which calls scheduler.start
|
||||
expect(mockSchedulerInstance.start).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
+8
@@ -17,6 +17,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
findAllMonitorIds: jest.fn().mockResolvedValue(["m1"]),
|
||||
deleteByTeamIdsNotIn: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
const jobsRepository = {
|
||||
deleteByMonitorIdsNotIn: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
const teamsRepository = {
|
||||
findAllTeamIds: jest.fn().mockResolvedValue(["team"]),
|
||||
};
|
||||
@@ -48,6 +51,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
checkService,
|
||||
settingsService,
|
||||
monitorsRepository,
|
||||
jobsRepository,
|
||||
teamsRepository,
|
||||
monitorStatsRepository,
|
||||
checksRepository,
|
||||
@@ -64,6 +68,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
defaults.checkService as any,
|
||||
defaults.settingsService as any,
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.jobsRepository as any,
|
||||
defaults.teamsRepository as any,
|
||||
defaults.monitorStatsRepository as any,
|
||||
defaults.checksRepository as any,
|
||||
@@ -170,6 +175,7 @@ describe("WorkerHelper", () => {
|
||||
(defaults.checksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(4);
|
||||
(defaults.incidentsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(1);
|
||||
(defaults.geoChecksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(5);
|
||||
(defaults.jobsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(6);
|
||||
|
||||
const job = helper.getCleanupOrphanedJob();
|
||||
await job();
|
||||
@@ -181,11 +187,13 @@ describe("WorkerHelper", () => {
|
||||
expect(defaults.checksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.incidentsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.geoChecksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.jobsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("2 orphaned monitors") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("3 orphaned monitor stats") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("4 orphaned checks") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 orphaned incidents") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("5 orphaned geo checks") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("6 orphaned jobs") }));
|
||||
});
|
||||
|
||||
it("skips info logs when no orphaned data is found", async () => {
|
||||
Reference in new issue
Block a user