Merge pull request #3728 from bluewave-labs/feat/db-queue

feat: db queue
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-06-17 09:48:40 -07:00
commit 8277c4ed13
42 files changed
+3499 -3514

No files matched your search

@@ -272,5 +272,6 @@ export const monitorResponseSchema = z
userId: z.string(),
createdAt: z.string(),
updatedAt: z.string(),
lastEvaluatedAt: z.number(),
})
.passthrough();
+2 -2
View File
@@ -1,5 +1,5 @@
import { z } from "zod";
import { DbTypes, QueueTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
import { booleanCoercion } from "@/api/validation/shared.js";
const envSchema = z.object({
@@ -12,8 +12,8 @@ const envSchema = z.object({
DB_CONNECTION_STRING: z.string().min(1, "Database connection string is required"),
DB_TYPE: z.enum(DbTypes).default("mongodb"),
QUEUE_TYPE: z.enum(QueueTypes).default("superSimpleQueue"),
QUEUE_MODE: z.enum(QueueModes).default("primary"),
QUEUE_PRIMARY_PROCESSES: booleanCoercion.default(true),
// JWT Authentication
JWT_SECRET: z.string().min(1, "JWT_SECRET is required"),
+32 -86
View File
@@ -1,4 +1,6 @@
import { Mongoose } from "mongoose";
import { hostname } from "node:os";
import { randomUUID } from "node:crypto";
import MongoDB from "../db/db.mongo.js";
import { IDb } from "@/db/db.interface.js";
import { EnvConfig, ISettingsService, SettingsService } from "@/domain/app-settings/app-settings.service.js";
@@ -30,8 +32,6 @@ import { EmailService, IEmailService } from "@/service/emailService.js";
import { GlobalPingService } from "@/service/globalPingService.js";
import { WorkerHelper } from "@/worker/worker.helper.js";
import { IWorker } from "@/worker/worker.interface.js";
import { LessSimpleWorker } from "@/worker/worker.less-simple.js";
import { SuperSimpleQueue } from "@/worker/worker.super-simple.js";
import { INetworkService, NetworkService } from "@/service/networkService.js";
import { IStatusService, StatusService } from "@/service/statusService.js";
import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
@@ -89,7 +89,6 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js";
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js";
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js";
import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js";
import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js";
@@ -101,12 +100,15 @@ import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
import { ILogger } from "@/utils/logger.js";
import { AppError } from "@/utils/AppError.js";
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
import { CheckProducer } from "@/worker/worker.check-producer.js";
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js";
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
export type InitializedServices = {
settingsService: ISettingsService;
@@ -151,12 +153,10 @@ export const initializeServices = async ({
logger,
envSettings,
settingsService,
queueMode = "primary",
}: {
logger: ILogger;
envSettings: EnvConfig;
settingsService: ISettingsService;
queueMode?: QueueMode;
}): Promise<InitializedServices> => {
// Create DB
@@ -164,58 +164,6 @@ export const initializeServices = async ({
db = new MongoDB(logger, envSettings);
await db.connect();
// ** NOTE **
// const dbType = envSettings.dbType;
// DB type has been fixed to MongoDB for now
// if (dbType === "mongodb") {
// db = new MongoDB(logger, envSettings);
// }
// if (!db) {
// throw new AppError({ message: "Unsupported database type", status: 500 });
// }
// await db.connect();
// Repositories
// ** NOTE **
// DB Type fixed to MongoDB for now
// let monitorsRepository: IMonitorsRepository;
// let checksRepository: IChecksRepository;
// let geoChecksRepository: IGeoChecksRepository;
// let monitorStatsRepository: IMonitorStatsRepository;
// let statusPagesRepository: IStatusPagesRepository;
// let usersRepository: IUsersRepository;
// let invitesRepository: IInvitesRepository;
// let recoveryTokensRepository: IRecoveryTokensRepository;
// let settingsRepository: ISettingsRepository;
// let notificationsRepository: INotificationsRepository;
// let tagsRepository: ITagsRepository;
// let incidentsRepository: IIncidentsRepository;
// let teamsRepository: ITeamsRepository;
// let maintenanceWindowsRepository: IMaintenanceWindowsRepository;
// if (dbType === "mongodb") {
// monitorsRepository = new MongoMonitorsRepository();
// checksRepository = new MongoChecksRepository(logger);
// geoChecksRepository = new MongoGeoChecksRepository(logger);
// monitorStatsRepository = new MongoMonitorStatsRepository();
// statusPagesRepository = new MongoStatusPagesRepository();
// usersRepository = new MongoUsersRepository();
// invitesRepository = new MongoInvitesRepository();
// recoveryTokensRepository = new MongoRecoveryTokensRepository();
// settingsRepository = new MongoSettingsRepository();
// notificationsRepository = new MongoNotificationsRepository();
// tagsRepository = new MongoTagsRepository();
// incidentsRepository = new MongoIncidentsRepository();
// teamsRepository = new MongoTeamsRepository();
// maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
// } else {
// throw new AppError({ message: "Unsupported database type", status: 500 });
// }
const monitorsRepository = new MongoMonitorsRepository();
const checksRepository = new MongoChecksRepository(logger);
const geoChecksRepository = new MongoGeoChecksRepository(logger);
@@ -230,6 +178,9 @@ export const initializeServices = async ({
const incidentsRepository = new MongoIncidentsRepository();
const teamsRepository = new MongoTeamsRepository();
const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
// One identity per process
const workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
const jobsRepository = new MongoJobsRepository(workerId);
const queueWorkersRepository = new MongoQueueWorkersRepository();
// Inject settings repository into settings service (now that DB is connected)
@@ -320,17 +271,12 @@ export const initializeServices = async ({
const incidentReactor = new IncidentReactor(incidentService);
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
// Check producer/evaluator
const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger);
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
// pipelines
const checkPipeline = new CheckPipeline(
monitorsRepository,
maintenanceWindowsRepository,
checkService,
networkService,
bufferService,
monitorStatusPolicy,
statusService,
logger
);
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
@@ -339,6 +285,7 @@ export const initializeServices = async ({
checkService,
settingsService,
monitorsRepository,
jobsRepository,
teamsRepository,
monitorStatsRepository,
checksRepository,
@@ -349,23 +296,22 @@ export const initializeServices = async ({
geoCheckPipeline
);
if (queueMode === "worker" && envSettings.queueType !== "lessSimpleQueue") {
throw new AppError({
message: `Worker mode requires QUEUE_TYPE="lessSimpleQueue" got ${envSettings.queueType}`,
status: 500,
service: "config services",
method: "initializeServices",
});
}
let worker: IWorker;
switch (envSettings.queueType) {
case "lessSimpleQueue":
worker = await LessSimpleWorker.create(logger, workerHelper, monitorsRepository, queueWorkersRepository, envSettings, queueMode);
break;
default:
worker = await SuperSimpleQueue.create(logger, workerHelper, monitorsRepository);
}
const worker = await DBQueueWorker.create(
logger,
jobsRepository,
monitorsRepository,
checksRepository,
checkService,
checkProducer,
checkEvaluator,
geoCheckPipeline,
reactorDispatcher,
workerHelper,
queueWorkersRepository,
envSettings.queueMode,
envSettings.queuePrimaryProcesses,
workerId
);
// Business services
const userService = new UserService({
@@ -0,0 +1,41 @@
import mongoose from "mongoose";
import { MonitorModel } from "../../domain/monitors/monitor.model.js";
import { logger } from "@/utils/logger.js";
const JOBS_COLLECTION = "jobs";
/**
* Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed.
*
* Stamp the upgrade moment, not 0. If set to 0, all checks will be replayed
*/
export async function backfillMonitorLastEvaluatedAt(): Promise<void> {
const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt";
try {
// Drop the legacy jobs collection first. The old super-simple-scheduler persisted jobs to the
// same "jobs" collection the new db-queue uses, so any leftover rows carry an incompatible schema.
// Migrations run before syncIndexes(), so the new Job indexes are rebuilt right after this drop.
const db = mongoose.connection.db;
if (db) {
const existing = await db.listCollections({ name: JOBS_COLLECTION }).toArray();
if (existing.length > 0) {
await db.collection(JOBS_COLLECTION).drop();
logger.info({ service: SERVICE_NAME, message: `Dropped legacy "${JOBS_COLLECTION}" collection` });
}
}
logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" });
const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: Date.now() } });
logger.info({
service: SERVICE_NAME,
message: `Backfilled lastEvaluatedAt on ${result.modifiedCount} monitors`,
});
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.error({ service: SERVICE_NAME, message: `Error backfilling lastEvaluatedAt: ${errorMessage}` });
throw error;
}
}
+2
View File
@@ -6,6 +6,7 @@ import MigrationModel from "../../domain/migrations/migration.model.js";
import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js";
import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js";
import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js";
import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js";
import type { ILogger } from "@/utils/logger.js";
type MigrationEntry = {
@@ -21,6 +22,7 @@ const migrations: MigrationEntry[] = [
{ name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray },
{ name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex },
{ name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray },
{ name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt },
];
const runMigrations = async (logger?: ILogger) => {
@@ -1,5 +1,5 @@
import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js";
import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { AppError } from "@/utils/AppError.js";
import { ValidatedEnv } from "@/config/envValidation.js";
import type { StringValue } from "ms";
@@ -13,8 +13,8 @@ export type EnvConfig = {
clientHost: string;
dbConnectionString: string;
dbType: DbType;
queueType: QueueType;
queueMode: QueueMode;
queuePrimaryProcesses: boolean;
statusPageThemesEnabled: boolean;
};
@@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService {
clientHost: env.CLIENT_HOST,
dbConnectionString: env.DB_CONNECTION_STRING,
dbType: env.DB_TYPE,
queueType: env.QUEUE_TYPE,
queueMode: env.QUEUE_MODE,
queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES,
statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED,
};
}
@@ -1,9 +1,6 @@
export const DbTypes = ["mongodb"] as const;
export type DbType = (typeof DbTypes)[number];
export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const;
export type QueueType = (typeof QueueTypes)[number];
export const QueueModes = ["primary", "worker"] as const;
export type QueueMode = (typeof QueueModes)[number];
@@ -42,6 +42,7 @@ export interface IChecksRepository {
options?: { type?: MonitorType }
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult>;
findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
// update
//delete
deleteByMonitorId(monitorId: string): Promise<number>;
@@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository {
};
};
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
const docs = await CheckModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
createdAt: { $gt: new Date(since) },
})
.sort({ createdAt: 1 })
.lean<CheckDocument[]>();
return docs.map(this.toEntity);
};
deleteByMonitorId = async (monitorId: string): Promise<number> => {
const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
return result.deletedCount;
+27 -3
View File
@@ -11,7 +11,9 @@ const SERVICE_NAME = "checkService";
export interface ICheckService {
createChecks(checks: Check[]): Promise<Check[]>;
buildCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toLastEvaluatedAt(check: Check): number;
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
getChecksByMonitor(params: {
monitorId: string;
teamId: string;
@@ -56,7 +58,11 @@ export class CheckService implements ICheckService {
return this.checksRepository.createChecks(checks);
};
buildCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
toLastEvaluatedAt = (check: Check) => {
return new Date(check.createdAt).getTime();
};
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
const now = new Date().toISOString();
@@ -82,7 +88,7 @@ export class CheckService implements ICheckService {
this.logger.warn({
message: "Failed to build check",
service: SERVICE_NAME,
method: "buildCheck",
method: "toCheck",
});
return undefined;
}
@@ -131,6 +137,24 @@ export class CheckService implements ICheckService {
return check;
};
toStatusResponse = (check: Check) => {
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
monitorId: check.metadata.monitorId,
teamId: check.metadata.teamId,
type: check.metadata.type,
status: check.status,
code: check.statusCode,
message: check.message,
responseTime: check.responseTime,
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
payload:
check.metadata.type === "hardware"
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
: undefined,
};
return statusResponse;
};
getChecksByMonitor = async ({
monitorId,
teamId,
+79
View File
@@ -0,0 +1,79 @@
import { Schema, model } from "mongoose";
import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js";
export interface JobDocument extends Omit<Job, "id"> {
_id: string;
}
const JobSchema = new Schema<JobDocument>(
{
_id: {
type: String,
required: true,
},
type: {
type: String,
enum: JOB_TYPES,
required: true,
},
// monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate).
refId: {
type: String,
default: null,
},
// paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob)
isActive: {
type: Boolean,
default: true,
},
// Scheduling — epoch ms, so the claim/$min queries operate on plain numbers
nextScheduledAt: {
type: Number,
required: true,
},
intervalMs: {
type: Number,
default: null,
},
// Lock / lease
lockedBy: {
type: String,
default: null,
},
lockedUntil: {
type: Number,
default: null,
},
// Observability
runCount: {
type: Number,
default: 0,
},
failCount: {
type: Number,
default: 0,
},
lastFinishedAt: {
type: Number,
default: null,
},
lastFailReason: {
type: String,
default: null,
},
},
{ timestamps: true }
);
// Used for scanning for claims
JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 });
// for findById, upsertEvaluate, deleteById
JobSchema.index({ refId: 1, type: 1 });
const JobModel = model<JobDocument>("Job", JobSchema);
export { JobModel };
export default JobModel;
@@ -0,0 +1,72 @@
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
export type JobPageQuery = {
page?: number;
rowsPerPage?: number;
};
export type JobPage = {
jobs: Job[];
count: number;
};
export interface IJobsRepository {
// ********************
// Claims
// ********************
// Atomically claim a job
claimDue(type: JobType, now: number): Promise<Job | null>;
// locks up to `limit` due rows in one batch
claimDueBatch(type: JobType, limit: number, now: number): Promise<Job[]>;
// Extends the locks we still hold. Returns how many were renewed
renewLocks(ids: string[], now: number): Promise<number>;
// Completes a repeating job
recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise<boolean>;
// Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry
recordFailure(id: string, error: unknown, now: number): Promise<boolean>;
recordOneShot(id: string, now: number): Promise<boolean>;
// ********************
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
// ********************
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
// ********************
// Job CRUD
// ********************
// Create jobs. Takes a JobSeed
upsertJob(job: JobSeed): Promise<boolean>;
// Creates cleanup jobs, these are executed immediately so need their own method
upsertCleanupJob(job: JobSeed): Promise<boolean>;
// Pause/Resume
setActiveById(refId: string, isActive: boolean): Promise<boolean>;
// Edit job schedule
updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
// Delete, drop all rows
deleteById(refId: string): Promise<boolean>;
// Delete a single job of type
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
// Delete jobs that reference monitors that no longer exist
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
// Get job
findById(refId: string): Promise<Job[]>;
// ********************
// Observability
// ********************
findPage(pagination: JobPageQuery): Promise<JobPage>;
findAll(): Promise<Job[]>;
}
@@ -0,0 +1,287 @@
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
class MongoJobsRepository implements IJobsRepository {
constructor(private readonly workerId: string) {}
private toEntity(doc: JobDocument): Job {
return {
id: doc._id,
type: doc.type,
refId: doc.refId,
isActive: doc.isActive,
nextScheduledAt: doc.nextScheduledAt,
intervalMs: doc.intervalMs,
lockedBy: doc.lockedBy,
lockedUntil: doc.lockedUntil,
runCount: doc.runCount,
failCount: doc.failCount,
lastFinishedAt: doc.lastFinishedAt,
lastFailReason: doc.lastFailReason,
};
}
claimDue = async (type: JobType, now: number) => {
const doc = await JobModel.findOneAndUpdate(
{
type,
isActive: true,
nextScheduledAt: { $lte: now }, // Find due jobs
$or: [
{ lockedUntil: null }, // Not locked
{ lockedUntil: { $lt: now } }, // Free or expired lock
],
},
{
$set: {
lockedBy: this.workerId,
lockedUntil: now + LOCK_MS,
},
},
{
new: true,
sort: { nextScheduledAt: 1 }, // Get the oldest due job
}
).lean<JobDocument>();
return doc ? this.toEntity(doc) : null;
};
claimDueBatch = async (type: JobType, limit: number, now: number) => {
if (limit <= 0) return [];
const dueAndFree = {
type,
isActive: true,
nextScheduledAt: { $lte: now }, // due
$or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock
};
// 1. Pick the oldest-due candidates
const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>();
if (candidates.length === 0) return [];
const ids = candidates.map((d) => d._id);
// 2. lock them. Re-check free/expired so we never steal another worker's live lock
// only the rows this updateMany actually flips end up with claimed
const lockedUntil = now + LOCK_MS;
await JobModel.updateMany(
{ _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] },
{ $set: { lockedBy: this.workerId, lockedUntil } }
);
// 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time
const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean<JobDocument[]>();
return docs.map(this.toEntity);
};
renewLocks = async (ids: string[], now: number) => {
if (ids.length === 0) return 0;
const res = await JobModel.updateMany(
{ _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed
{ $set: { lockedUntil: now + LOCK_MS } }
);
return res.modifiedCount;
};
recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => {
// Fixed rate job scheduling
let nextScheduledAt = scheduledRunAt + intervalMs;
// If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd
if (nextScheduledAt <= now) {
nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs);
}
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
},
$inc: {
runCount: 1, // Increment run count
},
}
);
return res.modifiedCount === 1;
};
recordFailure = async (id: string, error: unknown, now: number) => {
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt: now + BACKOFF_MS,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFailReason: error instanceof Error ? error.message : String(error),
},
$inc: {
failCount: 1, // Increment fail count
},
}
);
return res.modifiedCount === 1;
};
recordOneShot = async (id: string, now: number) => {
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt: PARKED,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
},
$inc: {
runCount: 1, // Increment run count
},
}
);
return res.modifiedCount === 1;
};
upsertEvaluate = async (monitorId: string, now: number) => {
const filter = { type: "evaluate", refId: monitorId };
const update = {
$setOnInsert: {
_id: jobId("evaluate", monitorId),
// refId and Type inferred from filter
isActive: true,
intervalMs: null,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
$min: {
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
},
};
try {
const res = await JobModel.updateOne(filter, update, { upsert: true });
return res.acknowledged;
} catch (error) {
// Two callers can race the first insert for the same monitor, so the
// loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min.
if (error && typeof error === "object" && (error as { code?: number }).code === 11000) {
const res = await JobModel.updateOne(filter, update, { upsert: true });
return res.acknowledged;
}
throw error;
}
};
upsertJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
{
$set: {
// These fields are owned by the monitor, always safe to set
type: job.type,
refId: job.refId,
isActive: job.isActive,
intervalMs: job.intervalMs,
},
$setOnInsert: {
// These fields are scheduling related, only set on insert so they aren't clobbered
nextScheduledAt: job.nextScheduledAt,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
},
{ upsert: true }
);
return res.acknowledged;
};
upsertCleanupJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
{
$set: {
type: job.type,
refId: job.refId,
isActive: job.isActive,
intervalMs: job.intervalMs,
nextScheduledAt: job.nextScheduledAt,
},
$setOnInsert: {
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
},
{ upsert: true }
);
return res.acknowledged;
};
setActiveById = async (refId: string, isActive: boolean) => {
const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together
return res.modifiedCount > 0;
};
updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => {
const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } });
return res.modifiedCount === 1;
};
deleteById = async (refId: string) => {
const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
return res.deletedCount > 0;
};
deleteByIdAndType = async (refId: string, type: JobType) => {
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
return res.deletedCount > 0;
};
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
return res.deletedCount ?? 0;
};
findById = async (refId: string) => {
const docs = await JobModel.find({ refId }).lean<JobDocument[]>();
return docs.map(this.toEntity);
};
findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => {
const count = await JobModel.countDocuments();
const query = JobModel.find().sort({ nextScheduledAt: 1 });
if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage);
const docs = await query.lean<JobDocument[]>();
return { jobs: docs.map(this.toEntity), count };
};
findAll = async () => {
const docs = await JobModel.find().lean<JobDocument[]>();
return docs.map(this.toEntity);
};
}
export default MongoJobsRepository;
+32
View File
@@ -0,0 +1,32 @@
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
export type JobType = (typeof JOB_TYPES)[number];
export const LOCK_MS = 60_000;
export const BACKOFF_MS = 5_000;
export const PARKED = Number.MAX_SAFE_INTEGER;
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
export type Job = {
id: string;
type: JobType;
refId: string | null;
isActive: boolean;
// Scheduling fields
nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now
intervalMs: number | null; // repeat period in ms; null = one time job
// Lock fields
lockedBy: string | null; // workerId holding the lease
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
// observability (replaces what getMetrics/getJobs)
runCount: number;
failCount: number;
lastFinishedAt: number | null;
lastFailReason: string | null;
};
// These are the only values to set when creating a job. Repo owns the rest
export type JobSeed = Pick<Job, "id" | "type" | "refId" | "isActive" | "intervalMs" | "nextScheduledAt">;
@@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase {
teamId: Types.ObjectId;
createdAt: Date;
updatedAt: Date;
lastEvaluatedAt: number; // epoch ms
}
const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
@@ -377,6 +378,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
type: [checkSnapshotSchema],
default: [],
},
lastEvaluatedAt: {
type: Number,
default: 0,
},
},
{
timestamps: true,
@@ -571,6 +571,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
dnsRecordType: doc.dnsRecordType ?? undefined,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
lastEvaluatedAt: doc.lastEvaluatedAt,
};
};
@@ -636,6 +637,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
dnsRecordType: doc.dnsRecordType ?? undefined,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
lastEvaluatedAt: doc.lastEvaluatedAt,
};
};
@@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService {
recentChecks: [],
createdAt: "",
updatedAt: "",
lastEvaluatedAt: 0,
}));
const createdMonitors = await this.createMonitors(cleanedMonitors);
@@ -88,6 +88,7 @@ export interface Monitor {
recentChecks: CheckSnapshot[];
createdAt: string;
updatedAt: string;
lastEvaluatedAt: number; // epoch ms
}
export interface MonitorsSummary {
@@ -2,11 +2,15 @@ import { Schema, model } from "mongoose";
import { QueueModes } from "@/domain/app-settings/app-settings.type.js";
import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js";
// Single source of truth for worker liveness: the TTL that GCs stale records and
// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it.
export const WORKER_TTL_SECONDS = 180;
const QueueWorkerSchema = new Schema<QueueWorkerDocument>(
{
_id: { type: String, required: true },
mode: { type: String, enum: QueueModes, required: true },
lastSeenAt: { type: Date, default: Date.now, expires: 30 },
lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS },
},
{ timestamps: true }
);
+1 -1
View File
@@ -24,7 +24,7 @@ const startApp = async () => {
logger = new Logger({ envSettings });
// Initialize services (connects DB, creates repositories, injects settingsRepository)
const services = await initializeServices({ logger, envSettings, settingsService, queueMode: envSettings.queueMode });
const services = await initializeServices({ logger, envSettings, settingsService });
// If this is a worker instance, we're done. No need for express
if (envSettings.queueMode === "worker") {
@@ -0,0 +1,33 @@
import { MonitorStatusResponse } from "@/types/network.js";
import { MonitorEvaluation } from "@/worker/worker.interface.js";
import { Check } from "@/domain/checks/check.type.js";
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IStatusService } from "@/service/statusService.js";
export interface ICheckEvaluator {
evaluate(status: MonitorStatusResponse, check: Check): Promise<MonitorEvaluation>;
}
export class CheckEvaluator implements ICheckEvaluator {
constructor(
private statusService: IStatusService,
private monitorStatusPolicy: IMonitorStatusPolicy
) {}
evaluate = async (status: MonitorStatusResponse, check: Check) => {
// ****************************
// Step 3: Evaluate and return result to reactors
// ****************************
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
// Step 5. Get decisions and create an evaluation obj
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
const evaluation: MonitorEvaluation = {
monitor: statusChangeResult.monitor,
status,
check,
statusChange: statusChangeResult,
decision,
};
return evaluation;
};
}
+8 -78
View File
@@ -1,17 +1,14 @@
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { MonitorEvaluation } from "@/worker/worker.interface.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { ILogger } from "@/utils/logger.js";
import { INetworkService } from "@/service/networkService.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { IBufferService } from "@/service/bufferService.js";
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IStatusService } from "@/service/statusService.js";
import { AppError } from "@/utils/AppError.js";
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
import { ICheckProducer } from "@/worker/worker.check-producer.js";
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
const SERVICE_NAME = "CheckPipeline";
@@ -21,83 +18,16 @@ export interface ICheckPipeline {
export class CheckPipeline implements ICheckPipeline {
constructor(
private monitorsRepository: IMonitorsRepository,
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
private checkService: ICheckService,
private networkService: INetworkService,
private bufferService: IBufferService,
private monitorStatusPolicy: IMonitorStatusPolicy,
private statusService: IStatusService,
private logger: ILogger
private checkProducer: ICheckProducer,
private checkEvaluator: ICheckEvaluator
) {}
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
const now = new Date();
return maintenanceWindows.some((window) => isWindowActive(window, now));
}
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
const monitorId = monitor.id;
if (!monitorId) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getMonitorJob" });
const checkResult = await this.checkProducer.produce(monitor);
if (!checkResult) {
return null; // skipped due to maintenance / geo guards / other guards
}
// ****************************
// Step 1: Acquire
// ****************************
// Step 1a: Maintenance window gate - skip if in maintenance
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
if (maintenanceWindowActive) {
this.logger.debug({
message: `Monitor ${monitor.id} is in maintenance window`,
service: SERVICE_NAME,
method: "runCheckPipeline",
});
if (monitor.status !== "maintenance") {
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
}
return null;
}
// Step 1b: Acquire status
const status = await this.networkService.requestStatus(monitor);
if (!status) {
throw new Error("No network response");
}
// ****************************
// Step 2: Record
// ****************************
// Step 2a: Create & record a check, return null if fail
const check = this.checkService.buildCheck(status);
if (!check) {
this.logger.warn({
message: `No check could be built for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "runCHeckPipeline",
details: { code: status.code, message: status.message },
});
return null;
}
// Step 2b: Add to buffer
this.bufferService.addToBuffer(check);
// ****************************
// Step 3: Evaluate and return result to reactors
// ****************************
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
// Step 5. Get decisions and create an evaluation obj
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
const evaluation: MonitorEvaluation = {
monitor: statusChangeResult.monitor,
status,
check,
statusChange: statusChangeResult,
decision,
};
const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check);
return evaluation;
};
}
@@ -0,0 +1,82 @@
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { Check } from "@/domain/checks/check.type.js";
import { MonitorStatusResponse } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import { IBufferService } from "@/service/bufferService.js";
import { INetworkService } from "@/service/networkService.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
export interface ICheckProducer {
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
}
const SERVICE_NAME = "CheckProducer";
export class CheckProducer implements ICheckProducer {
constructor(
private monitorsRepository: IMonitorsRepository,
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
private checkService: ICheckService,
private networkService: INetworkService,
private bufferService: IBufferService,
private logger: ILogger
) {}
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
const now = new Date();
return windows.some((w) => isWindowActive(w, now));
}
produce = async (monitor: Monitor) => {
if (!monitor.id) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
}
// ****************************
// Step 1: Acquire
// ****************************
// Step 1a: Maintenance window gate - skip if in maintenance
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
if (maintenanceWindowActive) {
this.logger.debug({
message: `Monitor ${monitor.id} is in maintenance window`,
service: SERVICE_NAME,
method: "produce",
});
if (monitor.status !== "maintenance") {
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
}
return null;
}
// Step 1b: Acquire status
const status = await this.networkService.requestStatus(monitor);
if (!status) {
throw new Error("No network response");
}
// ****************************
// Step 2: Record
// ****************************
// Step 2a: Create & record a check, return null if fail
const check = this.checkService.toCheck(status);
if (!check) {
this.logger.warn({
message: `No check could be built for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "produce",
details: { code: status.code, message: status.message },
});
return null;
}
// Step 2b: Add to buffer
this.bufferService.addToBuffer(check);
return { status, check };
};
}
+389
View File
@@ -0,0 +1,389 @@
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js";
import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js";
import { ILogger } from "@/utils/logger.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
import { ICheckService } from "@/domain/checks/check.service.js";
import { ICheckProducer } from "@/worker/worker.check-producer.js";
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
const SERVICE_NAME = "JobQueue";
const POLL_MS = 250;
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
const CONCURRENCY: Record<JobType, number> = {
check: 500,
"geo-check": 20,
evaluate: 20,
"cleanup-orphaned": 1,
"cleanup-retention": 1,
};
export class DBQueueWorker implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private stopped = false;
private timers = new Map<JobType | "heartbeat", NodeJS.Timeout>();
private inFlight: Record<JobType, number> = {
check: 0,
"geo-check": 0,
evaluate: 0,
"cleanup-orphaned": 0,
"cleanup-retention": 0,
};
constructor(
private logger: ILogger,
private jobsRepository: IJobsRepository,
private monitorsRepository: IMonitorsRepository,
private checksRepository: IChecksRepository,
private checkService: ICheckService, // toStatusResponse + toLastEvaluatedAt
private checkProducer: ICheckProducer,
private checkEvaluator: ICheckEvaluator,
private geoCheckPipeline: ICheckPipeline,
private dispatcher: IReactorDispatcher,
private helper: IWorkerHelper,
private queueWorkersRepository: IQueueWorkersRepository,
private queueMode: QueueMode,
private queuePrimaryProcesses: boolean,
private readonly workerId: string // same id MongoJobsRepository stamps into lockedBy
) {}
get serviceName() {
return DBQueueWorker.SERVICE_NAME;
}
static async create(
logger: ILogger,
jobsRepository: IJobsRepository,
monitorsRepository: IMonitorsRepository,
checksRepository: IChecksRepository,
checkService: ICheckService,
checkProducer: ICheckProducer,
checkEvaluator: ICheckEvaluator,
geoCheckPipeline: ICheckPipeline,
dispatcher: IReactorDispatcher,
helper: IWorkerHelper,
queueWorkersRepository: IQueueWorkersRepository,
queueMode: QueueMode,
queuePrimaryProcesses: boolean,
workerId: string
): Promise<DBQueueWorker> {
const instance = new DBQueueWorker(
logger,
jobsRepository,
monitorsRepository,
checksRepository,
checkService,
checkProducer,
checkEvaluator,
geoCheckPipeline,
dispatcher,
helper,
queueWorkersRepository,
queueMode,
queuePrimaryProcesses,
workerId
);
await instance.init();
return instance;
}
// Helpers, builds job seeds
private toCheckJob = (monitor: Monitor, now: number): JobSeed => ({
id: jobId("check", monitor.id),
type: "check",
refId: monitor.id,
isActive: monitor.isActive,
nextScheduledAt: now + Math.floor(Math.random() * monitor.interval), // Avoid herd
intervalMs: monitor.interval,
});
private toGeoCheckJob = (monitor: Monitor, now: number): JobSeed => ({
...this.toCheckJob(monitor, now),
id: jobId("geo-check", monitor.id),
type: "geo-check",
intervalMs: monitor.geoCheckInterval!,
});
private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({
id: jobId(type, null),
type,
refId: null,
isActive: true,
nextScheduledAt: now,
intervalMs: CLEANUP_INTERVAL_MS,
});
private toSummary = (job: Job): WorkerJobSummary => ({
monitorId: job.refId ?? job.id,
monitorUrl: null, // Jobs don't have URL
monitorType: job.type,
monitorInterval: job.intervalMs,
monitorGeoInterval: null,
monitorActive: job.isActive,
active: job.isActive,
lockedBy: job.lockedBy,
lockedUntil: job.lockedUntil,
lockedAt: null, // no lockedAt
runCount: job.runCount,
failCount: job.failCount,
failReason: job.lastFailReason,
lastRunAt: null, // not tracked
lastFinishedAt: job.lastFinishedAt,
lastRunTook: null, // not tracked
lastFailedAt: null, // not tracked
repeat: job.intervalMs,
});
// ********************
// Stage 1: This can eventually be offloaded to a separate service
// ********************
private runCheck = async (job: Job) => {
if (!job.refId) return;
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
if (!monitor) return;
const produced = await this.checkProducer.produce(monitor);
if (produced) {
await this.jobsRepository.upsertEvaluate(monitor.id, Date.now());
}
};
// ********************
// Stage 2: Evaluator loop stays here
// ********************
private runEvaluate = async (job: Job) => {
if (!job.refId) return;
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
if (!monitor) return;
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
for (const check of checks) {
const status = this.checkService.toStatusResponse(check);
const evaluation = await this.checkEvaluator.evaluate(status, check);
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
}
};
private runGeoCheck = async (job: Job) => {
const [monitor] = await this.monitorsRepository.findByIds([job.refId!]);
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
};
// Extend the lock while a job runs so a slow-but-alive job is never reclaimed.
// Returns false once we no longer hold the lock, so the renewal timer can stop.
private renewLock = async (job: Job): Promise<boolean> => {
try {
const held = await this.jobsRepository.renewLocks([job.id], Date.now());
if (held === 0) {
this.logger.warn({ message: `Lost lock on job ${job.id}, stopping renewal`, service: SERVICE_NAME, method: "renewLock" });
return false;
}
return true;
} catch (error: unknown) {
// Transient failure, keep renewing; a missed renewal is absorbed by the 1/3 margin
this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "renewLock" });
return true;
}
};
private runJob = async (job: Job) => {
const renewTimer = setInterval(async () => {
const renewLock = await this.renewLock(job);
if (!renewLock) clearInterval(renewTimer);
}, LOCK_RENEW_MS);
try {
switch (job.type) {
case "check":
await this.runCheck(job);
break;
case "evaluate":
await this.runEvaluate(job);
break;
case "geo-check":
await this.runGeoCheck(job);
break;
case "cleanup-orphaned":
await this.helper.getCleanupOrphanedJob()(); // Get job and execute
break;
case "cleanup-retention":
await this.helper.getCleanupRetentionJob()(); // Get job and execute
break;
}
if (job.intervalMs === null) {
// One-shot job
await this.jobsRepository.recordOneShot(job.id, Date.now());
} else {
await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now());
}
} catch (error: unknown) {
await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock
this.logger.warn({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: `runJob:${job.type}`,
});
} finally {
clearInterval(renewTimer);
}
};
private startLoop = (type: JobType) => {
const tick = async () => {
if (this.stopped) return;
try {
// Claim a batch sized to the free capacity
const capacity = CONCURRENCY[type] - this.inFlight[type];
const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now());
for (const job of jobs) {
this.inFlight[type]++; // stay within concurrency limits
this.runJob(job).finally(() => {
this.inFlight[type]--; // job done, free the slot
});
}
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: `loop:${type}`,
});
}
if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down
};
tick();
};
// Seed queue
private reconcile = async () => {
const now = Date.now();
const monitors = (await this.monitorsRepository.findAll()) ?? [];
for (const monitor of monitors) {
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
}
}
// Both cleanup jobs run immediately on every startup, then once a day
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now));
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now));
};
// Register worker
private heartbeat = async () => {
try {
await this.queueWorkersRepository.upsert(this.workerId, this.queueMode);
} catch (error: unknown) {
this.logger.warn({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: "heartbeat",
});
}
};
init = async () => {
this.stopped = false;
await this.heartbeat();
this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS));
if (this.queueMode === "primary") {
await this.reconcile();
}
if (this.queuePrimaryProcesses === true || this.queueMode === "worker") {
for (const type of Object.keys(this.inFlight) as JobType[]) {
this.startLoop(type);
}
}
return true;
};
addJob = async (_monitorId: string, monitor: Monitor) => {
const now = Date.now();
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
};
deleteJob = async (monitor: Monitor) => {
await this.jobsRepository.deleteById(monitor.id);
};
pauseJob = async (monitor: Monitor) => {
await this.jobsRepository.setActiveById(monitor.id, false);
};
resumeJob = async (monitor: Monitor) => {
await this.jobsRepository.setActiveById(monitor.id, true);
};
updateJob = async (monitor: Monitor) => {
await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval);
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now()));
} else {
await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check");
}
};
shutdown = async () => {
this.stopped = true;
// Stop all timers
for (const timer of this.timers.values()) {
clearTimeout(timer);
}
this.timers.clear(); // Clear the map out
// Remove workers from registry
await this.queueWorkersRepository.deleteById(this.workerId);
};
getMetrics = async (): Promise<WorkerMetrics> => {
const rows = await this.jobsRepository.findAll();
const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS);
const now = Date.now();
return rows.reduce<WorkerMetrics>(
(acc, job) => {
acc.jobs++;
acc.totalRuns += job.runCount;
acc.totalFailures += job.failCount;
if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++;
if (job.failCount > 0) {
acc.failingJobs++;
acc.jobsWithFailures.push({
monitorId: job.refId ?? job.id,
monitorUrl: null,
monitorType: job.type,
failedAt: job.lastFinishedAt,
failCount: job.failCount,
failReason: job.lastFailReason,
});
}
return acc;
},
{
jobs: 0,
activeJobs: 0,
failingJobs: 0,
jobsWithFailures: [],
totalRuns: 0,
totalFailures: 0,
workers,
}
);
};
getJobs = async (pagination: WorkerJobsPagination) => {
const { jobs, count } = await this.jobsRepository.findPage(pagination);
return { jobs: jobs.map(this.toSummary), count };
};
flushQueues = async () => {
await this.shutdown();
const ok = await this.init();
return { success: ok };
};
}
+14
View File
@@ -8,6 +8,7 @@ import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.i
import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
@@ -42,6 +43,7 @@ export class WorkerHelper implements IWorkerHelper {
private checkService: ICheckService;
private settingsService: ISettingsService;
private monitorsRepository: IMonitorsRepository;
private jobsRepository: IJobsRepository;
private teamsRepository: ITeamsRepository;
private monitorStatsRepository: IMonitorStatsRepository;
private checksRepository: IChecksRepository;
@@ -56,6 +58,7 @@ export class WorkerHelper implements IWorkerHelper {
checkService: ICheckService,
settingsService: ISettingsService,
monitorsRepository: IMonitorsRepository,
jobsRepository: IJobsRepository,
teamsRepository: ITeamsRepository,
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository,
@@ -69,6 +72,7 @@ export class WorkerHelper implements IWorkerHelper {
this.checkService = checkService;
this.settingsService = settingsService;
this.monitorsRepository = monitorsRepository;
this.jobsRepository = jobsRepository;
this.teamsRepository = teamsRepository;
this.monitorStatsRepository = monitorStatsRepository;
this.checksRepository = checksRepository;
@@ -190,6 +194,16 @@ export class WorkerHelper implements IWorkerHelper {
});
}
// Remove orphaned jobs
const deletedJobsCount = await this.jobsRepository.deleteByMonitorIdsNotIn(allMonitorIds);
if (deletedJobsCount > 0) {
this.logger.info({
message: `Deleted ${deletedJobsCount} orphaned jobs`,
service: SERVICE_NAME,
method: "getCleanupOrphanedJob",
});
}
this.logger.info({
message: "Cleanup of orphaned data completed",
service: SERVICE_NAME,
+1 -1
View File
@@ -1,8 +1,8 @@
import { Monitor } from "@/domain/monitors/monitor.types.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
import { Check } from "@/domain/checks/check.type.js";
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
import { MonitorActionDecision } from "@/worker/worker.helper.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
export type MonitorEvaluation = {
monitor: Monitor;
-427
View File
@@ -1,427 +0,0 @@
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { MongoStore, Scheduler, IJob, AddJobInput } from "less-simple-scheduler";
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { type WorkerMetrics, type WorkerJobsPagination, type WorkerJobsPage, IWorker } from "@/worker/worker.interface.js";
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
const SERVICE_NAME = "JobQueue";
// How long after its last heartbeat a worker is still considered alive. Must be longer than the scheduler's heartbeat
const WORKER_TTL_MS = 30_000;
// For discriminating job types
type MonitorJob = Omit<IJob, "data"> & { data: Monitor };
const MONITOR_TEMPLATES = new Set<string>(["monitor-job", "geo-check-job"]);
const isMonitorJob = (job: IJob): job is MonitorJob => MONITOR_TEMPLATES.has(job.template);
export class LessSimpleWorker implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private helper: IWorkerHelper;
private monitorsRepository: IMonitorsRepository;
private workersRepository: IQueueWorkersRepository;
private readonly scheduler: Scheduler;
private queueMode: QueueMode;
private heartbeatListener: ((workerId: string) => void) | null = null;
constructor(
logger: ILogger,
helper: IWorkerHelper,
monitorsRepository: IMonitorsRepository,
workersRepository: IQueueWorkersRepository,
scheduler: Scheduler,
queueMode: QueueMode
) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
this.workersRepository = workersRepository;
this.scheduler = scheduler;
this.queueMode = queueMode;
this.registerListeners();
}
get serviceName() {
return LessSimpleWorker.SERVICE_NAME;
}
private registerListeners = () => {
this.scheduler.on("scheduler:start", () => {
this.logger.info({
message: "Scheduler started",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:stop", () => {
this.logger.info({
message: "Scheduler stopped",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:error", (workerId, error) => {
this.logger.error({
message: `${workerId} error: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:locked", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} locked`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:abort", (workerId, job, reason) => {
this.logger.warn({
message: `${workerId} ${job.id} aborted: ${reason}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:attempt", (workerId, job, attempt) => {
this.logger.debug({
message: `${workerId} ${job.id} attempt ${attempt}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:complete", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} completed successfully`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:exhausted", (workerId, job, error) => {
this.logger.error({
message: `${workerId} ${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:fail", (workerId, job, error, attempt) => {
this.logger.warn({
message: `${workerId} ${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:start", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} started`,
service: SERVICE_NAME,
});
});
};
static async create(
logger: ILogger,
helper: IWorkerHelper,
monitorsRepository: IMonitorsRepository,
workersRepository: IQueueWorkersRepository,
envSettings: EnvConfig,
queueMode: QueueMode
) {
const store = new MongoStore({
url: envSettings.dbConnectionString ?? "mongodb://localhost:27017/uptime_db",
});
const scheduler = new Scheduler(store, {
concurrency: envSettings.queueMode === "primary" ? 0 : 50,
processEvery: 100,
});
const instance = new LessSimpleWorker(logger, helper, monitorsRepository, workersRepository, scheduler, queueMode);
await instance.init(queueMode);
return instance;
}
private startHeartbeat = (queueMode: QueueMode) => {
const beat = (
workerId: string // This is the heartbeat function that registers workers
) =>
this.workersRepository.upsert(workerId, queueMode).catch((error: unknown) => {
this.logger.warn({
message: `Worker heartbeat failed: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "startHeartbeat",
});
});
beat(this.scheduler.workerId); // Beat immediately to register
if (this.heartbeatListener) {
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); // remove old listeners
}
this.heartbeatListener = beat; /// Keep a reference to the listener
this.scheduler.on("scheduler:heartbeat", beat); // beat on every heartbeat
};
init = async (queueMode: QueueMode = "primary") => {
try {
await this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
// Register in the worker registry
this.startHeartbeat(queueMode);
// If this is a worker, return early - primary will populate queue
if (queueMode === "worker") {
this.logger.info({
message: `Worker queue: ${this.scheduler.workerId} initialized`,
service: SERVICE_NAME,
method: "init",
});
return true;
}
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
return true;
}
const jobInputs = monitors.flatMap((monitor) => this.buildJobInputs(monitor));
if (jobInputs.length > 0) {
await this.scheduler.addJobs(jobInputs);
}
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, upsert: true });
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000, upsert: true });
return true;
} catch (error: unknown) {
this.logger.error({
message: `Failed to initialize LessSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "init",
});
return false;
}
};
// Builds the scheduler inputs for a monitor: always a monitor-job, plus a
// geo-check job when the monitor supports and enables geo checks.
private buildJobInputs = (monitor: Monitor): AddJobInput[] => {
const inputs: AddJobInput[] = [
{
id: monitor.id,
template: "monitor-job",
repeat: monitor.interval,
active: monitor.isActive,
data: monitor,
upsert: true,
jitter: true,
},
];
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
inputs.push({
id: `${monitor.id}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
upsert: true,
jitter: true,
});
}
return inputs;
};
addJob = async (_monitorId: string, monitor: Monitor) => {
await this.scheduler.addJobs(this.buildJobInputs(monitor));
};
deleteJob = async (monitor: Monitor) => {
this.scheduler.removeJob(monitor.id);
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: Monitor) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to pause monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to pause geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor: Monitor) => {
const result = await this.scheduler.resumeJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to resume geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
private syncGeoJob = async (monitor: Monitor) => {
const geoJobId = `${monitor.id}-geo`;
const existingGeoJob = await this.scheduler.getJob(geoJobId);
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
return;
}
// If the job exists, update it
if (existingGeoJob) {
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
return;
}
// Otherwise, create it
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
upsert: true,
});
};
updateJob = async (monitor: Monitor) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
await this.syncGeoJob(monitor);
};
shutdown = async () => {
// Stop beating before we remove this worker, so a heartbeat firing between
// the remove and the scheduler stopping can't re-insert it.
if (this.heartbeatListener) {
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener);
this.heartbeatListener = null;
}
await this.workersRepository.deleteById(this.scheduler.workerId).catch((error: unknown) => {
this.logger.warn({
message: `Failed to deregister worker on shutdown: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "shutdown",
});
});
this.scheduler.stop();
};
getMetrics = async (): Promise<WorkerMetrics> => {
const stats = await this.scheduler.getStats();
const jobsWithFailures = stats.jobsWithFailures.map((failure) => {
// `data` is untyped at the store level: for monitor jobs it's the Monitor,
// for system jobs (e.g. cleanup) it has no url/type => null
const monitor = failure.data as Monitor | undefined;
return {
monitorId: failure.id,
monitorUrl: monitor?.url ?? null,
monitorType: monitor?.type ?? null,
failedAt: failure.failedAt,
failCount: failure.failCount,
failReason: failure.failReason,
};
});
const workers = await this.workersRepository.findRecent(WORKER_TTL_MS);
return {
jobs: stats.jobs,
activeJobs: stats.activeJobs,
failingJobs: stats.failingJobs,
totalRuns: stats.totalRuns,
totalFailures: stats.totalFailures,
jobsWithFailures,
workers,
};
};
getJobs = async ({ page = 0, rowsPerPage = 0 }: WorkerJobsPagination): Promise<WorkerJobsPage> => {
const listOptions = rowsPerPage > 0 ? { skip: Math.max(page, 0) * rowsPerPage, limit: rowsPerPage } : undefined;
const [jobs, count] = await Promise.all([this.scheduler.getJobs(listOptions), this.scheduler.countJobs()]);
const mapped = jobs.map((job) => {
const monitor = isMonitorJob(job) ? job.data : undefined;
return {
monitorId: job.id,
monitorUrl: monitor?.url || null,
monitorType: monitor?.type || null,
monitorInterval: monitor?.interval || null,
monitorGeoInterval: monitor?.geoCheckInterval || null,
monitorActive: monitor?.isActive ?? null,
active: job.active,
repeat: job.repeat ?? null,
lockedBy: job.lockedBy ?? null,
lockedUntil: job.lockedUntil ?? null,
lockedAt: job.lockedAt ?? null,
runCount: job.runCount ?? 0,
failCount: job.failCount ?? 0,
failReason: job.lastError ?? null,
lastRunAt: job.lastStartedAt ?? null,
lastFinishedAt: job.lastFinishedAt ?? null,
lastRunTook: job.lockedAt || !job.lastFinishedAt || !job.lastStartedAt ? null : job.lastFinishedAt - job.lastStartedAt,
lastFailedAt: job.lastFailedAt ?? null,
};
});
return { jobs: mapped, count };
};
flushQueues = async () => {
if (this.queueMode !== "primary") {
// Don't allow workers to flush, though this isn't currently reachable since workers have no API endpoints
this.logger.warn({ message: "Ignoring flush on non-primary worker", service: SERVICE_NAME, method: "flushQueues" });
return { success: false };
}
const flushRes = await this.scheduler.flushJobs(); // Less Simple Queue must flush before stopping as it needs DB
const stopRes = await this.scheduler.stop(); // Destructive, other workers will abort their jobs
const initRes = await this.init();
return {
success: Boolean(stopRes && flushRes && initRes),
};
};
}
-384
View File
@@ -1,384 +0,0 @@
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import Scheduler from "super-simple-scheduler";
import { Monitor, MonitorType, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { hostname } from "node:os";
import { randomUUID } from "node:crypto";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { IWorker, WorkerJobSummary } from "@/worker/worker.interface.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
const SERVICE_NAME = "JobQueue";
export class SuperSimpleQueue implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private helper: IWorkerHelper;
private workerId: string;
private monitorsRepository: IMonitorsRepository;
private readonly scheduler: Scheduler;
constructor(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
this.scheduler = scheduler;
this.registerListeners();
this.workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
}
get serviceName() {
return SuperSimpleQueue.SERVICE_NAME;
}
private registerListeners = () => {
this.scheduler.on("scheduler:start", () => {
this.logger.info({
message: "Scheduler started",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:stop", () => {
this.logger.info({
message: "Scheduler stopped",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:error", (error) => {
this.logger.error({
message: `Scheduler error: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:abort", (job, reason) => {
this.logger.warn({
message: `${job.id} aborted: ${reason}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:attempt", (job, attempt) => {
this.logger.debug({
message: `${job.id} attempt ${attempt}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:complete", (job) => {
this.logger.debug({
message: `${job.id} completed successfully`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:exhausted", (job, error) => {
this.logger.error({
message: `${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:fail", (job, error, attempt) => {
this.logger.warn({
message: `${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:start", (job) => {
this.logger.debug({
message: `${job.id} started`,
service: SERVICE_NAME,
});
});
};
static async create(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository) {
const scheduler = new Scheduler({
concurrency: 50,
// storeType: "mongo",
// storeType: "redis",
// dbUri: envSettings.dbConnectionString,
});
const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
await instance.init();
return instance;
}
init = async () => {
try {
this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
return true;
}
for (const monitor of monitors) {
const randomOffset = Math.floor(Math.random() * 100);
setTimeout(() => {
this.addJob(monitor.id, monitor);
}, randomOffset);
}
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 });
return true;
} catch (error: unknown) {
this.logger.error({
message: `Failed to initialize SuperSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "init",
});
return false;
}
};
addJob = async (monitorId: string, monitor: Monitor) => {
this.scheduler.addJob({
id: monitorId,
template: "monitor-job",
repeat: monitor.interval,
active: monitor.isActive,
data: monitor,
});
// Return early if we don't need geo checks
if (!supportsGeoCheck(monitor.type)) {
return;
}
// Add geo check job if enabled for HTTP monitors
if (monitor.geoCheckEnabled) {
this.scheduler.addJob({
id: `${monitorId}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
}
};
deleteJob = async (monitor: Monitor) => {
this.scheduler.removeJob(monitor.id);
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: Monitor) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to pause monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to pause geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor: Monitor) => {
const result = await this.scheduler.resumeJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to resume geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
private syncGeoJob = async (monitor: Monitor) => {
const geoJobId = `${monitor.id}-geo`;
const existingGeoJob = await this.scheduler.getJob(geoJobId);
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
return;
}
// If the job exists, update it
if (existingGeoJob) {
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
return;
}
// Otherwise, create it
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
};
updateJob = async (monitor: Monitor) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
await this.syncGeoJob(monitor);
};
shutdown = async () => {
this.scheduler.stop();
};
getMetrics = async () => {
const jobs = await this.scheduler.getJobs();
const metrics: {
jobs: number;
activeJobs: number;
failingJobs: number;
jobsWithFailures: Array<{
monitorId: string | number;
monitorUrl: string;
monitorType: MonitorType;
failedAt: number | null;
failCount: number;
failReason: string | null;
}>;
totalRuns: number;
totalFailures: number;
workers: Array<{
workerId: string;
mode: QueueMode;
lastSeenAt: number;
}>;
} = jobs.reduce(
(acc, job) => {
const runCount = job.runCount ?? 0;
const failCount = job.failCount ?? 0;
const lastFailedAt = job.lastFailedAt ?? 0;
const lastRunAt = job.lastRunAt ?? 0;
acc.totalRuns += runCount;
acc.totalFailures += failCount;
acc.jobs++;
if (failCount > 0 && lastFailedAt >= lastRunAt) {
acc.failingJobs++;
}
if (job.lockedAt) {
acc.activeJobs++;
}
if (failCount > 0) {
acc.jobsWithFailures.push({
monitorId: job.id,
monitorUrl: job.data?.url || null,
monitorType: job.data?.type || null,
failedAt: job.lastFailedAt ?? null,
failCount,
failReason: job.lastFailReason ?? null,
});
}
return acc;
},
{
jobs: 0,
activeJobs: 0,
failingJobs: 0,
jobsWithFailures: [] as Array<{
monitorId: string | number;
monitorUrl: string;
monitorType: MonitorType;
failedAt: number | null;
failCount: number;
failReason: string | null;
}>,
totalRuns: 0,
totalFailures: 0,
workers: [
{
workerId: this.workerId,
mode: "primary",
lastSeenAt: Date.now(),
},
],
}
);
return metrics;
};
getJobs = async ({ page = 0, rowsPerPage = 0 }: { page?: number; rowsPerPage?: number }): Promise<{ jobs: WorkerJobSummary[]; count: number }> => {
const jobs = await this.scheduler.getJobs();
const mapped = jobs.map((job) => {
return {
monitorId: job.id,
monitorUrl: job.data?.url || null,
monitorType: job.data?.type || null,
monitorInterval: job.data?.interval || null,
monitorGeoInterval: job.data?.geoCheckInterval || null,
monitorActive: job.data?.isActive ?? null,
active: job.active,
repeat: job.repeat ?? null,
lockedBy: job.lockedAt ? this.workerId : null,
lockedUntil: null,
lockedAt: job.lockedAt ?? null,
runCount: job.runCount ?? 0,
failCount: job.failCount ?? 0,
failReason: job.lastFailReason ?? null,
lastRunAt: job.lastRunAt ?? null,
lastFinishedAt: job.lastFinishedAt ?? null,
lastRunTook: job.lockedAt ? null : (job.lastFinishedAt ?? 0) - (job.lastRunAt ?? 0),
lastFailedAt: job.lastFailedAt ?? null,
};
});
const start = Math.max(page, 0) * rowsPerPage;
return {
jobs: rowsPerPage > 0 ? mapped.slice(start, start + rowsPerPage) : mapped,
count: mapped.length,
};
};
flushQueues = async () => {
const stopRes = await this.scheduler.stop();
const flushRes = await this.scheduler.flushJobs();
const initRes = await this.init();
return {
success: Boolean(stopRes && flushRes && initRes),
};
};
}