Merge pull request #3728 from bluewave-labs/feat/db-queue
feat: db queue
This commit is contained in:
42 files changed
+3499
-3514
No files matched your search
@@ -272,5 +272,6 @@ export const monitorResponseSchema = z
|
||||
userId: z.string(),
|
||||
createdAt: z.string(),
|
||||
updatedAt: z.string(),
|
||||
lastEvaluatedAt: z.number(),
|
||||
})
|
||||
.passthrough();
|
||||
@@ -1,5 +1,5 @@
|
||||
import { z } from "zod";
|
||||
import { DbTypes, QueueTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { booleanCoercion } from "@/api/validation/shared.js";
|
||||
|
||||
const envSchema = z.object({
|
||||
@@ -12,8 +12,8 @@ const envSchema = z.object({
|
||||
DB_CONNECTION_STRING: z.string().min(1, "Database connection string is required"),
|
||||
DB_TYPE: z.enum(DbTypes).default("mongodb"),
|
||||
|
||||
QUEUE_TYPE: z.enum(QueueTypes).default("superSimpleQueue"),
|
||||
QUEUE_MODE: z.enum(QueueModes).default("primary"),
|
||||
QUEUE_PRIMARY_PROCESSES: booleanCoercion.default(true),
|
||||
|
||||
// JWT Authentication
|
||||
JWT_SECRET: z.string().min(1, "JWT_SECRET is required"),
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Mongoose } from "mongoose";
|
||||
import { hostname } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import MongoDB from "../db/db.mongo.js";
|
||||
import { IDb } from "@/db/db.interface.js";
|
||||
import { EnvConfig, ISettingsService, SettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
@@ -30,8 +32,6 @@ import { EmailService, IEmailService } from "@/service/emailService.js";
|
||||
import { GlobalPingService } from "@/service/globalPingService.js";
|
||||
import { WorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IWorker } from "@/worker/worker.interface.js";
|
||||
import { LessSimpleWorker } from "@/worker/worker.less-simple.js";
|
||||
import { SuperSimpleQueue } from "@/worker/worker.super-simple.js";
|
||||
import { INetworkService, NetworkService } from "@/service/networkService.js";
|
||||
import { IStatusService, StatusService } from "@/service/statusService.js";
|
||||
import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
@@ -89,7 +89,6 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
|
||||
import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js";
|
||||
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
|
||||
import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js";
|
||||
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
|
||||
import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js";
|
||||
import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js";
|
||||
import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js";
|
||||
@@ -101,12 +100,15 @@ import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
|
||||
import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
|
||||
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
|
||||
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { CheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
|
||||
import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js";
|
||||
import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
|
||||
|
||||
export type InitializedServices = {
|
||||
settingsService: ISettingsService;
|
||||
@@ -151,12 +153,10 @@ export const initializeServices = async ({
|
||||
logger,
|
||||
envSettings,
|
||||
settingsService,
|
||||
queueMode = "primary",
|
||||
}: {
|
||||
logger: ILogger;
|
||||
envSettings: EnvConfig;
|
||||
settingsService: ISettingsService;
|
||||
queueMode?: QueueMode;
|
||||
}): Promise<InitializedServices> => {
|
||||
// Create DB
|
||||
|
||||
@@ -164,58 +164,6 @@ export const initializeServices = async ({
|
||||
db = new MongoDB(logger, envSettings);
|
||||
await db.connect();
|
||||
|
||||
// ** NOTE **
|
||||
// const dbType = envSettings.dbType;
|
||||
// DB type has been fixed to MongoDB for now
|
||||
|
||||
// if (dbType === "mongodb") {
|
||||
// db = new MongoDB(logger, envSettings);
|
||||
// }
|
||||
|
||||
// if (!db) {
|
||||
// throw new AppError({ message: "Unsupported database type", status: 500 });
|
||||
// }
|
||||
|
||||
// await db.connect();
|
||||
|
||||
// Repositories
|
||||
|
||||
// ** NOTE **
|
||||
// DB Type fixed to MongoDB for now
|
||||
// let monitorsRepository: IMonitorsRepository;
|
||||
// let checksRepository: IChecksRepository;
|
||||
// let geoChecksRepository: IGeoChecksRepository;
|
||||
// let monitorStatsRepository: IMonitorStatsRepository;
|
||||
// let statusPagesRepository: IStatusPagesRepository;
|
||||
// let usersRepository: IUsersRepository;
|
||||
// let invitesRepository: IInvitesRepository;
|
||||
// let recoveryTokensRepository: IRecoveryTokensRepository;
|
||||
// let settingsRepository: ISettingsRepository;
|
||||
// let notificationsRepository: INotificationsRepository;
|
||||
// let tagsRepository: ITagsRepository;
|
||||
// let incidentsRepository: IIncidentsRepository;
|
||||
// let teamsRepository: ITeamsRepository;
|
||||
// let maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
|
||||
// if (dbType === "mongodb") {
|
||||
// monitorsRepository = new MongoMonitorsRepository();
|
||||
// checksRepository = new MongoChecksRepository(logger);
|
||||
// geoChecksRepository = new MongoGeoChecksRepository(logger);
|
||||
// monitorStatsRepository = new MongoMonitorStatsRepository();
|
||||
// statusPagesRepository = new MongoStatusPagesRepository();
|
||||
// usersRepository = new MongoUsersRepository();
|
||||
// invitesRepository = new MongoInvitesRepository();
|
||||
// recoveryTokensRepository = new MongoRecoveryTokensRepository();
|
||||
// settingsRepository = new MongoSettingsRepository();
|
||||
// notificationsRepository = new MongoNotificationsRepository();
|
||||
// tagsRepository = new MongoTagsRepository();
|
||||
// incidentsRepository = new MongoIncidentsRepository();
|
||||
// teamsRepository = new MongoTeamsRepository();
|
||||
// maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
|
||||
// } else {
|
||||
// throw new AppError({ message: "Unsupported database type", status: 500 });
|
||||
// }
|
||||
|
||||
const monitorsRepository = new MongoMonitorsRepository();
|
||||
const checksRepository = new MongoChecksRepository(logger);
|
||||
const geoChecksRepository = new MongoGeoChecksRepository(logger);
|
||||
@@ -230,6 +178,9 @@ export const initializeServices = async ({
|
||||
const incidentsRepository = new MongoIncidentsRepository();
|
||||
const teamsRepository = new MongoTeamsRepository();
|
||||
const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
|
||||
// One identity per process
|
||||
const workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
|
||||
const jobsRepository = new MongoJobsRepository(workerId);
|
||||
const queueWorkersRepository = new MongoQueueWorkersRepository();
|
||||
|
||||
// Inject settings repository into settings service (now that DB is connected)
|
||||
@@ -320,17 +271,12 @@ export const initializeServices = async ({
|
||||
const incidentReactor = new IncidentReactor(incidentService);
|
||||
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
|
||||
|
||||
// Check producer/evaluator
|
||||
const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger);
|
||||
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
|
||||
|
||||
// pipelines
|
||||
const checkPipeline = new CheckPipeline(
|
||||
monitorsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
checkService,
|
||||
networkService,
|
||||
bufferService,
|
||||
monitorStatusPolicy,
|
||||
statusService,
|
||||
logger
|
||||
);
|
||||
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
|
||||
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
|
||||
|
||||
@@ -339,6 +285,7 @@ export const initializeServices = async ({
|
||||
checkService,
|
||||
settingsService,
|
||||
monitorsRepository,
|
||||
jobsRepository,
|
||||
teamsRepository,
|
||||
monitorStatsRepository,
|
||||
checksRepository,
|
||||
@@ -349,23 +296,22 @@ export const initializeServices = async ({
|
||||
geoCheckPipeline
|
||||
);
|
||||
|
||||
if (queueMode === "worker" && envSettings.queueType !== "lessSimpleQueue") {
|
||||
throw new AppError({
|
||||
message: `Worker mode requires QUEUE_TYPE="lessSimpleQueue" got ${envSettings.queueType}`,
|
||||
status: 500,
|
||||
service: "config services",
|
||||
method: "initializeServices",
|
||||
});
|
||||
}
|
||||
|
||||
let worker: IWorker;
|
||||
switch (envSettings.queueType) {
|
||||
case "lessSimpleQueue":
|
||||
worker = await LessSimpleWorker.create(logger, workerHelper, monitorsRepository, queueWorkersRepository, envSettings, queueMode);
|
||||
break;
|
||||
default:
|
||||
worker = await SuperSimpleQueue.create(logger, workerHelper, monitorsRepository);
|
||||
}
|
||||
const worker = await DBQueueWorker.create(
|
||||
logger,
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
reactorDispatcher,
|
||||
workerHelper,
|
||||
queueWorkersRepository,
|
||||
envSettings.queueMode,
|
||||
envSettings.queuePrimaryProcesses,
|
||||
workerId
|
||||
);
|
||||
|
||||
// Business services
|
||||
const userService = new UserService({
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import mongoose from "mongoose";
|
||||
import { MonitorModel } from "../../domain/monitors/monitor.model.js";
|
||||
import { logger } from "@/utils/logger.js";
|
||||
|
||||
const JOBS_COLLECTION = "jobs";
|
||||
|
||||
/**
|
||||
* Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed.
|
||||
*
|
||||
* Stamp the upgrade moment, not 0. If set to 0, all checks will be replayed
|
||||
*/
|
||||
export async function backfillMonitorLastEvaluatedAt(): Promise<void> {
|
||||
const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt";
|
||||
|
||||
try {
|
||||
// Drop the legacy jobs collection first. The old super-simple-scheduler persisted jobs to the
|
||||
// same "jobs" collection the new db-queue uses, so any leftover rows carry an incompatible schema.
|
||||
// Migrations run before syncIndexes(), so the new Job indexes are rebuilt right after this drop.
|
||||
const db = mongoose.connection.db;
|
||||
if (db) {
|
||||
const existing = await db.listCollections({ name: JOBS_COLLECTION }).toArray();
|
||||
if (existing.length > 0) {
|
||||
await db.collection(JOBS_COLLECTION).drop();
|
||||
logger.info({ service: SERVICE_NAME, message: `Dropped legacy "${JOBS_COLLECTION}" collection` });
|
||||
}
|
||||
}
|
||||
|
||||
logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" });
|
||||
|
||||
const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: Date.now() } });
|
||||
|
||||
logger.info({
|
||||
service: SERVICE_NAME,
|
||||
message: `Backfilled lastEvaluatedAt on ${result.modifiedCount} monitors`,
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
logger.error({ service: SERVICE_NAME, message: `Error backfilling lastEvaluatedAt: ${errorMessage}` });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import MigrationModel from "../../domain/migrations/migration.model.js";
|
||||
import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js";
|
||||
import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js";
|
||||
import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js";
|
||||
import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
|
||||
type MigrationEntry = {
|
||||
@@ -21,6 +22,7 @@ const migrations: MigrationEntry[] = [
|
||||
{ name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray },
|
||||
{ name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex },
|
||||
{ name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray },
|
||||
{ name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt },
|
||||
];
|
||||
|
||||
const runMigrations = async (logger?: ILogger) => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
|
||||
import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ValidatedEnv } from "@/config/envValidation.js";
|
||||
import type { StringValue } from "ms";
|
||||
@@ -13,8 +13,8 @@ export type EnvConfig = {
|
||||
clientHost: string;
|
||||
dbConnectionString: string;
|
||||
dbType: DbType;
|
||||
queueType: QueueType;
|
||||
queueMode: QueueMode;
|
||||
queuePrimaryProcesses: boolean;
|
||||
statusPageThemesEnabled: boolean;
|
||||
};
|
||||
|
||||
@@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService {
|
||||
clientHost: env.CLIENT_HOST,
|
||||
dbConnectionString: env.DB_CONNECTION_STRING,
|
||||
dbType: env.DB_TYPE,
|
||||
queueType: env.QUEUE_TYPE,
|
||||
queueMode: env.QUEUE_MODE,
|
||||
queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES,
|
||||
statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
export const DbTypes = ["mongodb"] as const;
|
||||
export type DbType = (typeof DbTypes)[number];
|
||||
|
||||
export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const;
|
||||
export type QueueType = (typeof QueueTypes)[number];
|
||||
|
||||
export const QueueModes = ["primary", "worker"] as const;
|
||||
export type QueueMode = (typeof QueueModes)[number];
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ export interface IChecksRepository {
|
||||
options?: { type?: MonitorType }
|
||||
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult>;
|
||||
findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
|
||||
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
|
||||
// update
|
||||
//delete
|
||||
deleteByMonitorId(monitorId: string): Promise<number>;
|
||||
|
||||
@@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
};
|
||||
};
|
||||
|
||||
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
|
||||
const docs = await CheckModel.find({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
createdAt: { $gt: new Date(since) },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.lean<CheckDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
deleteByMonitorId = async (monitorId: string): Promise<number> => {
|
||||
const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
|
||||
return result.deletedCount;
|
||||
|
||||
@@ -11,7 +11,9 @@ const SERVICE_NAME = "checkService";
|
||||
|
||||
export interface ICheckService {
|
||||
createChecks(checks: Check[]): Promise<Check[]>;
|
||||
buildCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toLastEvaluatedAt(check: Check): number;
|
||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||
getChecksByMonitor(params: {
|
||||
monitorId: string;
|
||||
teamId: string;
|
||||
@@ -56,7 +58,11 @@ export class CheckService implements ICheckService {
|
||||
return this.checksRepository.createChecks(checks);
|
||||
};
|
||||
|
||||
buildCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
toLastEvaluatedAt = (check: Check) => {
|
||||
return new Date(check.createdAt).getTime();
|
||||
};
|
||||
|
||||
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
||||
|
||||
const now = new Date().toISOString();
|
||||
@@ -82,7 +88,7 @@ export class CheckService implements ICheckService {
|
||||
this.logger.warn({
|
||||
message: "Failed to build check",
|
||||
service: SERVICE_NAME,
|
||||
method: "buildCheck",
|
||||
method: "toCheck",
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
@@ -131,6 +137,24 @@ export class CheckService implements ICheckService {
|
||||
return check;
|
||||
};
|
||||
|
||||
toStatusResponse = (check: Check) => {
|
||||
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
|
||||
monitorId: check.metadata.monitorId,
|
||||
teamId: check.metadata.teamId,
|
||||
type: check.metadata.type,
|
||||
status: check.status,
|
||||
code: check.statusCode,
|
||||
message: check.message,
|
||||
responseTime: check.responseTime,
|
||||
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
|
||||
payload:
|
||||
check.metadata.type === "hardware"
|
||||
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
|
||||
: undefined,
|
||||
};
|
||||
return statusResponse;
|
||||
};
|
||||
|
||||
getChecksByMonitor = async ({
|
||||
monitorId,
|
||||
teamId,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { Schema, model } from "mongoose";
|
||||
import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export interface JobDocument extends Omit<Job, "id"> {
|
||||
_id: string;
|
||||
}
|
||||
|
||||
const JobSchema = new Schema<JobDocument>(
|
||||
{
|
||||
_id: {
|
||||
type: String,
|
||||
required: true,
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
enum: JOB_TYPES,
|
||||
required: true,
|
||||
},
|
||||
// monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate).
|
||||
refId: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
// paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob)
|
||||
isActive: {
|
||||
type: Boolean,
|
||||
default: true,
|
||||
},
|
||||
|
||||
// Scheduling — epoch ms, so the claim/$min queries operate on plain numbers
|
||||
nextScheduledAt: {
|
||||
type: Number,
|
||||
required: true,
|
||||
},
|
||||
intervalMs: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Lock / lease
|
||||
lockedBy: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
lockedUntil: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Observability
|
||||
runCount: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
failCount: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
lastFinishedAt: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
lastFailReason: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
// Used for scanning for claims
|
||||
JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 });
|
||||
// for findById, upsertEvaluate, deleteById
|
||||
JobSchema.index({ refId: 1, type: 1 });
|
||||
|
||||
const JobModel = model<JobDocument>("Job", JobSchema);
|
||||
|
||||
export { JobModel };
|
||||
export default JobModel;
|
||||
@@ -0,0 +1,72 @@
|
||||
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export type JobPageQuery = {
|
||||
page?: number;
|
||||
rowsPerPage?: number;
|
||||
};
|
||||
export type JobPage = {
|
||||
jobs: Job[];
|
||||
count: number;
|
||||
};
|
||||
|
||||
export interface IJobsRepository {
|
||||
// ********************
|
||||
// Claims
|
||||
// ********************
|
||||
|
||||
// Atomically claim a job
|
||||
claimDue(type: JobType, now: number): Promise<Job | null>;
|
||||
|
||||
// locks up to `limit` due rows in one batch
|
||||
claimDueBatch(type: JobType, limit: number, now: number): Promise<Job[]>;
|
||||
|
||||
// Extends the locks we still hold. Returns how many were renewed
|
||||
renewLocks(ids: string[], now: number): Promise<number>;
|
||||
|
||||
// Completes a repeating job
|
||||
recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise<boolean>;
|
||||
|
||||
// Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry
|
||||
recordFailure(id: string, error: unknown, now: number): Promise<boolean>;
|
||||
|
||||
recordOneShot(id: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
|
||||
// ********************
|
||||
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Job CRUD
|
||||
// ********************
|
||||
|
||||
// Create jobs. Takes a JobSeed
|
||||
upsertJob(job: JobSeed): Promise<boolean>;
|
||||
|
||||
// Creates cleanup jobs, these are executed immediately so need their own method
|
||||
upsertCleanupJob(job: JobSeed): Promise<boolean>;
|
||||
|
||||
// Pause/Resume
|
||||
setActiveById(refId: string, isActive: boolean): Promise<boolean>;
|
||||
|
||||
// Edit job schedule
|
||||
updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
|
||||
|
||||
// Delete, drop all rows
|
||||
deleteById(refId: string): Promise<boolean>;
|
||||
|
||||
// Delete a single job of type
|
||||
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
|
||||
|
||||
// Delete jobs that reference monitors that no longer exist
|
||||
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
|
||||
|
||||
// Get job
|
||||
findById(refId: string): Promise<Job[]>;
|
||||
|
||||
// ********************
|
||||
// Observability
|
||||
// ********************
|
||||
findPage(pagination: JobPageQuery): Promise<JobPage>;
|
||||
findAll(): Promise<Job[]>;
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
|
||||
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
|
||||
|
||||
class MongoJobsRepository implements IJobsRepository {
|
||||
constructor(private readonly workerId: string) {}
|
||||
|
||||
private toEntity(doc: JobDocument): Job {
|
||||
return {
|
||||
id: doc._id,
|
||||
type: doc.type,
|
||||
refId: doc.refId,
|
||||
isActive: doc.isActive,
|
||||
nextScheduledAt: doc.nextScheduledAt,
|
||||
intervalMs: doc.intervalMs,
|
||||
lockedBy: doc.lockedBy,
|
||||
lockedUntil: doc.lockedUntil,
|
||||
runCount: doc.runCount,
|
||||
failCount: doc.failCount,
|
||||
lastFinishedAt: doc.lastFinishedAt,
|
||||
lastFailReason: doc.lastFailReason,
|
||||
};
|
||||
}
|
||||
|
||||
claimDue = async (type: JobType, now: number) => {
|
||||
const doc = await JobModel.findOneAndUpdate(
|
||||
{
|
||||
type,
|
||||
isActive: true,
|
||||
nextScheduledAt: { $lte: now }, // Find due jobs
|
||||
$or: [
|
||||
{ lockedUntil: null }, // Not locked
|
||||
{ lockedUntil: { $lt: now } }, // Free or expired lock
|
||||
],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
lockedBy: this.workerId,
|
||||
lockedUntil: now + LOCK_MS,
|
||||
},
|
||||
},
|
||||
{
|
||||
new: true,
|
||||
sort: { nextScheduledAt: 1 }, // Get the oldest due job
|
||||
}
|
||||
).lean<JobDocument>();
|
||||
return doc ? this.toEntity(doc) : null;
|
||||
};
|
||||
|
||||
claimDueBatch = async (type: JobType, limit: number, now: number) => {
|
||||
if (limit <= 0) return [];
|
||||
const dueAndFree = {
|
||||
type,
|
||||
isActive: true,
|
||||
nextScheduledAt: { $lte: now }, // due
|
||||
$or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock
|
||||
};
|
||||
|
||||
// 1. Pick the oldest-due candidates
|
||||
const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>();
|
||||
if (candidates.length === 0) return [];
|
||||
const ids = candidates.map((d) => d._id);
|
||||
|
||||
// 2. lock them. Re-check free/expired so we never steal another worker's live lock
|
||||
// only the rows this updateMany actually flips end up with claimed
|
||||
const lockedUntil = now + LOCK_MS;
|
||||
await JobModel.updateMany(
|
||||
{ _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] },
|
||||
{ $set: { lockedBy: this.workerId, lockedUntil } }
|
||||
);
|
||||
|
||||
// 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time
|
||||
const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
renewLocks = async (ids: string[], now: number) => {
|
||||
if (ids.length === 0) return 0;
|
||||
const res = await JobModel.updateMany(
|
||||
{ _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed
|
||||
{ $set: { lockedUntil: now + LOCK_MS } }
|
||||
);
|
||||
return res.modifiedCount;
|
||||
};
|
||||
|
||||
recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => {
|
||||
// Fixed rate job scheduling
|
||||
|
||||
let nextScheduledAt = scheduledRunAt + intervalMs;
|
||||
|
||||
// If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd
|
||||
if (nextScheduledAt <= now) {
|
||||
nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs);
|
||||
}
|
||||
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
recordFailure = async (id: string, error: unknown, now: number) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: now + BACKOFF_MS,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFailReason: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
$inc: {
|
||||
failCount: 1, // Increment fail count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
recordOneShot = async (id: string, now: number) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: PARKED,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
upsertEvaluate = async (monitorId: string, now: number) => {
|
||||
const filter = { type: "evaluate", refId: monitorId };
|
||||
const update = {
|
||||
$setOnInsert: {
|
||||
_id: jobId("evaluate", monitorId),
|
||||
// refId and Type inferred from filter
|
||||
isActive: true,
|
||||
intervalMs: null,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
$min: {
|
||||
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
||||
},
|
||||
};
|
||||
try {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
return res.acknowledged;
|
||||
} catch (error) {
|
||||
// Two callers can race the first insert for the same monitor, so the
|
||||
// loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min.
|
||||
if (error && typeof error === "object" && (error as { code?: number }).code === 11000) {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
return res.acknowledged;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
upsertJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
{
|
||||
$set: {
|
||||
// These fields are owned by the monitor, always safe to set
|
||||
type: job.type,
|
||||
refId: job.refId,
|
||||
isActive: job.isActive,
|
||||
intervalMs: job.intervalMs,
|
||||
},
|
||||
$setOnInsert: {
|
||||
// These fields are scheduling related, only set on insert so they aren't clobbered
|
||||
nextScheduledAt: job.nextScheduledAt,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
return res.acknowledged;
|
||||
};
|
||||
|
||||
upsertCleanupJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
{
|
||||
$set: {
|
||||
type: job.type,
|
||||
refId: job.refId,
|
||||
isActive: job.isActive,
|
||||
intervalMs: job.intervalMs,
|
||||
nextScheduledAt: job.nextScheduledAt,
|
||||
},
|
||||
$setOnInsert: {
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
return res.acknowledged;
|
||||
};
|
||||
|
||||
setActiveById = async (refId: string, isActive: boolean) => {
|
||||
const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together
|
||||
return res.modifiedCount > 0;
|
||||
};
|
||||
|
||||
updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => {
|
||||
const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } });
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
deleteById = async (refId: string) => {
|
||||
const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByIdAndType = async (refId: string, type: JobType) => {
|
||||
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
|
||||
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
|
||||
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
|
||||
return res.deletedCount ?? 0;
|
||||
};
|
||||
|
||||
findById = async (refId: string) => {
|
||||
const docs = await JobModel.find({ refId }).lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => {
|
||||
const count = await JobModel.countDocuments();
|
||||
const query = JobModel.find().sort({ nextScheduledAt: 1 });
|
||||
if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage);
|
||||
const docs = await query.lean<JobDocument[]>();
|
||||
return { jobs: docs.map(this.toEntity), count };
|
||||
};
|
||||
|
||||
findAll = async () => {
|
||||
const docs = await JobModel.find().lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
}
|
||||
|
||||
export default MongoJobsRepository;
|
||||
@@ -0,0 +1,32 @@
|
||||
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
|
||||
export type JobType = (typeof JOB_TYPES)[number];
|
||||
export const LOCK_MS = 60_000;
|
||||
export const BACKOFF_MS = 5_000;
|
||||
export const PARKED = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
|
||||
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
|
||||
|
||||
export type Job = {
|
||||
id: string;
|
||||
type: JobType;
|
||||
refId: string | null;
|
||||
isActive: boolean;
|
||||
|
||||
// Scheduling fields
|
||||
nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now
|
||||
intervalMs: number | null; // repeat period in ms; null = one time job
|
||||
|
||||
// Lock fields
|
||||
lockedBy: string | null; // workerId holding the lease
|
||||
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
|
||||
|
||||
// observability (replaces what getMetrics/getJobs)
|
||||
runCount: number;
|
||||
failCount: number;
|
||||
lastFinishedAt: number | null;
|
||||
lastFailReason: string | null;
|
||||
};
|
||||
|
||||
// These are the only values to set when creating a job. Repo owns the rest
|
||||
export type JobSeed = Pick<Job, "id" | "type" | "refId" | "isActive" | "intervalMs" | "nextScheduledAt">;
|
||||
@@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase {
|
||||
teamId: Types.ObjectId;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
lastEvaluatedAt: number; // epoch ms
|
||||
}
|
||||
|
||||
const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
|
||||
@@ -377,6 +378,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
|
||||
type: [checkSnapshotSchema],
|
||||
default: [],
|
||||
},
|
||||
lastEvaluatedAt: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
|
||||
@@ -571,6 +571,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
dnsRecordType: doc.dnsRecordType ?? undefined,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
lastEvaluatedAt: doc.lastEvaluatedAt,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -636,6 +637,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
dnsRecordType: doc.dnsRecordType ?? undefined,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
lastEvaluatedAt: doc.lastEvaluatedAt,
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService {
|
||||
recentChecks: [],
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
lastEvaluatedAt: 0,
|
||||
}));
|
||||
|
||||
const createdMonitors = await this.createMonitors(cleanedMonitors);
|
||||
|
||||
@@ -88,6 +88,7 @@ export interface Monitor {
|
||||
recentChecks: CheckSnapshot[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
lastEvaluatedAt: number; // epoch ms
|
||||
}
|
||||
|
||||
export interface MonitorsSummary {
|
||||
|
||||
@@ -2,11 +2,15 @@ import { Schema, model } from "mongoose";
|
||||
import { QueueModes } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
|
||||
// Single source of truth for worker liveness: the TTL that GCs stale records and
|
||||
// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it.
|
||||
export const WORKER_TTL_SECONDS = 180;
|
||||
|
||||
const QueueWorkerSchema = new Schema<QueueWorkerDocument>(
|
||||
{
|
||||
_id: { type: String, required: true },
|
||||
mode: { type: String, enum: QueueModes, required: true },
|
||||
lastSeenAt: { type: Date, default: Date.now, expires: 30 },
|
||||
lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS },
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ const startApp = async () => {
|
||||
logger = new Logger({ envSettings });
|
||||
|
||||
// Initialize services (connects DB, creates repositories, injects settingsRepository)
|
||||
const services = await initializeServices({ logger, envSettings, settingsService, queueMode: envSettings.queueMode });
|
||||
const services = await initializeServices({ logger, envSettings, settingsService });
|
||||
|
||||
// If this is a worker instance, we're done. No need for express
|
||||
if (envSettings.queueMode === "worker") {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
|
||||
export interface ICheckEvaluator {
|
||||
evaluate(status: MonitorStatusResponse, check: Check): Promise<MonitorEvaluation>;
|
||||
}
|
||||
|
||||
export class CheckEvaluator implements ICheckEvaluator {
|
||||
constructor(
|
||||
private statusService: IStatusService,
|
||||
private monitorStatusPolicy: IMonitorStatusPolicy
|
||||
) {}
|
||||
evaluate = async (status: MonitorStatusResponse, check: Check) => {
|
||||
// ****************************
|
||||
// Step 3: Evaluate and return result to reactors
|
||||
// ****************************
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
|
||||
|
||||
// Step 5. Get decisions and create an evaluation obj
|
||||
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
|
||||
const evaluation: MonitorEvaluation = {
|
||||
monitor: statusChangeResult.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: statusChangeResult,
|
||||
decision,
|
||||
};
|
||||
return evaluation;
|
||||
};
|
||||
}
|
||||
@@ -1,17 +1,14 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { INetworkService } from "@/service/networkService.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { IBufferService } from "@/service/bufferService.js";
|
||||
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
|
||||
import { ICheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
|
||||
const SERVICE_NAME = "CheckPipeline";
|
||||
|
||||
@@ -21,83 +18,16 @@ export interface ICheckPipeline {
|
||||
|
||||
export class CheckPipeline implements ICheckPipeline {
|
||||
constructor(
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
|
||||
private checkService: ICheckService,
|
||||
private networkService: INetworkService,
|
||||
private bufferService: IBufferService,
|
||||
private monitorStatusPolicy: IMonitorStatusPolicy,
|
||||
private statusService: IStatusService,
|
||||
private logger: ILogger
|
||||
private checkProducer: ICheckProducer,
|
||||
private checkEvaluator: ICheckEvaluator
|
||||
) {}
|
||||
|
||||
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
|
||||
const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
|
||||
const now = new Date();
|
||||
return maintenanceWindows.some((window) => isWindowActive(window, now));
|
||||
}
|
||||
|
||||
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
|
||||
const monitorId = monitor.id;
|
||||
if (!monitorId) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getMonitorJob" });
|
||||
const checkResult = await this.checkProducer.produce(monitor);
|
||||
if (!checkResult) {
|
||||
return null; // skipped due to maintenance / geo guards / other guards
|
||||
}
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
// ****************************
|
||||
|
||||
// Step 1a: Maintenance window gate - skip if in maintenance
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runCheckPipeline",
|
||||
});
|
||||
if (monitor.status !== "maintenance") {
|
||||
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 1b: Acquire status
|
||||
const status = await this.networkService.requestStatus(monitor);
|
||||
if (!status) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
// ****************************
|
||||
|
||||
// Step 2a: Create & record a check, return null if fail
|
||||
const check = this.checkService.buildCheck(status);
|
||||
if (!check) {
|
||||
this.logger.warn({
|
||||
message: `No check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runCHeckPipeline",
|
||||
details: { code: status.code, message: status.message },
|
||||
});
|
||||
return null;
|
||||
}
|
||||
// Step 2b: Add to buffer
|
||||
this.bufferService.addToBuffer(check);
|
||||
|
||||
// ****************************
|
||||
// Step 3: Evaluate and return result to reactors
|
||||
// ****************************
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
|
||||
|
||||
// Step 5. Get decisions and create an evaluation obj
|
||||
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
|
||||
const evaluation: MonitorEvaluation = {
|
||||
monitor: statusChangeResult.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: statusChangeResult,
|
||||
decision,
|
||||
};
|
||||
const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check);
|
||||
return evaluation;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IBufferService } from "@/service/bufferService.js";
|
||||
import { INetworkService } from "@/service/networkService.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
|
||||
export interface ICheckProducer {
|
||||
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "CheckProducer";
|
||||
|
||||
export class CheckProducer implements ICheckProducer {
|
||||
constructor(
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
|
||||
private checkService: ICheckService,
|
||||
private networkService: INetworkService,
|
||||
private bufferService: IBufferService,
|
||||
private logger: ILogger
|
||||
) {}
|
||||
|
||||
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
|
||||
const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
|
||||
const now = new Date();
|
||||
return windows.some((w) => isWindowActive(w, now));
|
||||
}
|
||||
|
||||
produce = async (monitor: Monitor) => {
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
|
||||
}
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
// ****************************
|
||||
|
||||
// Step 1a: Maintenance window gate - skip if in maintenance
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
});
|
||||
if (monitor.status !== "maintenance") {
|
||||
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 1b: Acquire status
|
||||
const status = await this.networkService.requestStatus(monitor);
|
||||
if (!status) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
// ****************************
|
||||
|
||||
// Step 2a: Create & record a check, return null if fail
|
||||
const check = this.checkService.toCheck(status);
|
||||
if (!check) {
|
||||
this.logger.warn({
|
||||
message: `No check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
details: { code: status.code, message: status.message },
|
||||
});
|
||||
return null;
|
||||
}
|
||||
// Step 2b: Add to buffer
|
||||
this.bufferService.addToBuffer(check);
|
||||
return { status, check };
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js";
|
||||
import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { ICheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
const POLL_MS = 250;
|
||||
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
|
||||
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
|
||||
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
|
||||
const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
||||
const CONCURRENCY: Record<JobType, number> = {
|
||||
check: 500,
|
||||
"geo-check": 20,
|
||||
evaluate: 20,
|
||||
"cleanup-orphaned": 1,
|
||||
"cleanup-retention": 1,
|
||||
};
|
||||
|
||||
export class DBQueueWorker implements IWorker {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
private stopped = false;
|
||||
private timers = new Map<JobType | "heartbeat", NodeJS.Timeout>();
|
||||
private inFlight: Record<JobType, number> = {
|
||||
check: 0,
|
||||
"geo-check": 0,
|
||||
evaluate: 0,
|
||||
"cleanup-orphaned": 0,
|
||||
"cleanup-retention": 0,
|
||||
};
|
||||
|
||||
constructor(
|
||||
private logger: ILogger,
|
||||
private jobsRepository: IJobsRepository,
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private checksRepository: IChecksRepository,
|
||||
private checkService: ICheckService, // toStatusResponse + toLastEvaluatedAt
|
||||
private checkProducer: ICheckProducer,
|
||||
private checkEvaluator: ICheckEvaluator,
|
||||
private geoCheckPipeline: ICheckPipeline,
|
||||
private dispatcher: IReactorDispatcher,
|
||||
private helper: IWorkerHelper,
|
||||
private queueWorkersRepository: IQueueWorkersRepository,
|
||||
private queueMode: QueueMode,
|
||||
private queuePrimaryProcesses: boolean,
|
||||
private readonly workerId: string // same id MongoJobsRepository stamps into lockedBy
|
||||
) {}
|
||||
|
||||
get serviceName() {
|
||||
return DBQueueWorker.SERVICE_NAME;
|
||||
}
|
||||
|
||||
static async create(
|
||||
logger: ILogger,
|
||||
jobsRepository: IJobsRepository,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
checksRepository: IChecksRepository,
|
||||
checkService: ICheckService,
|
||||
checkProducer: ICheckProducer,
|
||||
checkEvaluator: ICheckEvaluator,
|
||||
geoCheckPipeline: ICheckPipeline,
|
||||
dispatcher: IReactorDispatcher,
|
||||
helper: IWorkerHelper,
|
||||
queueWorkersRepository: IQueueWorkersRepository,
|
||||
queueMode: QueueMode,
|
||||
queuePrimaryProcesses: boolean,
|
||||
workerId: string
|
||||
): Promise<DBQueueWorker> {
|
||||
const instance = new DBQueueWorker(
|
||||
logger,
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
dispatcher,
|
||||
helper,
|
||||
queueWorkersRepository,
|
||||
queueMode,
|
||||
queuePrimaryProcesses,
|
||||
workerId
|
||||
);
|
||||
await instance.init();
|
||||
return instance;
|
||||
}
|
||||
|
||||
// Helpers, builds job seeds
|
||||
private toCheckJob = (monitor: Monitor, now: number): JobSeed => ({
|
||||
id: jobId("check", monitor.id),
|
||||
type: "check",
|
||||
refId: monitor.id,
|
||||
isActive: monitor.isActive,
|
||||
nextScheduledAt: now + Math.floor(Math.random() * monitor.interval), // Avoid herd
|
||||
intervalMs: monitor.interval,
|
||||
});
|
||||
|
||||
private toGeoCheckJob = (monitor: Monitor, now: number): JobSeed => ({
|
||||
...this.toCheckJob(monitor, now),
|
||||
id: jobId("geo-check", monitor.id),
|
||||
type: "geo-check",
|
||||
intervalMs: monitor.geoCheckInterval!,
|
||||
});
|
||||
|
||||
private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({
|
||||
id: jobId(type, null),
|
||||
type,
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: now,
|
||||
intervalMs: CLEANUP_INTERVAL_MS,
|
||||
});
|
||||
|
||||
private toSummary = (job: Job): WorkerJobSummary => ({
|
||||
monitorId: job.refId ?? job.id,
|
||||
monitorUrl: null, // Jobs don't have URL
|
||||
monitorType: job.type,
|
||||
monitorInterval: job.intervalMs,
|
||||
monitorGeoInterval: null,
|
||||
monitorActive: job.isActive,
|
||||
active: job.isActive,
|
||||
lockedBy: job.lockedBy,
|
||||
lockedUntil: job.lockedUntil,
|
||||
lockedAt: null, // no lockedAt
|
||||
runCount: job.runCount,
|
||||
failCount: job.failCount,
|
||||
failReason: job.lastFailReason,
|
||||
lastRunAt: null, // not tracked
|
||||
lastFinishedAt: job.lastFinishedAt,
|
||||
lastRunTook: null, // not tracked
|
||||
lastFailedAt: null, // not tracked
|
||||
repeat: job.intervalMs,
|
||||
});
|
||||
|
||||
// ********************
|
||||
// Stage 1: This can eventually be offloaded to a separate service
|
||||
// ********************
|
||||
|
||||
private runCheck = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
const produced = await this.checkProducer.produce(monitor);
|
||||
if (produced) {
|
||||
await this.jobsRepository.upsertEvaluate(monitor.id, Date.now());
|
||||
}
|
||||
};
|
||||
|
||||
// ********************
|
||||
// Stage 2: Evaluator loop stays here
|
||||
// ********************
|
||||
|
||||
private runEvaluate = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
|
||||
for (const check of checks) {
|
||||
const status = this.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.checkEvaluator.evaluate(status, check);
|
||||
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
|
||||
}
|
||||
};
|
||||
|
||||
private runGeoCheck = async (job: Job) => {
|
||||
const [monitor] = await this.monitorsRepository.findByIds([job.refId!]);
|
||||
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
|
||||
};
|
||||
|
||||
// Extend the lock while a job runs so a slow-but-alive job is never reclaimed.
|
||||
// Returns false once we no longer hold the lock, so the renewal timer can stop.
|
||||
private renewLock = async (job: Job): Promise<boolean> => {
|
||||
try {
|
||||
const held = await this.jobsRepository.renewLocks([job.id], Date.now());
|
||||
if (held === 0) {
|
||||
this.logger.warn({ message: `Lost lock on job ${job.id}, stopping renewal`, service: SERVICE_NAME, method: "renewLock" });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
// Transient failure, keep renewing; a missed renewal is absorbed by the 1/3 margin
|
||||
this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "renewLock" });
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
private runJob = async (job: Job) => {
|
||||
const renewTimer = setInterval(async () => {
|
||||
const renewLock = await this.renewLock(job);
|
||||
if (!renewLock) clearInterval(renewTimer);
|
||||
}, LOCK_RENEW_MS);
|
||||
try {
|
||||
switch (job.type) {
|
||||
case "check":
|
||||
await this.runCheck(job);
|
||||
break;
|
||||
case "evaluate":
|
||||
await this.runEvaluate(job);
|
||||
break;
|
||||
case "geo-check":
|
||||
await this.runGeoCheck(job);
|
||||
break;
|
||||
case "cleanup-orphaned":
|
||||
await this.helper.getCleanupOrphanedJob()(); // Get job and execute
|
||||
break;
|
||||
case "cleanup-retention":
|
||||
await this.helper.getCleanupRetentionJob()(); // Get job and execute
|
||||
break;
|
||||
}
|
||||
|
||||
if (job.intervalMs === null) {
|
||||
// One-shot job
|
||||
await this.jobsRepository.recordOneShot(job.id, Date.now());
|
||||
} else {
|
||||
await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now());
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock
|
||||
this.logger.warn({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: `runJob:${job.type}`,
|
||||
});
|
||||
} finally {
|
||||
clearInterval(renewTimer);
|
||||
}
|
||||
};
|
||||
|
||||
private startLoop = (type: JobType) => {
|
||||
const tick = async () => {
|
||||
if (this.stopped) return;
|
||||
try {
|
||||
// Claim a batch sized to the free capacity
|
||||
const capacity = CONCURRENCY[type] - this.inFlight[type];
|
||||
const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now());
|
||||
for (const job of jobs) {
|
||||
this.inFlight[type]++; // stay within concurrency limits
|
||||
this.runJob(job).finally(() => {
|
||||
this.inFlight[type]--; // job done, free the slot
|
||||
});
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: `loop:${type}`,
|
||||
});
|
||||
}
|
||||
if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down
|
||||
};
|
||||
tick();
|
||||
};
|
||||
|
||||
// Seed queue
|
||||
private reconcile = async () => {
|
||||
const now = Date.now();
|
||||
const monitors = (await this.monitorsRepository.findAll()) ?? [];
|
||||
for (const monitor of monitors) {
|
||||
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
|
||||
}
|
||||
}
|
||||
// Both cleanup jobs run immediately on every startup, then once a day
|
||||
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now));
|
||||
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now));
|
||||
};
|
||||
|
||||
// Register worker
|
||||
private heartbeat = async () => {
|
||||
try {
|
||||
await this.queueWorkersRepository.upsert(this.workerId, this.queueMode);
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: "heartbeat",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
init = async () => {
|
||||
this.stopped = false;
|
||||
|
||||
await this.heartbeat();
|
||||
this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS));
|
||||
|
||||
if (this.queueMode === "primary") {
|
||||
await this.reconcile();
|
||||
}
|
||||
|
||||
if (this.queuePrimaryProcesses === true || this.queueMode === "worker") {
|
||||
for (const type of Object.keys(this.inFlight) as JobType[]) {
|
||||
this.startLoop(type);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
addJob = async (_monitorId: string, monitor: Monitor) => {
|
||||
const now = Date.now();
|
||||
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
|
||||
};
|
||||
deleteJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.deleteById(monitor.id);
|
||||
};
|
||||
pauseJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.setActiveById(monitor.id, false);
|
||||
};
|
||||
resumeJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.setActiveById(monitor.id, true);
|
||||
};
|
||||
updateJob = async (monitor: Monitor) => {
|
||||
await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval);
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now()));
|
||||
} else {
|
||||
await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check");
|
||||
}
|
||||
};
|
||||
shutdown = async () => {
|
||||
this.stopped = true;
|
||||
// Stop all timers
|
||||
for (const timer of this.timers.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
this.timers.clear(); // Clear the map out
|
||||
// Remove workers from registry
|
||||
await this.queueWorkersRepository.deleteById(this.workerId);
|
||||
};
|
||||
getMetrics = async (): Promise<WorkerMetrics> => {
|
||||
const rows = await this.jobsRepository.findAll();
|
||||
const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS);
|
||||
const now = Date.now();
|
||||
return rows.reduce<WorkerMetrics>(
|
||||
(acc, job) => {
|
||||
acc.jobs++;
|
||||
acc.totalRuns += job.runCount;
|
||||
acc.totalFailures += job.failCount;
|
||||
if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++;
|
||||
if (job.failCount > 0) {
|
||||
acc.failingJobs++;
|
||||
acc.jobsWithFailures.push({
|
||||
monitorId: job.refId ?? job.id,
|
||||
monitorUrl: null,
|
||||
monitorType: job.type,
|
||||
failedAt: job.lastFinishedAt,
|
||||
failCount: job.failCount,
|
||||
failReason: job.lastFailReason,
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
jobs: 0,
|
||||
activeJobs: 0,
|
||||
failingJobs: 0,
|
||||
jobsWithFailures: [],
|
||||
totalRuns: 0,
|
||||
totalFailures: 0,
|
||||
workers,
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
getJobs = async (pagination: WorkerJobsPagination) => {
|
||||
const { jobs, count } = await this.jobsRepository.findPage(pagination);
|
||||
return { jobs: jobs.map(this.toSummary), count };
|
||||
};
|
||||
flushQueues = async () => {
|
||||
await this.shutdown();
|
||||
const ok = await this.init();
|
||||
return { success: ok };
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.i
|
||||
import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
|
||||
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
@@ -42,6 +43,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
private checkService: ICheckService;
|
||||
private settingsService: ISettingsService;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private jobsRepository: IJobsRepository;
|
||||
private teamsRepository: ITeamsRepository;
|
||||
private monitorStatsRepository: IMonitorStatsRepository;
|
||||
private checksRepository: IChecksRepository;
|
||||
@@ -56,6 +58,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
checkService: ICheckService,
|
||||
settingsService: ISettingsService,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
jobsRepository: IJobsRepository,
|
||||
teamsRepository: ITeamsRepository,
|
||||
monitorStatsRepository: IMonitorStatsRepository,
|
||||
checksRepository: IChecksRepository,
|
||||
@@ -69,6 +72,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
this.checkService = checkService;
|
||||
this.settingsService = settingsService;
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.jobsRepository = jobsRepository;
|
||||
this.teamsRepository = teamsRepository;
|
||||
this.monitorStatsRepository = monitorStatsRepository;
|
||||
this.checksRepository = checksRepository;
|
||||
@@ -190,6 +194,16 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
});
|
||||
}
|
||||
|
||||
// Remove orphaned jobs
|
||||
const deletedJobsCount = await this.jobsRepository.deleteByMonitorIdsNotIn(allMonitorIds);
|
||||
if (deletedJobsCount > 0) {
|
||||
this.logger.info({
|
||||
message: `Deleted ${deletedJobsCount} orphaned jobs`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getCleanupOrphanedJob",
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.info({
|
||||
message: "Cleanup of orphaned data completed",
|
||||
service: SERVICE_NAME,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.types.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
|
||||
export type MonitorEvaluation = {
|
||||
monitor: Monitor;
|
||||
|
||||
@@ -1,427 +0,0 @@
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { MongoStore, Scheduler, IJob, AddJobInput } from "less-simple-scheduler";
|
||||
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { type WorkerMetrics, type WorkerJobsPagination, type WorkerJobsPage, IWorker } from "@/worker/worker.interface.js";
|
||||
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
|
||||
// How long after its last heartbeat a worker is still considered alive. Must be longer than the scheduler's heartbeat
|
||||
const WORKER_TTL_MS = 30_000;
|
||||
|
||||
// For discriminating job types
|
||||
type MonitorJob = Omit<IJob, "data"> & { data: Monitor };
|
||||
const MONITOR_TEMPLATES = new Set<string>(["monitor-job", "geo-check-job"]);
|
||||
const isMonitorJob = (job: IJob): job is MonitorJob => MONITOR_TEMPLATES.has(job.template);
|
||||
|
||||
export class LessSimpleWorker implements IWorker {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private logger: ILogger;
|
||||
private helper: IWorkerHelper;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private workersRepository: IQueueWorkersRepository;
|
||||
private readonly scheduler: Scheduler;
|
||||
private queueMode: QueueMode;
|
||||
private heartbeatListener: ((workerId: string) => void) | null = null;
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
helper: IWorkerHelper,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
workersRepository: IQueueWorkersRepository,
|
||||
scheduler: Scheduler,
|
||||
queueMode: QueueMode
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.helper = helper;
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.workersRepository = workersRepository;
|
||||
this.scheduler = scheduler;
|
||||
this.queueMode = queueMode;
|
||||
this.registerListeners();
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return LessSimpleWorker.SERVICE_NAME;
|
||||
}
|
||||
|
||||
private registerListeners = () => {
|
||||
this.scheduler.on("scheduler:start", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler started",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:stop", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler stopped",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:error", (workerId, error) => {
|
||||
this.logger.error({
|
||||
message: `${workerId} error: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:locked", (workerId, job) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} locked`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:abort", (workerId, job, reason) => {
|
||||
this.logger.warn({
|
||||
message: `${workerId} ${job.id} aborted: ${reason}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:attempt", (workerId, job, attempt) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} attempt ${attempt}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:complete", (workerId, job) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} completed successfully`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:exhausted", (workerId, job, error) => {
|
||||
this.logger.error({
|
||||
message: `${workerId} ${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:fail", (workerId, job, error, attempt) => {
|
||||
this.logger.warn({
|
||||
message: `${workerId} ${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:start", (workerId, job) => {
|
||||
this.logger.debug({
|
||||
message: `${workerId} ${job.id} started`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
static async create(
|
||||
logger: ILogger,
|
||||
helper: IWorkerHelper,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
workersRepository: IQueueWorkersRepository,
|
||||
envSettings: EnvConfig,
|
||||
queueMode: QueueMode
|
||||
) {
|
||||
const store = new MongoStore({
|
||||
url: envSettings.dbConnectionString ?? "mongodb://localhost:27017/uptime_db",
|
||||
});
|
||||
const scheduler = new Scheduler(store, {
|
||||
concurrency: envSettings.queueMode === "primary" ? 0 : 50,
|
||||
processEvery: 100,
|
||||
});
|
||||
const instance = new LessSimpleWorker(logger, helper, monitorsRepository, workersRepository, scheduler, queueMode);
|
||||
await instance.init(queueMode);
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
private startHeartbeat = (queueMode: QueueMode) => {
|
||||
const beat = (
|
||||
workerId: string // This is the heartbeat function that registers workers
|
||||
) =>
|
||||
this.workersRepository.upsert(workerId, queueMode).catch((error: unknown) => {
|
||||
this.logger.warn({
|
||||
message: `Worker heartbeat failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "startHeartbeat",
|
||||
});
|
||||
});
|
||||
beat(this.scheduler.workerId); // Beat immediately to register
|
||||
if (this.heartbeatListener) {
|
||||
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); // remove old listeners
|
||||
}
|
||||
this.heartbeatListener = beat; /// Keep a reference to the listener
|
||||
this.scheduler.on("scheduler:heartbeat", beat); // beat on every heartbeat
|
||||
};
|
||||
|
||||
init = async (queueMode: QueueMode = "primary") => {
|
||||
try {
|
||||
await this.scheduler.start();
|
||||
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
|
||||
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
|
||||
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
|
||||
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
|
||||
|
||||
// Register in the worker registry
|
||||
this.startHeartbeat(queueMode);
|
||||
|
||||
// If this is a worker, return early - primary will populate queue
|
||||
if (queueMode === "worker") {
|
||||
this.logger.info({
|
||||
message: `Worker queue: ${this.scheduler.workerId} initialized`,
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
const monitors = await this.monitorsRepository.findAll();
|
||||
if (!monitors) {
|
||||
return true;
|
||||
}
|
||||
const jobInputs = monitors.flatMap((monitor) => this.buildJobInputs(monitor));
|
||||
if (jobInputs.length > 0) {
|
||||
await this.scheduler.addJobs(jobInputs);
|
||||
}
|
||||
|
||||
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, upsert: true });
|
||||
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000, upsert: true });
|
||||
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: `Failed to initialize LessSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
});
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Builds the scheduler inputs for a monitor: always a monitor-job, plus a
|
||||
// geo-check job when the monitor supports and enables geo checks.
|
||||
private buildJobInputs = (monitor: Monitor): AddJobInput[] => {
|
||||
const inputs: AddJobInput[] = [
|
||||
{
|
||||
id: monitor.id,
|
||||
template: "monitor-job",
|
||||
repeat: monitor.interval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
upsert: true,
|
||||
jitter: true,
|
||||
},
|
||||
];
|
||||
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
inputs.push({
|
||||
id: `${monitor.id}-geo`,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
upsert: true,
|
||||
jitter: true,
|
||||
});
|
||||
}
|
||||
|
||||
return inputs;
|
||||
};
|
||||
|
||||
addJob = async (_monitorId: string, monitor: Monitor) => {
|
||||
await this.scheduler.addJobs(this.buildJobInputs(monitor));
|
||||
};
|
||||
|
||||
deleteJob = async (monitor: Monitor) => {
|
||||
this.scheduler.removeJob(monitor.id);
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
|
||||
};
|
||||
|
||||
pauseJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.pauseJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to pause monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to pause geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
|
||||
this.logger.debug({
|
||||
message: `Paused monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
};
|
||||
|
||||
resumeJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.resumeJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to resume monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to resume geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
this.logger.debug({
|
||||
message: `Resumed monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
};
|
||||
|
||||
private syncGeoJob = async (monitor: Monitor) => {
|
||||
const geoJobId = `${monitor.id}-geo`;
|
||||
const existingGeoJob = await this.scheduler.getJob(geoJobId);
|
||||
|
||||
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
|
||||
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
|
||||
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
|
||||
return;
|
||||
}
|
||||
|
||||
// If the job exists, update it
|
||||
if (existingGeoJob) {
|
||||
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, create it
|
||||
this.scheduler.addJob({
|
||||
id: geoJobId,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
upsert: true,
|
||||
});
|
||||
};
|
||||
|
||||
updateJob = async (monitor: Monitor) => {
|
||||
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
|
||||
await this.syncGeoJob(monitor);
|
||||
};
|
||||
|
||||
shutdown = async () => {
|
||||
// Stop beating before we remove this worker, so a heartbeat firing between
|
||||
// the remove and the scheduler stopping can't re-insert it.
|
||||
if (this.heartbeatListener) {
|
||||
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener);
|
||||
this.heartbeatListener = null;
|
||||
}
|
||||
await this.workersRepository.deleteById(this.scheduler.workerId).catch((error: unknown) => {
|
||||
this.logger.warn({
|
||||
message: `Failed to deregister worker on shutdown: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "shutdown",
|
||||
});
|
||||
});
|
||||
this.scheduler.stop();
|
||||
};
|
||||
|
||||
getMetrics = async (): Promise<WorkerMetrics> => {
|
||||
const stats = await this.scheduler.getStats();
|
||||
|
||||
const jobsWithFailures = stats.jobsWithFailures.map((failure) => {
|
||||
// `data` is untyped at the store level: for monitor jobs it's the Monitor,
|
||||
// for system jobs (e.g. cleanup) it has no url/type => null
|
||||
const monitor = failure.data as Monitor | undefined;
|
||||
return {
|
||||
monitorId: failure.id,
|
||||
monitorUrl: monitor?.url ?? null,
|
||||
monitorType: monitor?.type ?? null,
|
||||
failedAt: failure.failedAt,
|
||||
failCount: failure.failCount,
|
||||
failReason: failure.failReason,
|
||||
};
|
||||
});
|
||||
|
||||
const workers = await this.workersRepository.findRecent(WORKER_TTL_MS);
|
||||
return {
|
||||
jobs: stats.jobs,
|
||||
activeJobs: stats.activeJobs,
|
||||
failingJobs: stats.failingJobs,
|
||||
totalRuns: stats.totalRuns,
|
||||
totalFailures: stats.totalFailures,
|
||||
jobsWithFailures,
|
||||
workers,
|
||||
};
|
||||
};
|
||||
|
||||
getJobs = async ({ page = 0, rowsPerPage = 0 }: WorkerJobsPagination): Promise<WorkerJobsPage> => {
|
||||
const listOptions = rowsPerPage > 0 ? { skip: Math.max(page, 0) * rowsPerPage, limit: rowsPerPage } : undefined;
|
||||
const [jobs, count] = await Promise.all([this.scheduler.getJobs(listOptions), this.scheduler.countJobs()]);
|
||||
const mapped = jobs.map((job) => {
|
||||
const monitor = isMonitorJob(job) ? job.data : undefined;
|
||||
|
||||
return {
|
||||
monitorId: job.id,
|
||||
monitorUrl: monitor?.url || null,
|
||||
monitorType: monitor?.type || null,
|
||||
monitorInterval: monitor?.interval || null,
|
||||
monitorGeoInterval: monitor?.geoCheckInterval || null,
|
||||
monitorActive: monitor?.isActive ?? null,
|
||||
active: job.active,
|
||||
repeat: job.repeat ?? null,
|
||||
lockedBy: job.lockedBy ?? null,
|
||||
lockedUntil: job.lockedUntil ?? null,
|
||||
lockedAt: job.lockedAt ?? null,
|
||||
runCount: job.runCount ?? 0,
|
||||
failCount: job.failCount ?? 0,
|
||||
failReason: job.lastError ?? null,
|
||||
lastRunAt: job.lastStartedAt ?? null,
|
||||
lastFinishedAt: job.lastFinishedAt ?? null,
|
||||
lastRunTook: job.lockedAt || !job.lastFinishedAt || !job.lastStartedAt ? null : job.lastFinishedAt - job.lastStartedAt,
|
||||
lastFailedAt: job.lastFailedAt ?? null,
|
||||
};
|
||||
});
|
||||
|
||||
return { jobs: mapped, count };
|
||||
};
|
||||
|
||||
flushQueues = async () => {
|
||||
if (this.queueMode !== "primary") {
|
||||
// Don't allow workers to flush, though this isn't currently reachable since workers have no API endpoints
|
||||
this.logger.warn({ message: "Ignoring flush on non-primary worker", service: SERVICE_NAME, method: "flushQueues" });
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
const flushRes = await this.scheduler.flushJobs(); // Less Simple Queue must flush before stopping as it needs DB
|
||||
const stopRes = await this.scheduler.stop(); // Destructive, other workers will abort their jobs
|
||||
const initRes = await this.init();
|
||||
return {
|
||||
success: Boolean(stopRes && flushRes && initRes),
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,384 +0,0 @@
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import Scheduler from "super-simple-scheduler";
|
||||
import { Monitor, MonitorType, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { hostname } from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { IWorker, WorkerJobSummary } from "@/worker/worker.interface.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
|
||||
export class SuperSimpleQueue implements IWorker {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private logger: ILogger;
|
||||
private helper: IWorkerHelper;
|
||||
private workerId: string;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private readonly scheduler: Scheduler;
|
||||
|
||||
constructor(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
|
||||
this.logger = logger;
|
||||
this.helper = helper;
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.scheduler = scheduler;
|
||||
this.registerListeners();
|
||||
this.workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return SuperSimpleQueue.SERVICE_NAME;
|
||||
}
|
||||
|
||||
private registerListeners = () => {
|
||||
this.scheduler.on("scheduler:start", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler started",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:stop", () => {
|
||||
this.logger.info({
|
||||
message: "Scheduler stopped",
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("scheduler:error", (error) => {
|
||||
this.logger.error({
|
||||
message: `Scheduler error: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:abort", (job, reason) => {
|
||||
this.logger.warn({
|
||||
message: `${job.id} aborted: ${reason}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:attempt", (job, attempt) => {
|
||||
this.logger.debug({
|
||||
message: `${job.id} attempt ${attempt}`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:complete", (job) => {
|
||||
this.logger.debug({
|
||||
message: `${job.id} completed successfully`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:exhausted", (job, error) => {
|
||||
this.logger.error({
|
||||
message: `${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:fail", (job, error, attempt) => {
|
||||
this.logger.warn({
|
||||
message: `${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
this.scheduler.on("job:start", (job) => {
|
||||
this.logger.debug({
|
||||
message: `${job.id} started`,
|
||||
service: SERVICE_NAME,
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
static async create(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository) {
|
||||
const scheduler = new Scheduler({
|
||||
concurrency: 50,
|
||||
// storeType: "mongo",
|
||||
// storeType: "redis",
|
||||
// dbUri: envSettings.dbConnectionString,
|
||||
});
|
||||
const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
|
||||
await instance.init();
|
||||
|
||||
return instance;
|
||||
}
|
||||
|
||||
init = async () => {
|
||||
try {
|
||||
this.scheduler.start();
|
||||
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
|
||||
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
|
||||
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
|
||||
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
|
||||
const monitors = await this.monitorsRepository.findAll();
|
||||
if (!monitors) {
|
||||
return true;
|
||||
}
|
||||
for (const monitor of monitors) {
|
||||
const randomOffset = Math.floor(Math.random() * 100);
|
||||
setTimeout(() => {
|
||||
this.addJob(monitor.id, monitor);
|
||||
}, randomOffset);
|
||||
}
|
||||
|
||||
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
|
||||
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 });
|
||||
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: `Failed to initialize SuperSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
});
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
addJob = async (monitorId: string, monitor: Monitor) => {
|
||||
this.scheduler.addJob({
|
||||
id: monitorId,
|
||||
template: "monitor-job",
|
||||
repeat: monitor.interval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
});
|
||||
|
||||
// Return early if we don't need geo checks
|
||||
if (!supportsGeoCheck(monitor.type)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Add geo check job if enabled for HTTP monitors
|
||||
if (monitor.geoCheckEnabled) {
|
||||
this.scheduler.addJob({
|
||||
id: `${monitorId}-geo`,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
deleteJob = async (monitor: Monitor) => {
|
||||
this.scheduler.removeJob(monitor.id);
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
|
||||
};
|
||||
|
||||
pauseJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.pauseJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to pause monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to pause geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
|
||||
this.logger.debug({
|
||||
message: `Paused monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
};
|
||||
|
||||
resumeJob = async (monitor: Monitor) => {
|
||||
const result = await this.scheduler.resumeJob(monitor.id);
|
||||
if (result === false) {
|
||||
throw new Error("Failed to resume monitor");
|
||||
}
|
||||
this.scheduler.updateJob(monitor.id, { data: monitor });
|
||||
|
||||
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
|
||||
if (geoJob) {
|
||||
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
|
||||
if (geoRes === false) {
|
||||
this.logger.error({
|
||||
message: `Failed to resume geo check job for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
}
|
||||
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
|
||||
}
|
||||
this.logger.debug({
|
||||
message: `Resumed monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
};
|
||||
|
||||
private syncGeoJob = async (monitor: Monitor) => {
|
||||
const geoJobId = `${monitor.id}-geo`;
|
||||
const existingGeoJob = await this.scheduler.getJob(geoJobId);
|
||||
|
||||
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
|
||||
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
|
||||
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
|
||||
return;
|
||||
}
|
||||
|
||||
// If the job exists, update it
|
||||
if (existingGeoJob) {
|
||||
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, create it
|
||||
this.scheduler.addJob({
|
||||
id: geoJobId,
|
||||
template: "geo-check-job",
|
||||
repeat: monitor.geoCheckInterval,
|
||||
active: monitor.isActive,
|
||||
data: monitor,
|
||||
});
|
||||
};
|
||||
|
||||
updateJob = async (monitor: Monitor) => {
|
||||
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
|
||||
await this.syncGeoJob(monitor);
|
||||
};
|
||||
|
||||
shutdown = async () => {
|
||||
this.scheduler.stop();
|
||||
};
|
||||
|
||||
getMetrics = async () => {
|
||||
const jobs = await this.scheduler.getJobs();
|
||||
const metrics: {
|
||||
jobs: number;
|
||||
activeJobs: number;
|
||||
failingJobs: number;
|
||||
jobsWithFailures: Array<{
|
||||
monitorId: string | number;
|
||||
monitorUrl: string;
|
||||
monitorType: MonitorType;
|
||||
failedAt: number | null;
|
||||
failCount: number;
|
||||
failReason: string | null;
|
||||
}>;
|
||||
totalRuns: number;
|
||||
totalFailures: number;
|
||||
workers: Array<{
|
||||
workerId: string;
|
||||
mode: QueueMode;
|
||||
lastSeenAt: number;
|
||||
}>;
|
||||
} = jobs.reduce(
|
||||
(acc, job) => {
|
||||
const runCount = job.runCount ?? 0;
|
||||
const failCount = job.failCount ?? 0;
|
||||
const lastFailedAt = job.lastFailedAt ?? 0;
|
||||
const lastRunAt = job.lastRunAt ?? 0;
|
||||
acc.totalRuns += runCount;
|
||||
acc.totalFailures += failCount;
|
||||
acc.jobs++;
|
||||
if (failCount > 0 && lastFailedAt >= lastRunAt) {
|
||||
acc.failingJobs++;
|
||||
}
|
||||
|
||||
if (job.lockedAt) {
|
||||
acc.activeJobs++;
|
||||
}
|
||||
|
||||
if (failCount > 0) {
|
||||
acc.jobsWithFailures.push({
|
||||
monitorId: job.id,
|
||||
monitorUrl: job.data?.url || null,
|
||||
monitorType: job.data?.type || null,
|
||||
failedAt: job.lastFailedAt ?? null,
|
||||
failCount,
|
||||
failReason: job.lastFailReason ?? null,
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
jobs: 0,
|
||||
activeJobs: 0,
|
||||
failingJobs: 0,
|
||||
jobsWithFailures: [] as Array<{
|
||||
monitorId: string | number;
|
||||
monitorUrl: string;
|
||||
monitorType: MonitorType;
|
||||
failedAt: number | null;
|
||||
failCount: number;
|
||||
failReason: string | null;
|
||||
}>,
|
||||
totalRuns: 0,
|
||||
totalFailures: 0,
|
||||
workers: [
|
||||
{
|
||||
workerId: this.workerId,
|
||||
mode: "primary",
|
||||
lastSeenAt: Date.now(),
|
||||
},
|
||||
],
|
||||
}
|
||||
);
|
||||
return metrics;
|
||||
};
|
||||
|
||||
getJobs = async ({ page = 0, rowsPerPage = 0 }: { page?: number; rowsPerPage?: number }): Promise<{ jobs: WorkerJobSummary[]; count: number }> => {
|
||||
const jobs = await this.scheduler.getJobs();
|
||||
const mapped = jobs.map((job) => {
|
||||
return {
|
||||
monitorId: job.id,
|
||||
monitorUrl: job.data?.url || null,
|
||||
monitorType: job.data?.type || null,
|
||||
monitorInterval: job.data?.interval || null,
|
||||
monitorGeoInterval: job.data?.geoCheckInterval || null,
|
||||
monitorActive: job.data?.isActive ?? null,
|
||||
active: job.active,
|
||||
repeat: job.repeat ?? null,
|
||||
lockedBy: job.lockedAt ? this.workerId : null,
|
||||
lockedUntil: null,
|
||||
lockedAt: job.lockedAt ?? null,
|
||||
runCount: job.runCount ?? 0,
|
||||
failCount: job.failCount ?? 0,
|
||||
failReason: job.lastFailReason ?? null,
|
||||
lastRunAt: job.lastRunAt ?? null,
|
||||
lastFinishedAt: job.lastFinishedAt ?? null,
|
||||
lastRunTook: job.lockedAt ? null : (job.lastFinishedAt ?? 0) - (job.lastRunAt ?? 0),
|
||||
lastFailedAt: job.lastFailedAt ?? null,
|
||||
};
|
||||
});
|
||||
|
||||
const start = Math.max(page, 0) * rowsPerPage;
|
||||
return {
|
||||
jobs: rowsPerPage > 0 ? mapped.slice(start, start + rowsPerPage) : mapped,
|
||||
count: mapped.length,
|
||||
};
|
||||
};
|
||||
|
||||
flushQueues = async () => {
|
||||
const stopRes = await this.scheduler.stop();
|
||||
const flushRes = await this.scheduler.flushJobs();
|
||||
const initRes = await this.init();
|
||||
return {
|
||||
success: Boolean(stopRes && flushRes && initRes),
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user