remove old queues

This commit is contained in:
Alex Holliday committed 2026-06-15 10:39:48 -07:00
1 parent 47162f2064
commit d2461ef118
11 files changed
+29 -1102

No files matched your search

+15 -7
View File
@@ -13,6 +13,7 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { QueueMode, QueueModes } from "@/domain/app-settings/app-settings.type.js";
const SERVICE_NAME = "JobQueue";
const POLL_MS = 250;
const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
@@ -47,7 +48,8 @@ export class DBQueueWorker implements IWorker {
private checkEvaluator: ICheckEvaluator,
private geoCheckPipeline: ICheckPipeline,
private dispatcher: IReactorDispatcher,
private helper: IWorkerHelper
private helper: IWorkerHelper,
private mode: QueueMode
) {}
get serviceName() {
@@ -64,7 +66,8 @@ export class DBQueueWorker implements IWorker {
checkEvaluator: ICheckEvaluator,
geoCheckPipeline: ICheckPipeline,
dispatcher: IReactorDispatcher,
helper: IWorkerHelper
helper: IWorkerHelper,
mode: QueueMode
): Promise<DBQueueWorker> {
const instance = new DBQueueWorker(
logger,
@@ -76,7 +79,8 @@ export class DBQueueWorker implements IWorker {
checkEvaluator,
geoCheckPipeline,
dispatcher,
helper
helper,
mode
);
await instance.init();
return instance;
@@ -242,9 +246,13 @@ export class DBQueueWorker implements IWorker {
init = async () => {
this.stopped = false;
await this.reconcile();
for (const type of Object.keys(this.inFlight) as JobType[]) {
this.startLoop(type);
if (this.mode === "primary") {
await this.reconcile();
return true;
} else {
for (const type of Object.keys(this.inFlight) as JobType[]) {
this.startLoop(type);
}
}
return true;
};
@@ -307,7 +315,7 @@ export class DBQueueWorker implements IWorker {
jobsWithFailures: [],
totalRuns: 0,
totalFailures: 0,
workers: [{ workerId: this.workerId, mode: "primary", lastSeenAt: now }],
workers: [{ workerId: this.workerId, mode: this.mode, lastSeenAt: now }],
}
);
};
-427
View File
@@ -1,427 +0,0 @@
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { MongoStore, Scheduler, IJob, AddJobInput } from "less-simple-scheduler";
import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { type WorkerMetrics, type WorkerJobsPagination, type WorkerJobsPage, IWorker } from "@/worker/worker.interface.js";
import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
const SERVICE_NAME = "JobQueue";
// How long after its last heartbeat a worker is still considered alive. Must be longer than the scheduler's heartbeat
const WORKER_TTL_MS = 30_000;
// For discriminating job types
type MonitorJob = Omit<IJob, "data"> & { data: Monitor };
const MONITOR_TEMPLATES = new Set<string>(["monitor-job", "geo-check-job"]);
const isMonitorJob = (job: IJob): job is MonitorJob => MONITOR_TEMPLATES.has(job.template);
export class LessSimpleWorker implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private helper: IWorkerHelper;
private monitorsRepository: IMonitorsRepository;
private workersRepository: IQueueWorkersRepository;
private readonly scheduler: Scheduler;
private queueMode: QueueMode;
private heartbeatListener: ((workerId: string) => void) | null = null;
constructor(
logger: ILogger,
helper: IWorkerHelper,
monitorsRepository: IMonitorsRepository,
workersRepository: IQueueWorkersRepository,
scheduler: Scheduler,
queueMode: QueueMode
) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
this.workersRepository = workersRepository;
this.scheduler = scheduler;
this.queueMode = queueMode;
this.registerListeners();
}
get serviceName() {
return LessSimpleWorker.SERVICE_NAME;
}
private registerListeners = () => {
this.scheduler.on("scheduler:start", () => {
this.logger.info({
message: "Scheduler started",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:stop", () => {
this.logger.info({
message: "Scheduler stopped",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:error", (workerId, error) => {
this.logger.error({
message: `${workerId} error: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:locked", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} locked`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:abort", (workerId, job, reason) => {
this.logger.warn({
message: `${workerId} ${job.id} aborted: ${reason}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:attempt", (workerId, job, attempt) => {
this.logger.debug({
message: `${workerId} ${job.id} attempt ${attempt}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:complete", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} completed successfully`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:exhausted", (workerId, job, error) => {
this.logger.error({
message: `${workerId} ${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:fail", (workerId, job, error, attempt) => {
this.logger.warn({
message: `${workerId} ${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:start", (workerId, job) => {
this.logger.debug({
message: `${workerId} ${job.id} started`,
service: SERVICE_NAME,
});
});
};
static async create(
logger: ILogger,
helper: IWorkerHelper,
monitorsRepository: IMonitorsRepository,
workersRepository: IQueueWorkersRepository,
envSettings: EnvConfig,
queueMode: QueueMode
) {
const store = new MongoStore({
url: envSettings.dbConnectionString ?? "mongodb://localhost:27017/uptime_db",
});
const scheduler = new Scheduler(store, {
concurrency: envSettings.queueMode === "primary" ? 0 : 50,
processEvery: 100,
});
const instance = new LessSimpleWorker(logger, helper, monitorsRepository, workersRepository, scheduler, queueMode);
await instance.init(queueMode);
return instance;
}
private startHeartbeat = (queueMode: QueueMode) => {
const beat = (
workerId: string // This is the heartbeat function that registers workers
) =>
this.workersRepository.upsert(workerId, queueMode).catch((error: unknown) => {
this.logger.warn({
message: `Worker heartbeat failed: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "startHeartbeat",
});
});
beat(this.scheduler.workerId); // Beat immediately to register
if (this.heartbeatListener) {
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); // remove old listeners
}
this.heartbeatListener = beat; /// Keep a reference to the listener
this.scheduler.on("scheduler:heartbeat", beat); // beat on every heartbeat
};
init = async (queueMode: QueueMode = "primary") => {
try {
await this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
// Register in the worker registry
this.startHeartbeat(queueMode);
// If this is a worker, return early - primary will populate queue
if (queueMode === "worker") {
this.logger.info({
message: `Worker queue: ${this.scheduler.workerId} initialized`,
service: SERVICE_NAME,
method: "init",
});
return true;
}
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
return true;
}
const jobInputs = monitors.flatMap((monitor) => this.buildJobInputs(monitor));
if (jobInputs.length > 0) {
await this.scheduler.addJobs(jobInputs);
}
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, upsert: true });
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000, upsert: true });
return true;
} catch (error: unknown) {
this.logger.error({
message: `Failed to initialize LessSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "init",
});
return false;
}
};
// Builds the scheduler inputs for a monitor: always a monitor-job, plus a
// geo-check job when the monitor supports and enables geo checks.
private buildJobInputs = (monitor: Monitor): AddJobInput[] => {
const inputs: AddJobInput[] = [
{
id: monitor.id,
template: "monitor-job",
repeat: monitor.interval,
active: monitor.isActive,
data: monitor,
upsert: true,
jitter: true,
},
];
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
inputs.push({
id: `${monitor.id}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
upsert: true,
jitter: true,
});
}
return inputs;
};
addJob = async (_monitorId: string, monitor: Monitor) => {
await this.scheduler.addJobs(this.buildJobInputs(monitor));
};
deleteJob = async (monitor: Monitor) => {
this.scheduler.removeJob(monitor.id);
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: Monitor) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to pause monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to pause geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor: Monitor) => {
const result = await this.scheduler.resumeJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to resume geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
private syncGeoJob = async (monitor: Monitor) => {
const geoJobId = `${monitor.id}-geo`;
const existingGeoJob = await this.scheduler.getJob(geoJobId);
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
return;
}
// If the job exists, update it
if (existingGeoJob) {
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
return;
}
// Otherwise, create it
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
upsert: true,
});
};
updateJob = async (monitor: Monitor) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
await this.syncGeoJob(monitor);
};
shutdown = async () => {
// Stop beating before we remove this worker, so a heartbeat firing between
// the remove and the scheduler stopping can't re-insert it.
if (this.heartbeatListener) {
this.scheduler.off("scheduler:heartbeat", this.heartbeatListener);
this.heartbeatListener = null;
}
await this.workersRepository.deleteById(this.scheduler.workerId).catch((error: unknown) => {
this.logger.warn({
message: `Failed to deregister worker on shutdown: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "shutdown",
});
});
this.scheduler.stop();
};
getMetrics = async (): Promise<WorkerMetrics> => {
const stats = await this.scheduler.getStats();
const jobsWithFailures = stats.jobsWithFailures.map((failure) => {
// `data` is untyped at the store level: for monitor jobs it's the Monitor,
// for system jobs (e.g. cleanup) it has no url/type => null
const monitor = failure.data as Monitor | undefined;
return {
monitorId: failure.id,
monitorUrl: monitor?.url ?? null,
monitorType: monitor?.type ?? null,
failedAt: failure.failedAt,
failCount: failure.failCount,
failReason: failure.failReason,
};
});
const workers = await this.workersRepository.findRecent(WORKER_TTL_MS);
return {
jobs: stats.jobs,
activeJobs: stats.activeJobs,
failingJobs: stats.failingJobs,
totalRuns: stats.totalRuns,
totalFailures: stats.totalFailures,
jobsWithFailures,
workers,
};
};
getJobs = async ({ page = 0, rowsPerPage = 0 }: WorkerJobsPagination): Promise<WorkerJobsPage> => {
const listOptions = rowsPerPage > 0 ? { skip: Math.max(page, 0) * rowsPerPage, limit: rowsPerPage } : undefined;
const [jobs, count] = await Promise.all([this.scheduler.getJobs(listOptions), this.scheduler.countJobs()]);
const mapped = jobs.map((job) => {
const monitor = isMonitorJob(job) ? job.data : undefined;
return {
monitorId: job.id,
monitorUrl: monitor?.url || null,
monitorType: monitor?.type || null,
monitorInterval: monitor?.interval || null,
monitorGeoInterval: monitor?.geoCheckInterval || null,
monitorActive: monitor?.isActive ?? null,
active: job.active,
repeat: job.repeat ?? null,
lockedBy: job.lockedBy ?? null,
lockedUntil: job.lockedUntil ?? null,
lockedAt: job.lockedAt ?? null,
runCount: job.runCount ?? 0,
failCount: job.failCount ?? 0,
failReason: job.lastError ?? null,
lastRunAt: job.lastStartedAt ?? null,
lastFinishedAt: job.lastFinishedAt ?? null,
lastRunTook: job.lockedAt || !job.lastFinishedAt || !job.lastStartedAt ? null : job.lastFinishedAt - job.lastStartedAt,
lastFailedAt: job.lastFailedAt ?? null,
};
});
return { jobs: mapped, count };
};
flushQueues = async () => {
if (this.queueMode !== "primary") {
// Don't allow workers to flush, though this isn't currently reachable since workers have no API endpoints
this.logger.warn({ message: "Ignoring flush on non-primary worker", service: SERVICE_NAME, method: "flushQueues" });
return { success: false };
}
const flushRes = await this.scheduler.flushJobs(); // Less Simple Queue must flush before stopping as it needs DB
const stopRes = await this.scheduler.stop(); // Destructive, other workers will abort their jobs
const initRes = await this.init();
return {
success: Boolean(stopRes && flushRes && initRes),
};
};
}
-384
View File
@@ -1,384 +0,0 @@
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import Scheduler from "super-simple-scheduler";
import { Monitor, MonitorType, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { hostname } from "node:os";
import { randomUUID } from "node:crypto";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { IWorker, WorkerJobSummary } from "@/worker/worker.interface.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
const SERVICE_NAME = "JobQueue";
export class SuperSimpleQueue implements IWorker {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private helper: IWorkerHelper;
private workerId: string;
private monitorsRepository: IMonitorsRepository;
private readonly scheduler: Scheduler;
constructor(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
this.scheduler = scheduler;
this.registerListeners();
this.workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
}
get serviceName() {
return SuperSimpleQueue.SERVICE_NAME;
}
private registerListeners = () => {
this.scheduler.on("scheduler:start", () => {
this.logger.info({
message: "Scheduler started",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:stop", () => {
this.logger.info({
message: "Scheduler stopped",
service: SERVICE_NAME,
});
});
this.scheduler.on("scheduler:error", (error) => {
this.logger.error({
message: `Scheduler error: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:abort", (job, reason) => {
this.logger.warn({
message: `${job.id} aborted: ${reason}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:attempt", (job, attempt) => {
this.logger.debug({
message: `${job.id} attempt ${attempt}`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:complete", (job) => {
this.logger.debug({
message: `${job.id} completed successfully`,
service: SERVICE_NAME,
});
});
this.scheduler.on("job:exhausted", (job, error) => {
this.logger.error({
message: `${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:fail", (job, error, attempt) => {
this.logger.warn({
message: `${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
stack: error instanceof Error ? error.stack : undefined,
});
});
this.scheduler.on("job:start", (job) => {
this.logger.debug({
message: `${job.id} started`,
service: SERVICE_NAME,
});
});
};
static async create(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository) {
const scheduler = new Scheduler({
concurrency: 50,
// storeType: "mongo",
// storeType: "redis",
// dbUri: envSettings.dbConnectionString,
});
const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
await instance.init();
return instance;
}
init = async () => {
try {
this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
return true;
}
for (const monitor of monitors) {
const randomOffset = Math.floor(Math.random() * 100);
setTimeout(() => {
this.addJob(monitor.id, monitor);
}, randomOffset);
}
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 });
return true;
} catch (error: unknown) {
this.logger.error({
message: `Failed to initialize SuperSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "init",
});
return false;
}
};
addJob = async (monitorId: string, monitor: Monitor) => {
this.scheduler.addJob({
id: monitorId,
template: "monitor-job",
repeat: monitor.interval,
active: monitor.isActive,
data: monitor,
});
// Return early if we don't need geo checks
if (!supportsGeoCheck(monitor.type)) {
return;
}
// Add geo check job if enabled for HTTP monitors
if (monitor.geoCheckEnabled) {
this.scheduler.addJob({
id: `${monitorId}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
}
};
deleteJob = async (monitor: Monitor) => {
this.scheduler.removeJob(monitor.id);
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: Monitor) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to pause monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to pause geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor: Monitor) => {
const result = await this.scheduler.resumeJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.scheduler.updateJob(monitor.id, { data: monitor });
const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
if (geoJob) {
const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
if (geoRes === false) {
this.logger.error({
message: `Failed to resume geo check job for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
}
this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
}
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
private syncGeoJob = async (monitor: Monitor) => {
const geoJobId = `${monitor.id}-geo`;
const existingGeoJob = await this.scheduler.getJob(geoJobId);
// If geoChecks have been disabled, or the monitor type doesn't support them, remove
if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
if (existingGeoJob) this.scheduler.removeJob(geoJobId);
return;
}
// If the job exists, update it
if (existingGeoJob) {
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
return;
}
// Otherwise, create it
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
};
updateJob = async (monitor: Monitor) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
await this.syncGeoJob(monitor);
};
shutdown = async () => {
this.scheduler.stop();
};
getMetrics = async () => {
const jobs = await this.scheduler.getJobs();
const metrics: {
jobs: number;
activeJobs: number;
failingJobs: number;
jobsWithFailures: Array<{
monitorId: string | number;
monitorUrl: string;
monitorType: MonitorType;
failedAt: number | null;
failCount: number;
failReason: string | null;
}>;
totalRuns: number;
totalFailures: number;
workers: Array<{
workerId: string;
mode: QueueMode;
lastSeenAt: number;
}>;
} = jobs.reduce(
(acc, job) => {
const runCount = job.runCount ?? 0;
const failCount = job.failCount ?? 0;
const lastFailedAt = job.lastFailedAt ?? 0;
const lastRunAt = job.lastRunAt ?? 0;
acc.totalRuns += runCount;
acc.totalFailures += failCount;
acc.jobs++;
if (failCount > 0 && lastFailedAt >= lastRunAt) {
acc.failingJobs++;
}
if (job.lockedAt) {
acc.activeJobs++;
}
if (failCount > 0) {
acc.jobsWithFailures.push({
monitorId: job.id,
monitorUrl: job.data?.url || null,
monitorType: job.data?.type || null,
failedAt: job.lastFailedAt ?? null,
failCount,
failReason: job.lastFailReason ?? null,
});
}
return acc;
},
{
jobs: 0,
activeJobs: 0,
failingJobs: 0,
jobsWithFailures: [] as Array<{
monitorId: string | number;
monitorUrl: string;
monitorType: MonitorType;
failedAt: number | null;
failCount: number;
failReason: string | null;
}>,
totalRuns: 0,
totalFailures: 0,
workers: [
{
workerId: this.workerId,
mode: "primary",
lastSeenAt: Date.now(),
},
],
}
);
return metrics;
};
getJobs = async ({ page = 0, rowsPerPage = 0 }: { page?: number; rowsPerPage?: number }): Promise<{ jobs: WorkerJobSummary[]; count: number }> => {
const jobs = await this.scheduler.getJobs();
const mapped = jobs.map((job) => {
return {
monitorId: job.id,
monitorUrl: job.data?.url || null,
monitorType: job.data?.type || null,
monitorInterval: job.data?.interval || null,
monitorGeoInterval: job.data?.geoCheckInterval || null,
monitorActive: job.data?.isActive ?? null,
active: job.active,
repeat: job.repeat ?? null,
lockedBy: job.lockedAt ? this.workerId : null,
lockedUntil: null,
lockedAt: job.lockedAt ?? null,
runCount: job.runCount ?? 0,
failCount: job.failCount ?? 0,
failReason: job.lastFailReason ?? null,
lastRunAt: job.lastRunAt ?? null,
lastFinishedAt: job.lastFinishedAt ?? null,
lastRunTook: job.lockedAt ? null : (job.lastFinishedAt ?? 0) - (job.lastRunAt ?? 0),
lastFailedAt: job.lastFailedAt ?? null,
};
});
const start = Math.max(page, 0) * rowsPerPage;
return {
jobs: rowsPerPage > 0 ? mapped.slice(start, start + rowsPerPage) : mapped,
count: mapped.length,
};
};
flushQueues = async () => {
const stopRes = await this.scheduler.stop();
const flushRes = await this.scheduler.flushJobs();
const initRes = await this.init();
return {
success: Boolean(stopRes && flushRes && initRes),
};
};
}