import { IJobScheduler } from "@/worker/worker.interface.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; import { type Job, type JobSeed, type JobType, jobId } from "@/domain/jobs/job.type.js"; import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; import { WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js"; import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; import { ILogger } from "@/utils/logger.js"; export const POLL_MS = 250; // base poll interval while a loop is actively claiming work const SERVICE_NAME = "JobScheduler"; const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; export class JobScheduler implements IJobScheduler { static SERVICE_NAME = SERVICE_NAME; constructor( protected jobsRepository: IJobsRepository, protected queueWorkersRepository: IQueueWorkersRepository, protected monitorsRepository: IMonitorsRepository, protected queueMode: QueueMode, protected logger: ILogger, protected readonly workerId: string // same id MongoJobsRepository stamps into lockedBy ) {} private heartbeatInFlight: Promise = Promise.resolve(); protected processesJobs = false; // scheduler-only by default; DBQueueWorker flips this on when it runs job loops protected stopped = false; protected draining = false; protected ticking: Partial> = {}; // Whether a loop is mid tick protected tickFns = new Map void>(); // Store tick fns so we can run them immediately on wake protected timers = new Map(); // Per job type poll intervals. These back off on idle and reset on work or wake() protected pollMs: Record = { check: POLL_MS, "geo-check": POLL_MS, evaluate: POLL_MS, "cleanup-orphaned": POLL_MS, "cleanup-retention": POLL_MS, }; // Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd. protected toCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ id: jobId("check", monitor.id), type: "check", refId: monitor.id, isActive: monitor.isActive, nextScheduledAt: immediate ? now : now + Math.floor(Math.random() * monitor.interval), // Avoid herd intervalMs: monitor.interval, }); protected toGeoCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ ...this.toCheckJob(monitor, now, immediate), id: jobId("geo-check", monitor.id), type: "geo-check", intervalMs: monitor.geoCheckInterval!, }); protected toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({ id: jobId(type, null), type, refId: null, isActive: true, nextScheduledAt: now, intervalMs: CLEANUP_INTERVAL_MS, }); private toSummary = (job: Job): WorkerJobSummary => ({ monitorId: job.refId ?? job.id, monitorType: job.type, monitorInterval: job.intervalMs, monitorActive: job.isActive, lockedBy: job.lockedBy, lockedUntil: job.lockedUntil, nextScheduledAt: job.nextScheduledAt, runCount: job.runCount, failCount: job.failCount, failReason: job.lastFailReason, lastFinishedAt: job.lastFinishedAt, }); wake = (type: JobType) => { this.pollMs[type] = POLL_MS; if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing const tick = this.tickFns.get(type); if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done const pending = this.timers.get(type); if (pending) clearTimeout(pending); // Clear out scheduled tick if any this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick }; addJob = async (_monitorId: string, monitor: Monitor) => { const now = Date.now(); await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true)); this.wake("check"); if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); this.wake("geo-check"); } }; deleteJob = async (monitor: Monitor) => { await this.jobsRepository.deleteById(monitor.id); }; pauseJob = async (monitor: Monitor) => { await this.jobsRepository.setActiveById(monitor.id, false); }; resumeJob = async (monitor: Monitor) => { await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately this.wake("check"); if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check"); }; updateJob = async (monitor: Monitor) => { await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval); if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away this.wake("geo-check"); } else { await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check"); } }; getMetrics = async (): Promise => { const rows = await this.jobsRepository.findAll(); const recent = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS); const workers = recent.filter((worker) => worker.processesJobs); const now = Date.now(); return rows.reduce( (acc, job) => { acc.jobs++; acc.totalRuns += job.runCount; acc.totalFailures += job.failCount; if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++; if (job.failCount > 0) { acc.failingJobs++; acc.jobsWithFailures.push({ monitorId: job.refId ?? job.id, monitorUrl: null, monitorType: job.type, failedAt: job.lastFinishedAt, failCount: job.failCount, failReason: job.lastFailReason, }); } return acc; }, { jobs: 0, activeJobs: 0, failingJobs: 0, jobsWithFailures: [], totalRuns: 0, totalFailures: 0, workers, } ); }; getJobs = async (pagination: WorkerJobsPagination) => { const { jobs, count } = await this.jobsRepository.findPage(pagination); return { jobs: jobs.map(this.toSummary), count }; }; flushQueues = async () => { await this.shutdown(); return { success: true }; }; // init is the scheduler's startup: register in the registry + (if primary) seed the queue. // Subclasses override to ALSO start processing, calling super.init() first. // Declared as a prototype method (not an arrow field) so subclasses can reach it via super.init(). async init(): Promise { this.stopped = false; await this.heartbeat(); this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS)); if (this.queueMode === "primary") { await this.reconcile(); } return true; } // Seed queue protected reconcile = async () => { const now = Date.now(); const monitors = (await this.monitorsRepository.findAll()) ?? []; for (const monitor of monitors) { await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now)); if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now)); } } // Both cleanup jobs run immediately on every startup, then once a day await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now)); await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now)); }; // Register worker protected heartbeat = async () => { try { if (this.stopped) return; // Don't re-register if shutting down this.heartbeatInFlight = this.queueWorkersRepository.upsert(this.workerId, this.queueMode, this.processesJobs).catch((error: unknown) => { this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "heartbeat", }); }); await this.heartbeatInFlight; } catch (error: unknown) { this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "heartbeat", }); } }; drain = async () => { this.stopped = true; this.draining = true; }; shutdown = async () => { this.stopped = true; // Stop all timers for (const timer of this.timers.values()) { clearTimeout(timer); } this.timers.clear(); // Clear the map out await this.heartbeatInFlight; // Wait for in flight jobs // Remove workers from registry await this.queueWorkersRepository.deleteById(this.workerId); }; }