bugfix, move reconcile to job scheduler

This commit is contained in:
Alex Holliday committed 2026-06-25 11:47:22 -07:00
1 parent b21ef60cda
commit ef353dcb36
9 files changed
+88 -71

No files matched your search

+16 -61
View File
@@ -1,6 +1,5 @@
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
import { IQueueWorker } from "@/worker/worker.interface.js";
import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js";
import { type Job, type JobType, LOCK_MS } from "@/domain/jobs/job.type.js";
import { ILogger } from "@/utils/logger.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
@@ -20,9 +19,7 @@ const SERVICE_NAME = "JobQueue";
const POLL_MS = 250; // base poll interval while a loop is actively claiming work
const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
const DRAIN_TIMEOUT_MS = 25_000;
const DRAIN_POLL_MS = 200;
@@ -48,11 +45,13 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
"cleanup-retention": 0,
};
private processesJobs: boolean = false;
constructor(
private logger: ILogger,
logger: ILogger, // forwarded to super
private isDbConnected: () => boolean,
jobsRepository: IJobsRepository, // ← no modifier: forwarded to super
private monitorsRepository: IMonitorsRepository,
jobsRepository: IJobsRepository, // forwarded
monitorsRepository: IMonitorsRepository, // forwarded
private checksRepository: IChecksRepository,
private checkService: ICheckService,
private bufferService: IBufferService,
@@ -61,12 +60,13 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
private geoCheckPipeline: ICheckPipeline,
private dispatcher: IReactorDispatcher,
private helper: IWorkerHelper,
queueWorkersRepository: IQueueWorkersRepository, // ← no modifier: forwarded
private queueMode: QueueMode,
private queuePrimaryProcesses: boolean,
queueWorkersRepository: IQueueWorkersRepository, // forwarded
queueMode: QueueMode, // forwarded
queuePrimaryProcesses: boolean, // consumed immediately
workerId: string
) {
super(jobsRepository, queueWorkersRepository, workerId);
super(jobsRepository, queueWorkersRepository, monitorsRepository, queueMode, logger, workerId);
this.processesJobs = queuePrimaryProcesses === true || queueMode === "worker";
}
get serviceName() {
@@ -115,15 +115,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
private getInFlightCount = () => Object.values(this.inFlight).reduce((sum, n) => sum + n, 0);
private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({
id: jobId(type, null),
type,
refId: null,
isActive: true,
nextScheduledAt: now,
intervalMs: CLEANUP_INTERVAL_MS,
});
// ********************
// Stage 1: This can eventually be offloaded to a separate service
// ********************
@@ -253,52 +244,16 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
// Wake a worker loop after adding a job that's due immediately.
// Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off
// Seed queue
private reconcile = async () => {
const now = Date.now();
const monitors = (await this.monitorsRepository.findAll()) ?? [];
for (const monitor of monitors) {
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
}
}
// Both cleanup jobs run immediately on every startup, then once a day
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now));
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now));
};
// Register worker
private heartbeat = async () => {
try {
await this.queueWorkersRepository.upsert(this.workerId, this.queueMode);
} catch (error: unknown) {
this.logger.warn({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: "heartbeat",
});
}
};
init = async () => {
this.stopped = false;
await this.heartbeat();
this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS));
if (this.queueMode === "primary") {
await this.reconcile();
}
if (this.queuePrimaryProcesses === true || this.queueMode === "worker") {
override async init() {
await super.init(); // heartbeat + (primary?) reconcile
if (this.processesJobs) {
for (const type of Object.keys(this.inFlight) as JobType[]) {
this.startLoop(type);
}
}
this.initComplete = true;
return true;
};
}
override flushQueues = async () => {
await this.shutdown();
@@ -354,6 +309,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
countAliveWorkers = async () => {
const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS);
return workers.length;
return workers.filter((worker) => worker.processesJobs).length;
};
}