bugfix, move reconcile to job scheduler
This commit is contained in:
1 parent
b21ef60cda
commit
ef353dcb36
9 files changed
+88
-71
No files matched your search
@@ -1,6 +1,5 @@
|
||||
import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
|
||||
import { IQueueWorker } from "@/worker/worker.interface.js";
|
||||
import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js";
|
||||
import { type Job, type JobType, LOCK_MS } from "@/domain/jobs/job.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
@@ -20,9 +19,7 @@ const SERVICE_NAME = "JobQueue";
|
||||
const POLL_MS = 250; // base poll interval while a loop is actively claiming work
|
||||
const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs
|
||||
const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
|
||||
const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
|
||||
const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
|
||||
const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
||||
const DRAIN_TIMEOUT_MS = 25_000;
|
||||
const DRAIN_POLL_MS = 200;
|
||||
|
||||
@@ -48,11 +45,13 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
"cleanup-retention": 0,
|
||||
};
|
||||
|
||||
private processesJobs: boolean = false;
|
||||
|
||||
constructor(
|
||||
private logger: ILogger,
|
||||
logger: ILogger, // forwarded to super
|
||||
private isDbConnected: () => boolean,
|
||||
jobsRepository: IJobsRepository, // ← no modifier: forwarded to super
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
jobsRepository: IJobsRepository, // forwarded
|
||||
monitorsRepository: IMonitorsRepository, // forwarded
|
||||
private checksRepository: IChecksRepository,
|
||||
private checkService: ICheckService,
|
||||
private bufferService: IBufferService,
|
||||
@@ -61,12 +60,13 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
private geoCheckPipeline: ICheckPipeline,
|
||||
private dispatcher: IReactorDispatcher,
|
||||
private helper: IWorkerHelper,
|
||||
queueWorkersRepository: IQueueWorkersRepository, // ← no modifier: forwarded
|
||||
private queueMode: QueueMode,
|
||||
private queuePrimaryProcesses: boolean,
|
||||
queueWorkersRepository: IQueueWorkersRepository, // forwarded
|
||||
queueMode: QueueMode, // forwarded
|
||||
queuePrimaryProcesses: boolean, // consumed immediately
|
||||
workerId: string
|
||||
) {
|
||||
super(jobsRepository, queueWorkersRepository, workerId);
|
||||
super(jobsRepository, queueWorkersRepository, monitorsRepository, queueMode, logger, workerId);
|
||||
this.processesJobs = queuePrimaryProcesses === true || queueMode === "worker";
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
@@ -115,15 +115,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
|
||||
private getInFlightCount = () => Object.values(this.inFlight).reduce((sum, n) => sum + n, 0);
|
||||
|
||||
private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({
|
||||
id: jobId(type, null),
|
||||
type,
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: now,
|
||||
intervalMs: CLEANUP_INTERVAL_MS,
|
||||
});
|
||||
|
||||
// ********************
|
||||
// Stage 1: This can eventually be offloaded to a separate service
|
||||
// ********************
|
||||
@@ -253,52 +244,16 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
// Wake a worker loop after adding a job that's due immediately.
|
||||
// Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off
|
||||
|
||||
// Seed queue
|
||||
private reconcile = async () => {
|
||||
const now = Date.now();
|
||||
const monitors = (await this.monitorsRepository.findAll()) ?? [];
|
||||
for (const monitor of monitors) {
|
||||
await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
|
||||
if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
|
||||
await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
|
||||
}
|
||||
}
|
||||
// Both cleanup jobs run immediately on every startup, then once a day
|
||||
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now));
|
||||
await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now));
|
||||
};
|
||||
|
||||
// Register worker
|
||||
private heartbeat = async () => {
|
||||
try {
|
||||
await this.queueWorkersRepository.upsert(this.workerId, this.queueMode);
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: "heartbeat",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
init = async () => {
|
||||
this.stopped = false;
|
||||
|
||||
await this.heartbeat();
|
||||
this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS));
|
||||
|
||||
if (this.queueMode === "primary") {
|
||||
await this.reconcile();
|
||||
}
|
||||
|
||||
if (this.queuePrimaryProcesses === true || this.queueMode === "worker") {
|
||||
override async init() {
|
||||
await super.init(); // heartbeat + (primary?) reconcile
|
||||
if (this.processesJobs) {
|
||||
for (const type of Object.keys(this.inFlight) as JobType[]) {
|
||||
this.startLoop(type);
|
||||
}
|
||||
}
|
||||
this.initComplete = true;
|
||||
return true;
|
||||
};
|
||||
}
|
||||
|
||||
override flushQueues = async () => {
|
||||
await this.shutdown();
|
||||
@@ -354,6 +309,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
|
||||
countAliveWorkers = async () => {
|
||||
const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS);
|
||||
return workers.length;
|
||||
return workers.filter((worker) => worker.processesJobs).length;
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user