Replace the egress recovery timer with a scheduled job (#3937)

A failed check still calls assessAfterFailure, which now shares one
in-flight assessment between concurrent failures (same cache shape as
ProxyResolver). When egress degrades it upserts a global "egress" job;
DBQueueWorker runs checkRecovery for that type at the configured
interval, and the job deletes its own row once any target answers.
The unref'd timer, startRecoveryPolling, stop and init are gone: a
degraded episode survives a restart through the jobs collection.
deleteByIdAndType accepts a null refId for global rows.

🔧
This commit is contained in:
Craig Lawson committed 2026-09-14 20:41:58 +01:00
1 parent ec01a5dfe4
commit 46eb0b4063
9 files changed
+203 -309

No files matched your search

+9
View File
@@ -10,6 +10,7 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { IEgressService } from "@/domain/egress/egress.service.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
@@ -28,6 +29,7 @@ const CONCURRENCY: Record<JobType, number> = {
evaluate: 20,
"cleanup-orphaned": 1,
"cleanup-retention": 1,
egress: 1,
};
export interface DBQueueWorkerDependencies {
@@ -43,6 +45,7 @@ export interface DBQueueWorkerDependencies {
geoCheckPipeline: ICheckPipeline;
dispatcher: IReactorDispatcher;
helper: IWorkerHelper;
egressService: IEgressService;
queueWorkersRepository: IQueueWorkersRepository; // forwarded
queueMode: QueueMode; // forwarded
queuePrimaryProcesses: boolean; // consumed immediately
@@ -61,6 +64,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
evaluate: 0,
"cleanup-orphaned": 0,
"cleanup-retention": 0,
egress: 0,
};
private checksRepository: IChecksRepository;
@@ -70,6 +74,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
private dispatcher: IReactorDispatcher;
private geoCheckPipeline: ICheckPipeline;
private helper: IWorkerHelper;
private egressService: IEgressService;
private bufferService: IBufferService;
private isDbConnected: () => boolean;
@@ -90,6 +95,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
this.dispatcher = dependencies.dispatcher;
this.geoCheckPipeline = dependencies.geoCheckPipeline;
this.helper = dependencies.helper;
this.egressService = dependencies.egressService;
this.bufferService = dependencies.bufferService;
this.isDbConnected = dependencies.isDbConnected;
}
@@ -175,6 +181,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
case "cleanup-retention":
await this.helper.getCleanupRetentionJob()(); // Get job and execute
break;
case "egress":
await this.egressService.checkRecovery(); // Re-probe while degraded; removes its own row on recovery
break;
}
if (job.intervalMs === null) {
@@ -41,6 +41,7 @@ export class JobScheduler implements IJobScheduler {
evaluate: POLL_MS,
"cleanup-orphaned": POLL_MS,
"cleanup-retention": POLL_MS,
egress: POLL_MS,
};
// Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd.
protected toCheckJob = (monitor: MonitorScheduleFields, now: number, immediate = false): JobSeed => ({