Replace the egress recovery timer with a scheduled job (#3937)
A failed check still calls assessAfterFailure, which now shares one
in-flight assessment between concurrent failures (same cache shape as
ProxyResolver). When egress degrades it upserts a global "egress" job;
DBQueueWorker runs checkRecovery for that type at the configured
interval, and the job deletes its own row once any target answers.
The unref'd timer, startRecoveryPolling, stop and init are gone: a
degraded episode survives a restart through the jobs collection.
deleteByIdAndType accepts a null refId for global rows.
🔧
This commit is contained in:
1 parent
ec01a5dfe4
commit
46eb0b4063
9 files changed
+203
-309
No files matched your search
@@ -10,6 +10,7 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
@@ -28,6 +29,7 @@ const CONCURRENCY: Record<JobType, number> = {
|
||||
evaluate: 20,
|
||||
"cleanup-orphaned": 1,
|
||||
"cleanup-retention": 1,
|
||||
egress: 1,
|
||||
};
|
||||
|
||||
export interface DBQueueWorkerDependencies {
|
||||
@@ -43,6 +45,7 @@ export interface DBQueueWorkerDependencies {
|
||||
geoCheckPipeline: ICheckPipeline;
|
||||
dispatcher: IReactorDispatcher;
|
||||
helper: IWorkerHelper;
|
||||
egressService: IEgressService;
|
||||
queueWorkersRepository: IQueueWorkersRepository; // forwarded
|
||||
queueMode: QueueMode; // forwarded
|
||||
queuePrimaryProcesses: boolean; // consumed immediately
|
||||
@@ -61,6 +64,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
evaluate: 0,
|
||||
"cleanup-orphaned": 0,
|
||||
"cleanup-retention": 0,
|
||||
egress: 0,
|
||||
};
|
||||
|
||||
private checksRepository: IChecksRepository;
|
||||
@@ -70,6 +74,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
private dispatcher: IReactorDispatcher;
|
||||
private geoCheckPipeline: ICheckPipeline;
|
||||
private helper: IWorkerHelper;
|
||||
private egressService: IEgressService;
|
||||
private bufferService: IBufferService;
|
||||
private isDbConnected: () => boolean;
|
||||
|
||||
@@ -90,6 +95,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
this.dispatcher = dependencies.dispatcher;
|
||||
this.geoCheckPipeline = dependencies.geoCheckPipeline;
|
||||
this.helper = dependencies.helper;
|
||||
this.egressService = dependencies.egressService;
|
||||
this.bufferService = dependencies.bufferService;
|
||||
this.isDbConnected = dependencies.isDbConnected;
|
||||
}
|
||||
@@ -175,6 +181,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
case "cleanup-retention":
|
||||
await this.helper.getCleanupRetentionJob()(); // Get job and execute
|
||||
break;
|
||||
case "egress":
|
||||
await this.egressService.checkRecovery(); // Re-probe while degraded; removes its own row on recovery
|
||||
break;
|
||||
}
|
||||
|
||||
if (job.intervalMs === null) {
|
||||
|
||||
@@ -41,6 +41,7 @@ export class JobScheduler implements IJobScheduler {
|
||||
evaluate: POLL_MS,
|
||||
"cleanup-orphaned": POLL_MS,
|
||||
"cleanup-retention": POLL_MS,
|
||||
egress: POLL_MS,
|
||||
};
|
||||
// Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd.
|
||||
protected toCheckJob = (monitor: MonitorScheduleFields, now: number, immediate = false): JobSeed => ({
|
||||
|
||||
Reference in new issue
Block a user