Address review on the egress recovery job (#3937)

- The egress job runs through WorkerHelper.getEgressRecoveryJob(), like
  the cleanup jobs, instead of a direct service call from the queue
- Toggling egressCheckEnabled resets the egress state and removes any
  pending recovery job
- Probes resolve and pass the proxy, so http targets honour the global
  proxy as monitors do
- PingProvider keeps IPv6 addresses intact instead of truncating at the
  first colon
- Egress targets are parsed by one function shared by the validator and
  the probe; malformed addresses such as unbalanced brackets are rejected
- Notification id pattern drops the /i flag, which produced an invalid
  OpenAPI pattern
- EgressState findSingleton reads before it writes, so a read no longer
  bumps updatedAt
- "disabled" is a distinct assessment result and is cached like any other
- Concurrent failures share an in-flight assessment for as long as it
  runs, then reuse the settled result for 5 s
- The recovery job row is only removed if nobody rescheduled it since
  this run claimed it, so a concurrent new episode keeps its job
- Deleting a notification removes it from egressNotifications

🔧
This commit is contained in:
Craig Lawson committed 2026-09-15 08:42:09 +01:00
1 parent 46eb0b4063
commit 08f104e283
26 files changed
+600 -157

No files matched your search

+1 -5
View File
@@ -10,7 +10,6 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { IWorkerHelper } from "@/worker/worker.helper.js";
import { IEgressService } from "@/domain/egress/egress.service.js";
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js";
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
@@ -45,7 +44,6 @@ export interface DBQueueWorkerDependencies {
geoCheckPipeline: ICheckPipeline;
dispatcher: IReactorDispatcher;
helper: IWorkerHelper;
egressService: IEgressService;
queueWorkersRepository: IQueueWorkersRepository; // forwarded
queueMode: QueueMode; // forwarded
queuePrimaryProcesses: boolean; // consumed immediately
@@ -74,7 +72,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
private dispatcher: IReactorDispatcher;
private geoCheckPipeline: ICheckPipeline;
private helper: IWorkerHelper;
private egressService: IEgressService;
private bufferService: IBufferService;
private isDbConnected: () => boolean;
@@ -95,7 +92,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
this.dispatcher = dependencies.dispatcher;
this.geoCheckPipeline = dependencies.geoCheckPipeline;
this.helper = dependencies.helper;
this.egressService = dependencies.egressService;
this.bufferService = dependencies.bufferService;
this.isDbConnected = dependencies.isDbConnected;
}
@@ -182,7 +178,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
await this.helper.getCleanupRetentionJob()(); // Get job and execute
break;
case "egress":
await this.egressService.checkRecovery(); // Re-probe while degraded; removes its own row on recovery
await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
break;
}
+15 -1
View File
@@ -11,10 +11,13 @@ import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
import { IEgressService } from "@/domain/egress/egress.service.js";
import type { Job } from "@/domain/jobs/job.type.js";
export interface IWorkerHelper {
getCleanupOrphanedJob(): () => Promise<void>;
getCleanupRetentionJob(): () => Promise<void>;
getEgressRecoveryJob(): (job: Job) => Promise<void>;
}
export interface MonitorActionDecision {
@@ -45,6 +48,7 @@ export class WorkerHelper implements IWorkerHelper {
private incidentsRepository: IIncidentsRepository;
private geoChecksRepository: IGeoChecksRepository;
private dockerLogsRepository: IDockerLogsRepository;
private egressService: IEgressService;
constructor(
logger: ILogger,
@@ -57,7 +61,8 @@ export class WorkerHelper implements IWorkerHelper {
checksRepository: IChecksRepository,
incidentsRepository: IIncidentsRepository,
geoChecksRepository: IGeoChecksRepository,
dockerLogsRepository: IDockerLogsRepository
dockerLogsRepository: IDockerLogsRepository,
egressService: IEgressService
) {
this.logger = logger;
this.checkService = checkService;
@@ -70,8 +75,17 @@ export class WorkerHelper implements IWorkerHelper {
this.incidentsRepository = incidentsRepository;
this.geoChecksRepository = geoChecksRepository;
this.dockerLogsRepository = dockerLogsRepository;
this.egressService = egressService;
}
// Runs while instance egress is degraded; the service removes the row once egress is back.
// Errors propagate so the queue records the failure and retries with its usual backoff.
getEgressRecoveryJob = () => {
return async (job: Job) => {
await this.egressService.checkRecovery(job);
};
};
getCleanupOrphanedJob = () => {
return async () => {
try {