Address review on the egress recovery job (#3937)
- The egress job runs through WorkerHelper.getEgressRecoveryJob(), like
the cleanup jobs, instead of a direct service call from the queue
- Toggling egressCheckEnabled resets the egress state and removes any
pending recovery job
- Probes resolve and pass the proxy, so http targets honour the global
proxy as monitors do
- PingProvider keeps IPv6 addresses intact instead of truncating at the
first colon
- Egress targets are parsed by one function shared by the validator and
the probe; malformed addresses such as unbalanced brackets are rejected
- Notification id pattern drops the /i flag, which produced an invalid
OpenAPI pattern
- EgressState findSingleton reads before it writes, so a read no longer
bumps updatedAt
- "disabled" is a distinct assessment result and is cached like any other
- Concurrent failures share an in-flight assessment for as long as it
runs, then reuse the settled result for 5 s
- The recovery job row is only removed if nobody rescheduled it since
this run claimed it, so a concurrent new episode keeps its job
- Deleting a notification removes it from egressNotifications
🔧
This commit is contained in:
1 parent
46eb0b4063
commit
08f104e283
26 files changed
+600
-157
No files matched your search
@@ -10,7 +10,6 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
@@ -45,7 +44,6 @@ export interface DBQueueWorkerDependencies {
|
||||
geoCheckPipeline: ICheckPipeline;
|
||||
dispatcher: IReactorDispatcher;
|
||||
helper: IWorkerHelper;
|
||||
egressService: IEgressService;
|
||||
queueWorkersRepository: IQueueWorkersRepository; // forwarded
|
||||
queueMode: QueueMode; // forwarded
|
||||
queuePrimaryProcesses: boolean; // consumed immediately
|
||||
@@ -74,7 +72,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
private dispatcher: IReactorDispatcher;
|
||||
private geoCheckPipeline: ICheckPipeline;
|
||||
private helper: IWorkerHelper;
|
||||
private egressService: IEgressService;
|
||||
private bufferService: IBufferService;
|
||||
private isDbConnected: () => boolean;
|
||||
|
||||
@@ -95,7 +92,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
this.dispatcher = dependencies.dispatcher;
|
||||
this.geoCheckPipeline = dependencies.geoCheckPipeline;
|
||||
this.helper = dependencies.helper;
|
||||
this.egressService = dependencies.egressService;
|
||||
this.bufferService = dependencies.bufferService;
|
||||
this.isDbConnected = dependencies.isDbConnected;
|
||||
}
|
||||
@@ -182,7 +178,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
await this.helper.getCleanupRetentionJob()(); // Get job and execute
|
||||
break;
|
||||
case "egress":
|
||||
await this.egressService.checkRecovery(); // Re-probe while degraded; removes its own row on recovery
|
||||
await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,10 +11,13 @@ import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import type { Job } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export interface IWorkerHelper {
|
||||
getCleanupOrphanedJob(): () => Promise<void>;
|
||||
getCleanupRetentionJob(): () => Promise<void>;
|
||||
getEgressRecoveryJob(): (job: Job) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface MonitorActionDecision {
|
||||
@@ -45,6 +48,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
private incidentsRepository: IIncidentsRepository;
|
||||
private geoChecksRepository: IGeoChecksRepository;
|
||||
private dockerLogsRepository: IDockerLogsRepository;
|
||||
private egressService: IEgressService;
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
@@ -57,7 +61,8 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
checksRepository: IChecksRepository,
|
||||
incidentsRepository: IIncidentsRepository,
|
||||
geoChecksRepository: IGeoChecksRepository,
|
||||
dockerLogsRepository: IDockerLogsRepository
|
||||
dockerLogsRepository: IDockerLogsRepository,
|
||||
egressService: IEgressService
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.checkService = checkService;
|
||||
@@ -70,8 +75,17 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
this.incidentsRepository = incidentsRepository;
|
||||
this.geoChecksRepository = geoChecksRepository;
|
||||
this.dockerLogsRepository = dockerLogsRepository;
|
||||
this.egressService = egressService;
|
||||
}
|
||||
|
||||
// Runs while instance egress is degraded; the service removes the row once egress is back.
|
||||
// Errors propagate so the queue records the failure and retries with its usual backoff.
|
||||
getEgressRecoveryJob = () => {
|
||||
return async (job: Job) => {
|
||||
await this.egressService.checkRecovery(job);
|
||||
};
|
||||
};
|
||||
|
||||
getCleanupOrphanedJob = () => {
|
||||
return async () => {
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user