Replace the egress recovery timer with a scheduled job (#3937)

A failed check still calls assessAfterFailure, which now shares one
in-flight assessment between concurrent failures (same cache shape as
ProxyResolver). When egress degrades it upserts a global "egress" job;
DBQueueWorker runs checkRecovery for that type at the configured
interval, and the job deletes its own row once any target answers.
The unref'd timer, startRecoveryPolling, stop and init are gone: a
degraded episode survives a restart through the jobs collection.
deleteByIdAndType accepts a null refId for global rows.

🔧
This commit is contained in:
Craig Lawson committed 2026-09-14 20:41:58 +01:00
1 parent ec01a5dfe4
commit 46eb0b4063
9 files changed
+203 -309

No files matched your search

@@ -9,7 +9,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
// ── Notes ──────────────────────────────────────────────────────────────────────
//
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and
// renews their locks, and registers itself in the worker heartbeat. The repository
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
// test only the orchestration around it, with the repository fully mocked.
@@ -114,6 +114,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
const egressService = { checkRecovery: jest.fn<any>().mockResolvedValue(undefined) };
const queueWorkersRepository = {
upsert: jest.fn<any>().mockResolvedValue(undefined),
deleteById: jest.fn<any>().mockResolvedValue(undefined),
@@ -134,6 +135,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
geoCheckPipeline,
dispatcher,
helper,
egressService,
queueWorkersRepository,
logger,
...overrides?.mocks,
@@ -226,7 +228,7 @@ describe("DBQueueWorker", () => {
const { mocks } = await start({ queueMode: "worker" });
const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"]));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
@@ -440,6 +442,14 @@ describe("DBQueueWorker", () => {
expect(retentionFn).toHaveBeenCalled();
});
it("egress job runs the egress recovery check", async () => {
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.egressService.checkRecovery).toHaveBeenCalledTimes(1);
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
});
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
+97 -176
View File
@@ -46,6 +46,10 @@ const createService = (overrides?: Record<string, any>) => {
markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })),
},
jobsRepository: {
upsertJob: jest.fn().mockResolvedValue(true),
deleteByIdAndType: jest.fn().mockResolvedValue(true),
},
networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) },
logger: createMockLogger(),
...overrides,
@@ -53,17 +57,20 @@ const createService = (overrides?: Record<string, any>) => {
const service = new EgressService(
defaults.settingsService as any,
defaults.egressStateRepository as any,
defaults.jobsRepository as any,
defaults.networkService as any,
defaults.logger as any
);
return { service, defaults };
};
// Lets the setTimeout chain in startRecoveryPolling settle (settings read, then timer armed) before advancing time.
const runOneTick = async (intervalMs: number) => {
await jest.advanceTimersByTimeAsync(0);
await jest.advanceTimersByTimeAsync(intervalMs);
};
const degradedRepository = (overrides?: Record<string, any>) => ({
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
markDegraded: jest.fn().mockResolvedValue(null),
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
...overrides,
});
// ── Tests ────────────────────────────────────────────────────────────────────
@@ -149,8 +156,7 @@ describe("EgressService", () => {
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
});
it("marks degraded, warns and returns degraded when every target is unreachable", async () => {
jest.useFakeTimers();
it("marks degraded, warns, inserts the recovery job and returns degraded when every target is unreachable", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } });
const result = await service.assessAfterFailure();
@@ -162,20 +168,18 @@ describe("EgressService", () => {
);
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" }));
service.stop();
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith({
id: "egress",
type: "egress",
refId: null,
isActive: true,
nextScheduledAt: expect.any(Number),
intervalMs: 30_000,
});
});
it("returns degraded without probing when the persisted state is already degraded", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
recordProbe: jest.fn(),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
},
});
it("returns degraded without probing when the persisted state is already degraded, re-arming the recovery job", async () => {
const { service, defaults } = createService({ egressStateRepository: degradedRepository() });
const result = await service.assessAfterFailure();
@@ -183,7 +187,52 @@ describe("EgressService", () => {
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
service.stop();
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
});
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
const { service, defaults } = createService({
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
networkService: { requestStatus: statusFor([]) },
});
await service.assessAfterFailure();
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
});
it("shares one probe between concurrent failures", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
expect(results).toEqual(["ok", "ok", "ok"]);
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
});
it("reuses a settled assessment within the cache window and probes again after it", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
await service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(1000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
await jest.advanceTimersByTimeAsync(5000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
});
it("does not cache an internal failure", async () => {
const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
const { service, defaults } = createService({ settingsService: { getDBSettings } });
expect(await service.assessAfterFailure()).toBeNull();
expect(await service.assessAfterFailure()).toBe("ok");
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
});
it("falls back to the default targets when the configured list is empty", async () => {
@@ -209,196 +258,68 @@ describe("EgressService", () => {
});
});
// ── recovery polling ──────────────────────────────────────────────────────
// ── checkRecovery (the "egress" job) ──────────────────────────────────────
describe("recovery polling", () => {
it("marks recovered once and stops polling when a target becomes reachable", async () => {
jest.useFakeTimers();
const recovered = makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" });
describe("checkRecovery", () => {
it("marks recovered, removes the job and logs when a target becomes reachable", async () => {
const { service, defaults } = createService({
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
recordProbe: jest.fn(),
markDegraded: jest.fn(),
markRecovered: jest.fn().mockResolvedValue(recovered),
},
egressStateRepository: degradedRepository(),
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
});
service.startRecoveryPolling();
await runOneTick(30_000);
await service.checkRecovery();
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1);
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith(
[expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
expect.any(Date)
);
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }));
// Loop has stopped: a further interval produces no more work
await runOneTick(30_000);
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1);
});
it("does not log a recovery when another process already recorded it", async () => {
jest.useFakeTimers();
it("removes the job but does not log a recovery when another process already recorded it", async () => {
const { service, defaults } = createService({
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
recordProbe: jest.fn(),
markDegraded: jest.fn(),
markRecovered: jest.fn().mockResolvedValue(null),
},
egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }),
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
});
service.startRecoveryPolling();
await runOneTick(30_000);
await service.checkRecovery();
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1);
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
expect(defaults.logger.info).not.toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })
);
});
it("records the probe and keeps polling while every target stays unreachable", async () => {
jest.useFakeTimers();
it("records the probe and keeps the job while every target stays unreachable", async () => {
const { service, defaults } = createService({
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
},
egressStateRepository: degradedRepository(),
networkService: { requestStatus: statusFor([]) },
});
service.startRecoveryPolling();
await runOneTick(30_000);
await runOneTick(30_000);
await service.checkRecovery();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
service.stop();
expect(defaults.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled();
});
it("stops polling when the persisted state is no longer degraded", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "ok" })),
recordProbe: jest.fn(),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
},
});
service.startRecoveryPolling();
await runOneTick(30_000);
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
await runOneTick(30_000);
expect(defaults.egressStateRepository.findSingleton).toHaveBeenCalledTimes(1);
});
it("survives a failing tick and reschedules", async () => {
jest.useFakeTimers();
const findSingleton = jest
.fn()
.mockRejectedValueOnce(new Error("db blip"))
.mockResolvedValue(makeState({ status: "degraded" }));
const { service, defaults } = createService({
egressStateRepository: {
findSingleton,
recordProbe: jest.fn().mockResolvedValue(makeState()),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
},
networkService: { requestStatus: statusFor([]) },
});
service.startRecoveryPolling();
await runOneTick(30_000);
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db blip", method: "tick" }));
await runOneTick(30_000);
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
service.stop();
});
it("re-reads the poll interval from settings on each tick", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: 5 })) },
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
},
networkService: { requestStatus: statusFor([]) },
});
service.startRecoveryPolling();
await runOneTick(5_000);
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
service.stop();
});
it("is idempotent: a second start does not arm a second loop", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
},
networkService: { requestStatus: statusFor([]) },
});
service.startRecoveryPolling();
service.startRecoveryPolling();
await runOneTick(30_000);
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
service.stop();
});
});
// ── init ──────────────────────────────────────────────────────────────────
describe("init", () => {
it("resumes polling when the feature is enabled and the persisted state is degraded", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
recordProbe: jest.fn(),
markDegraded: jest.fn(),
markRecovered: jest.fn().mockResolvedValue(makeState()),
},
});
await service.init();
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" }));
service.stop();
});
it("does nothing when the persisted state is ok", async () => {
it("removes the job without probing when the persisted state is no longer degraded", async () => {
const { service, defaults } = createService();
await service.init();
await service.checkRecovery();
expect(defaults.logger.info).not.toHaveBeenCalled();
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
});
it("does nothing when the feature is disabled", async () => {
const { service, defaults } = createService({
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
it("lets errors propagate so the queue records the failure and retries", async () => {
const { service } = createService({
egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }),
});
await service.init();
expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled();
await expect(service.checkRecovery()).rejects.toThrow("db down");
});
});
});