Address review on the egress recovery job (#3937)
- The egress job runs through WorkerHelper.getEgressRecoveryJob(), like
the cleanup jobs, instead of a direct service call from the queue
- Toggling egressCheckEnabled resets the egress state and removes any
pending recovery job
- Probes resolve and pass the proxy, so http targets honour the global
proxy as monitors do
- PingProvider keeps IPv6 addresses intact instead of truncating at the
first colon
- Egress targets are parsed by one function shared by the validator and
the probe; malformed addresses such as unbalanced brackets are rejected
- Notification id pattern drops the /i flag, which produced an invalid
OpenAPI pattern
- EgressState findSingleton reads before it writes, so a read no longer
bumps updatedAt
- "disabled" is a distinct assessment result and is cached like any other
- Concurrent failures share an in-flight assessment for as long as it
runs, then reuse the settled result for 5 s
- The recovery job row is only removed if nobody rescheduled it since
this run claimed it, so a concurrent new episode keeps its job
- Deleting a notification removes it from egressNotifications
🔧
This commit is contained in:
1 parent
46eb0b4063
commit
08f104e283
26 files changed
+600
-157
No files matched your search
@@ -110,6 +110,24 @@ describe("PingProvider", () => {
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("example.com");
|
||||
});
|
||||
|
||||
it("keeps a bare IPv6 address intact", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
|
||||
});
|
||||
|
||||
it("unwraps a bracketed IPv6 address and drops its port", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
|
||||
});
|
||||
|
||||
it("throws AppError when url is missing", async () => {
|
||||
const provider = new PingProvider(createMockPing() as any);
|
||||
|
||||
|
||||
@@ -113,8 +113,8 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const egressService = { checkRecovery: jest.fn<any>().mockResolvedValue(undefined) };
|
||||
const queueWorkersRepository = {
|
||||
upsert: jest.fn<any>().mockResolvedValue(undefined),
|
||||
deleteById: jest.fn<any>().mockResolvedValue(undefined),
|
||||
@@ -135,7 +135,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
geoCheckPipeline,
|
||||
dispatcher,
|
||||
helper,
|
||||
egressService,
|
||||
queueWorkersRepository,
|
||||
logger,
|
||||
...overrides?.mocks,
|
||||
@@ -442,11 +441,17 @@ describe("DBQueueWorker", () => {
|
||||
expect(retentionFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("egress job runs the egress recovery check", async () => {
|
||||
it("egress job runs the helper's egress recovery function with the claimed job", async () => {
|
||||
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
const egressFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(mocks.egressService.checkRecovery).toHaveBeenCalledTimes(1);
|
||||
expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { afterEach, describe, expect, it, jest } from "@jest/globals";
|
||||
import { EgressService } from "../../../src/domain/egress/egress.service.ts";
|
||||
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
|
||||
import type { Job } from "../../../src/domain/jobs/job.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
@@ -25,6 +26,23 @@ const makeSettings = (overrides?: Record<string, unknown>) => ({
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// The claimed "egress" job row as the queue hands it to the handler
|
||||
const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
id: "egress",
|
||||
type: "egress",
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: 1_000,
|
||||
intervalMs: 30_000,
|
||||
lockedBy: "worker-1",
|
||||
lockedUntil: 61_000,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// requestStatus mock that answers per target url
|
||||
const statusFor = (reachableTargets: string[]) =>
|
||||
jest.fn().mockImplementation(async (monitor: any) => ({
|
||||
@@ -37,6 +55,15 @@ const statusFor = (reachableTargets: string[]) =>
|
||||
responseTime: 12,
|
||||
}));
|
||||
|
||||
const degradedRepository = (overrides?: Record<string, any>) => ({
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
|
||||
markDegraded: jest.fn().mockResolvedValue(null),
|
||||
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
reset: jest.fn(),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const createService = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) },
|
||||
@@ -45,12 +72,15 @@ const createService = (overrides?: Record<string, any>) => {
|
||||
recordProbe: jest.fn().mockResolvedValue(makeState()),
|
||||
markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })),
|
||||
reset: jest.fn(),
|
||||
},
|
||||
jobsRepository: {
|
||||
upsertJob: jest.fn().mockResolvedValue(true),
|
||||
deleteByIdAndType: jest.fn().mockResolvedValue(true),
|
||||
upsertCleanupJob: jest.fn().mockResolvedValue(true),
|
||||
deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true),
|
||||
},
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) },
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
|
||||
logger: createMockLogger(),
|
||||
...overrides,
|
||||
};
|
||||
@@ -59,18 +89,14 @@ const createService = (overrides?: Record<string, any>) => {
|
||||
defaults.egressStateRepository as any,
|
||||
defaults.jobsRepository as any,
|
||||
defaults.networkService as any,
|
||||
defaults.proxyResolver as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { service, defaults };
|
||||
};
|
||||
|
||||
const degradedRepository = (overrides?: Record<string, any>) => ({
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
|
||||
markDegraded: jest.fn().mockResolvedValue(null),
|
||||
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
...overrides,
|
||||
});
|
||||
const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) =>
|
||||
(defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -85,7 +111,7 @@ describe("EgressService", () => {
|
||||
it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443"]);
|
||||
await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]);
|
||||
|
||||
const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any);
|
||||
expect(monitors).toEqual([
|
||||
@@ -93,9 +119,22 @@ describe("EgressService", () => {
|
||||
expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }),
|
||||
expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }),
|
||||
expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }),
|
||||
expect.objectContaining({ type: "ping", url: "2606:4700::1111" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => {
|
||||
const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128");
|
||||
const { service, defaults } = createService({ proxyResolver: { resolve } });
|
||||
|
||||
await service.probeTargets(["https://example.com/health"]);
|
||||
|
||||
expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" }));
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), {
|
||||
proxyUrl: "http://proxy.internal:3128",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports reachable only when the provider says status is true", async () => {
|
||||
const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } });
|
||||
|
||||
@@ -107,6 +146,15 @@ describe("EgressService", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("counts an unparseable stored target as unreachable without probing", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
const results = await service.probeTargets(["[1.1.1.1"]);
|
||||
|
||||
expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]);
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("counts a thrown provider error as unreachable", async () => {
|
||||
const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } });
|
||||
|
||||
@@ -154,9 +202,11 @@ describe("EgressService", () => {
|
||||
expect.any(Date)
|
||||
);
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("marks degraded, warns, inserts the recovery job and returns degraded when every target is unreachable", async () => {
|
||||
it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
@@ -168,7 +218,8 @@ describe("EgressService", () => {
|
||||
);
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith({
|
||||
// The worker that performed the transition sets the schedule
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
|
||||
id: "egress",
|
||||
type: "egress",
|
||||
refId: null,
|
||||
@@ -176,9 +227,26 @@ describe("EgressService", () => {
|
||||
nextScheduledAt: expect.any(Number),
|
||||
intervalMs: 30_000,
|
||||
});
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns degraded without probing when the persisted state is already degraded, re-arming the recovery job", async () => {
|
||||
it("only makes sure the job row exists when another worker performed the transition", async () => {
|
||||
const { service, defaults } = createService({
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
egressStateRepository: degradedRepository({
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState()),
|
||||
markDegraded: jest.fn().mockResolvedValue(null),
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
|
||||
const { service, defaults } = createService({ egressStateRepository: degradedRepository() });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
@@ -188,6 +256,7 @@ describe("EgressService", () => {
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
|
||||
@@ -198,41 +267,7 @@ describe("EgressService", () => {
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
|
||||
});
|
||||
|
||||
it("shares one probe between concurrent failures", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
|
||||
|
||||
expect(results).toEqual(["ok", "ok", "ok"]);
|
||||
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reuses a settled assessment within the cache window and probes again after it", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
await service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(5000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not cache an internal failure", async () => {
|
||||
const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
|
||||
const { service, defaults } = createService({ settingsService: { getDBSettings } });
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(await service.assessAfterFailure()).toBe("ok");
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
|
||||
});
|
||||
|
||||
it("falls back to the default targets when the configured list is empty", async () => {
|
||||
@@ -242,8 +277,7 @@ describe("EgressService", () => {
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
const urls = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
|
||||
expect(urls).toEqual(["1.1.1.1", "8.8.8.8"]);
|
||||
expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]);
|
||||
});
|
||||
|
||||
it("returns null and logs when the assessment itself fails", async () => {
|
||||
@@ -258,34 +292,97 @@ describe("EgressService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── shared assessment ─────────────────────────────────────────────────────
|
||||
|
||||
describe("shared assessment", () => {
|
||||
it("shares one probe between concurrent failures", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
|
||||
|
||||
expect(results).toEqual(["ok", "ok", "ok"]);
|
||||
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps sharing an in-flight assessment that outlives the reuse window", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
|
||||
|
||||
const first = service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call
|
||||
const second = service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle
|
||||
|
||||
expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]);
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds
|
||||
});
|
||||
|
||||
it("reuses a settled assessment within the reuse window and probes again after it", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
await service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(5_000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("reuses a disabled result instead of re-reading settings for every failure", async () => {
|
||||
const getDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false }));
|
||||
const { service } = createService({ settingsService: { getDBSettings } });
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
|
||||
expect(getDBSettings).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not cache an internal failure", async () => {
|
||||
const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
|
||||
const { service, defaults } = createService({ settingsService: { getDBSettings } });
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(await service.assessAfterFailure()).toBe("ok");
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── checkRecovery (the "egress" job) ──────────────────────────────────────
|
||||
|
||||
describe("checkRecovery", () => {
|
||||
it("marks recovered, removes the job and logs when a target becomes reachable", async () => {
|
||||
it("marks recovered, releases the job and logs when a target becomes reachable", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository(),
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
|
||||
});
|
||||
const job = makeJob({ nextScheduledAt: 123_456 });
|
||||
|
||||
await service.checkRecovery();
|
||||
await service.checkRecovery(job);
|
||||
|
||||
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith(
|
||||
[expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
|
||||
expect.any(Date)
|
||||
);
|
||||
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
|
||||
// Conditional on the schedule this run claimed, so a concurrent new episode keeps its job
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456);
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }));
|
||||
});
|
||||
|
||||
it("removes the job but does not log a recovery when another process already recorded it", async () => {
|
||||
it("releases the job but does not log a recovery when another process already recorded it", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }),
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
|
||||
});
|
||||
|
||||
await service.checkRecovery();
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
expect(defaults.logger.info).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })
|
||||
);
|
||||
@@ -297,21 +394,33 @@ describe("EgressService", () => {
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.checkRecovery();
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("removes the job without probing when the persisted state is no longer degraded", async () => {
|
||||
it("releases the job without probing when the persisted state is no longer degraded", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
await service.checkRecovery();
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
});
|
||||
|
||||
it("releases the job without probing when the feature has been disabled mid-episode", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
|
||||
egressStateRepository: degradedRepository(),
|
||||
});
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
});
|
||||
|
||||
it("lets errors propagate so the queue records the failure and retries", async () => {
|
||||
@@ -319,7 +428,7 @@ describe("EgressService", () => {
|
||||
egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }),
|
||||
});
|
||||
|
||||
await expect(service.checkRecovery()).rejects.toThrow("db down");
|
||||
await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -9,12 +9,18 @@ const createEgressStateRepo = () => ({
|
||||
recordProbe: jest.fn(),
|
||||
markDegraded: jest.fn(),
|
||||
markRecovered: jest.fn(),
|
||||
reset: jest.fn(),
|
||||
});
|
||||
|
||||
const createJobsRepo = () => ({
|
||||
deleteByIdAndType: jest.fn().mockResolvedValue(true),
|
||||
});
|
||||
|
||||
const createService = () => {
|
||||
const egressStateRepository = createEgressStateRepo();
|
||||
const service = new EgressStateService(egressStateRepository as any);
|
||||
return { service, egressStateRepository };
|
||||
const jobsRepository = createJobsRepo();
|
||||
const service = new EgressStateService(egressStateRepository as any, jobsRepository as any);
|
||||
return { service, egressStateRepository, jobsRepository };
|
||||
};
|
||||
|
||||
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
@@ -67,4 +73,28 @@ describe("EgressStateService", () => {
|
||||
await expect(service.getState()).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
|
||||
describe("reset", () => {
|
||||
it("removes any pending recovery job and returns the state to ok", async () => {
|
||||
const { service, egressStateRepository, jobsRepository } = createService();
|
||||
const reset = makeState();
|
||||
(egressStateRepository.reset as jest.Mock).mockResolvedValue(reset);
|
||||
|
||||
const result = await service.reset();
|
||||
|
||||
expect(jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
|
||||
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual(reset);
|
||||
});
|
||||
|
||||
it("does not touch the probe-driven transitions", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
(egressStateRepository.reset as jest.Mock).mockResolvedValue(makeState());
|
||||
|
||||
await service.reset();
|
||||
|
||||
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -26,8 +26,10 @@ const createMonitorsRepo = () => ({
|
||||
removeNotificationFromMonitors: jest.fn(),
|
||||
});
|
||||
|
||||
const createSettingsService = (clientHost = "https://app.example.com") => ({
|
||||
const createSettingsService = (clientHost = "https://app.example.com", egressNotifications: string[] = []) => ({
|
||||
getSettings: jest.fn().mockReturnValue({ clientHost }),
|
||||
getDBSettings: jest.fn().mockResolvedValue({ egressNotifications }),
|
||||
updateDbSettings: jest.fn().mockResolvedValue({ egressNotifications: [] }),
|
||||
});
|
||||
|
||||
const createMessageBuilder = () => ({
|
||||
@@ -371,5 +373,25 @@ describe("NotificationsService", () => {
|
||||
expect(result).toBe(deleted);
|
||||
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
|
||||
});
|
||||
|
||||
it("removes the notification from the egress recovered recipients in app settings", async () => {
|
||||
const settingsService = createSettingsService("https://app.example.com", ["notif-0", "notif-1"]);
|
||||
const { service, notificationsRepository } = createService({ settingsService });
|
||||
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
|
||||
|
||||
await service.deleteById("notif-1", "team-1");
|
||||
|
||||
expect(settingsService.updateDbSettings).toHaveBeenCalledWith({ egressNotifications: ["notif-0"] });
|
||||
});
|
||||
|
||||
it("leaves app settings alone when the notification is not an egress recipient", async () => {
|
||||
const settingsService = createSettingsService("https://app.example.com", ["notif-0"]);
|
||||
const { service, notificationsRepository } = createService({ settingsService });
|
||||
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
|
||||
|
||||
await service.deleteById("notif-1", "team-1");
|
||||
|
||||
expect(settingsService.updateDbSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -41,6 +41,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
const checkService = {
|
||||
deleteOlderThan: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
const egressService = {
|
||||
checkRecovery: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
@@ -54,6 +57,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
incidentsRepository,
|
||||
geoChecksRepository,
|
||||
dockerLogsRepository,
|
||||
egressService,
|
||||
...overrides,
|
||||
};
|
||||
|
||||
@@ -68,7 +72,8 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
defaults.checksRepository as any,
|
||||
defaults.incidentsRepository as any,
|
||||
defaults.geoChecksRepository as any,
|
||||
defaults.dockerLogsRepository as any
|
||||
defaults.dockerLogsRepository as any,
|
||||
defaults.egressService as any
|
||||
);
|
||||
return { helper, defaults };
|
||||
};
|
||||
@@ -184,4 +189,22 @@ describe("WorkerHelper", () => {
|
||||
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
|
||||
});
|
||||
});
|
||||
|
||||
describe("getEgressRecoveryJob", () => {
|
||||
it("returns a function that hands the claimed job to the egress service", async () => {
|
||||
const { helper, defaults } = createHelper();
|
||||
const job = { id: "egress", type: "egress", refId: null, nextScheduledAt: 1000 } as any;
|
||||
|
||||
await helper.getEgressRecoveryJob()(job);
|
||||
|
||||
expect(defaults.egressService.checkRecovery).toHaveBeenCalledWith(job);
|
||||
});
|
||||
|
||||
it("lets the egress service's error propagate so the queue records the failure", async () => {
|
||||
const { helper, defaults } = createHelper();
|
||||
(defaults.egressService.checkRecovery as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
await expect(helper.getEgressRecoveryJob()({ id: "egress" } as any)).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { parseEgressTarget } from "../../../src/utils/egressTarget.ts";
|
||||
|
||||
describe("parseEgressTarget", () => {
|
||||
it("parses http(s) URLs", () => {
|
||||
expect(parseEgressTarget("https://example.com/health")).toEqual({ kind: "http", url: "https://example.com/health" });
|
||||
expect(parseEgressTarget("HTTP://example.com")).toEqual({ kind: "http", url: "HTTP://example.com" });
|
||||
expect(parseEgressTarget("http://[2606:4700::1111]:8080/")).toEqual({ kind: "http", url: "http://[2606:4700::1111]:8080/" });
|
||||
});
|
||||
|
||||
it("parses host:port with a hostname, IPv4 address, localhost or bracketed IPv6 address", () => {
|
||||
expect(parseEgressTarget("dns.google:53")).toEqual({ kind: "port", host: "dns.google", port: 53 });
|
||||
expect(parseEgressTarget("8.8.8.8:53")).toEqual({ kind: "port", host: "8.8.8.8", port: 53 });
|
||||
expect(parseEgressTarget("localhost:8080")).toEqual({ kind: "port", host: "localhost", port: 8080 });
|
||||
expect(parseEgressTarget("[2606:4700::1111]:443")).toEqual({ kind: "port", host: "2606:4700::1111", port: 443 });
|
||||
});
|
||||
|
||||
it("parses bare hosts and addresses as ping targets", () => {
|
||||
expect(parseEgressTarget("1.1.1.1")).toEqual({ kind: "ping", host: "1.1.1.1" });
|
||||
expect(parseEgressTarget("example.com")).toEqual({ kind: "ping", host: "example.com" });
|
||||
expect(parseEgressTarget("2606:4700:4700::1111")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
|
||||
expect(parseEgressTarget("[2606:4700:4700::1111]")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
|
||||
expect(parseEgressTarget(" 1.1.1.1 ")).toEqual({ kind: "ping", host: "1.1.1.1" });
|
||||
});
|
||||
|
||||
it("rejects anything the probe could not reach", () => {
|
||||
for (const target of [
|
||||
"",
|
||||
"ftp://example.com",
|
||||
"http://",
|
||||
"bad host",
|
||||
"1.1.1.1;rm",
|
||||
"[1.1.1.1",
|
||||
"1.1.1.1]",
|
||||
"[::1",
|
||||
"::1]",
|
||||
"[[::1]]",
|
||||
"[not-an-address]:53",
|
||||
"[1.1.1.1]:53",
|
||||
"999.1.1.1",
|
||||
"1.1.1.1:0",
|
||||
"1.1.1.1:70000",
|
||||
"1.1.1.1:",
|
||||
"host-without-tld",
|
||||
]) {
|
||||
expect(parseEgressTarget(target)).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -34,8 +34,30 @@ describe("settingsValidation", () => {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow();
|
||||
});
|
||||
|
||||
it("accepts IPv6 targets, bare or bracketed with a port", () => {
|
||||
const targets = ["2606:4700:4700::1111", "[2606:4700:4700::1111]", "[2606:4700:4700::1111]:53", "localhost", "localhost:8080"];
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets }).egressCheckTargets).toEqual(targets);
|
||||
});
|
||||
|
||||
it("rejects targets that are not a host, host:port or http(s) URL", () => {
|
||||
for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm"]) {
|
||||
for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm", "http://"]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects malformed addresses that the probe could never reach", () => {
|
||||
for (const target of [
|
||||
"[1.1.1.1",
|
||||
"1.1.1.1]",
|
||||
"[::1",
|
||||
"::1]",
|
||||
"[[::1]]",
|
||||
"[not-an-address]:53",
|
||||
"999.1.1.1",
|
||||
"1.1.1.1:0",
|
||||
"1.1.1.1:70000",
|
||||
"[::1]:",
|
||||
]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
|
||||
}
|
||||
});
|
||||
@@ -59,6 +81,11 @@ describe("settingsValidation", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
|
||||
});
|
||||
|
||||
it("accepts upper-case hex notification ids", () => {
|
||||
const id = "64B7F0C2A1D2E3F4A5B6C7D8";
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [id] }).egressNotifications).toEqual([id]);
|
||||
});
|
||||
|
||||
it("rejects notification ids that are not 24-character hex", () => {
|
||||
for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow();
|
||||
|
||||
Reference in new issue
Block a user