Address review on the egress recovery job (#3937)

- The egress job runs through WorkerHelper.getEgressRecoveryJob(), like
  the cleanup jobs, instead of a direct service call from the queue
- Toggling egressCheckEnabled resets the egress state and removes any
  pending recovery job
- Probes resolve and pass the proxy, so http targets honour the global
  proxy as monitors do
- PingProvider keeps IPv6 addresses intact instead of truncating at the
  first colon
- Egress targets are parsed by one function shared by the validator and
  the probe; malformed addresses such as unbalanced brackets are rejected
- Notification id pattern drops the /i flag, which produced an invalid
  OpenAPI pattern
- EgressState findSingleton reads before it writes, so a read no longer
  bumps updatedAt
- "disabled" is a distinct assessment result and is cached like any other
- Concurrent failures share an in-flight assessment for as long as it
  runs, then reuse the settled result for 5 s
- The recovery job row is only removed if nobody rescheduled it since
  this run claimed it, so a concurrent new episode keeps its job
- Deleting a notification removes it from egressNotifications

🔧
This commit is contained in:
Craig Lawson committed 2026-09-15 08:42:09 +01:00
1 parent 46eb0b4063
commit 08f104e283
26 files changed
+600 -157

No files matched your search

@@ -110,6 +110,24 @@ describe("PingProvider", () => {
expect(mockPing.promise.probe).toHaveBeenCalledWith("example.com");
});
it("keeps a bare IPv6 address intact", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any);
await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" }));
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
});
it("unwraps a bracketed IPv6 address and drops its port", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any);
await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" }));
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
});
it("throws AppError when url is missing", async () => {
const provider = new PingProvider(createMockPing() as any);
@@ -113,8 +113,8 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
const egressService = { checkRecovery: jest.fn<any>().mockResolvedValue(undefined) };
const queueWorkersRepository = {
upsert: jest.fn<any>().mockResolvedValue(undefined),
deleteById: jest.fn<any>().mockResolvedValue(undefined),
@@ -135,7 +135,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
geoCheckPipeline,
dispatcher,
helper,
egressService,
queueWorkersRepository,
logger,
...overrides?.mocks,
@@ -442,11 +441,17 @@ describe("DBQueueWorker", () => {
expect(retentionFn).toHaveBeenCalled();
});
it("egress job runs the egress recovery check", async () => {
it("egress job runs the helper's egress recovery function with the claimed job", async () => {
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
const egressFn = jest.fn<any>().mockResolvedValue(undefined);
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
};
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
expect(mocks.egressService.checkRecovery).toHaveBeenCalledTimes(1);
expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
});
+170 -61
View File
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, jest } from "@jest/globals";
import { EgressService } from "../../../src/domain/egress/egress.service.ts";
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
import type { Job } from "../../../src/domain/jobs/job.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -25,6 +26,23 @@ const makeSettings = (overrides?: Record<string, unknown>) => ({
...overrides,
});
// The claimed "egress" job row as the queue hands it to the handler
const makeJob = (overrides?: Partial<Job>): Job => ({
id: "egress",
type: "egress",
refId: null,
isActive: true,
nextScheduledAt: 1_000,
intervalMs: 30_000,
lockedBy: "worker-1",
lockedUntil: 61_000,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
...overrides,
});
// requestStatus mock that answers per target url
const statusFor = (reachableTargets: string[]) =>
jest.fn().mockImplementation(async (monitor: any) => ({
@@ -37,6 +55,15 @@ const statusFor = (reachableTargets: string[]) =>
responseTime: 12,
}));
const degradedRepository = (overrides?: Record<string, any>) => ({
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
markDegraded: jest.fn().mockResolvedValue(null),
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
reset: jest.fn(),
...overrides,
});
const createService = (overrides?: Record<string, any>) => {
const defaults = {
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) },
@@ -45,12 +72,15 @@ const createService = (overrides?: Record<string, any>) => {
recordProbe: jest.fn().mockResolvedValue(makeState()),
markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })),
reset: jest.fn(),
},
jobsRepository: {
upsertJob: jest.fn().mockResolvedValue(true),
deleteByIdAndType: jest.fn().mockResolvedValue(true),
upsertCleanupJob: jest.fn().mockResolvedValue(true),
deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true),
},
networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) },
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
logger: createMockLogger(),
...overrides,
};
@@ -59,18 +89,14 @@ const createService = (overrides?: Record<string, any>) => {
defaults.egressStateRepository as any,
defaults.jobsRepository as any,
defaults.networkService as any,
defaults.proxyResolver as any,
defaults.logger as any
);
return { service, defaults };
};
const degradedRepository = (overrides?: Record<string, any>) => ({
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
markDegraded: jest.fn().mockResolvedValue(null),
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
...overrides,
});
const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) =>
(defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
// ── Tests ────────────────────────────────────────────────────────────────────
@@ -85,7 +111,7 @@ describe("EgressService", () => {
it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => {
const { service, defaults } = createService();
await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443"]);
await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]);
const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any);
expect(monitors).toEqual([
@@ -93,9 +119,22 @@ describe("EgressService", () => {
expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }),
expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }),
expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }),
expect.objectContaining({ type: "ping", url: "2606:4700::1111" }),
]);
});
it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => {
const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128");
const { service, defaults } = createService({ proxyResolver: { resolve } });
await service.probeTargets(["https://example.com/health"]);
expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" }));
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), {
proxyUrl: "http://proxy.internal:3128",
});
});
it("reports reachable only when the provider says status is true", async () => {
const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } });
@@ -107,6 +146,15 @@ describe("EgressService", () => {
]);
});
it("counts an unparseable stored target as unreachable without probing", async () => {
const { service, defaults } = createService();
const results = await service.probeTargets(["[1.1.1.1"]);
expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]);
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
});
it("counts a thrown provider error as unreachable", async () => {
const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } });
@@ -154,9 +202,11 @@ describe("EgressService", () => {
expect.any(Date)
);
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("marks degraded, warns, inserts the recovery job and returns degraded when every target is unreachable", async () => {
it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } });
const result = await service.assessAfterFailure();
@@ -168,7 +218,8 @@ describe("EgressService", () => {
);
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith({
// The worker that performed the transition sets the schedule
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
id: "egress",
type: "egress",
refId: null,
@@ -176,9 +227,26 @@ describe("EgressService", () => {
nextScheduledAt: expect.any(Number),
intervalMs: 30_000,
});
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
});
it("returns degraded without probing when the persisted state is already degraded, re-arming the recovery job", async () => {
it("only makes sure the job row exists when another worker performed the transition", async () => {
const { service, defaults } = createService({
networkService: { requestStatus: statusFor([]) },
egressStateRepository: degradedRepository({
findSingleton: jest.fn().mockResolvedValue(makeState()),
markDegraded: jest.fn().mockResolvedValue(null),
}),
});
const result = await service.assessAfterFailure();
expect(result).toBe("degraded");
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
const { service, defaults } = createService({ egressStateRepository: degradedRepository() });
const result = await service.assessAfterFailure();
@@ -188,6 +256,7 @@ describe("EgressService", () => {
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
@@ -198,41 +267,7 @@ describe("EgressService", () => {
await service.assessAfterFailure();
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
});
it("shares one probe between concurrent failures", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
expect(results).toEqual(["ok", "ok", "ok"]);
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
});
it("reuses a settled assessment within the cache window and probes again after it", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
await service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(1000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
await jest.advanceTimersByTimeAsync(5000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
});
it("does not cache an internal failure", async () => {
const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
const { service, defaults } = createService({ settingsService: { getDBSettings } });
expect(await service.assessAfterFailure()).toBeNull();
expect(await service.assessAfterFailure()).toBe("ok");
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
});
it("falls back to the default targets when the configured list is empty", async () => {
@@ -242,8 +277,7 @@ describe("EgressService", () => {
await service.assessAfterFailure();
const urls = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
expect(urls).toEqual(["1.1.1.1", "8.8.8.8"]);
expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]);
});
it("returns null and logs when the assessment itself fails", async () => {
@@ -258,34 +292,97 @@ describe("EgressService", () => {
});
});
// ── shared assessment ─────────────────────────────────────────────────────
describe("shared assessment", () => {
it("shares one probe between concurrent failures", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
expect(results).toEqual(["ok", "ok", "ok"]);
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
});
it("keeps sharing an in-flight assessment that outlives the reuse window", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
const first = service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call
const second = service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle
expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]);
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds
});
it("reuses a settled assessment within the reuse window and probes again after it", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
await service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(1_000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
await jest.advanceTimersByTimeAsync(5_000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
});
it("reuses a disabled result instead of re-reading settings for every failure", async () => {
const getDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false }));
const { service } = createService({ settingsService: { getDBSettings } });
expect(await service.assessAfterFailure()).toBeNull();
expect(await service.assessAfterFailure()).toBeNull();
expect(getDBSettings).toHaveBeenCalledTimes(1);
});
it("does not cache an internal failure", async () => {
const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
const { service, defaults } = createService({ settingsService: { getDBSettings } });
expect(await service.assessAfterFailure()).toBeNull();
expect(await service.assessAfterFailure()).toBe("ok");
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
});
});
// ── checkRecovery (the "egress" job) ──────────────────────────────────────
describe("checkRecovery", () => {
it("marks recovered, removes the job and logs when a target becomes reachable", async () => {
it("marks recovered, releases the job and logs when a target becomes reachable", async () => {
const { service, defaults } = createService({
egressStateRepository: degradedRepository(),
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
});
const job = makeJob({ nextScheduledAt: 123_456 });
await service.checkRecovery();
await service.checkRecovery(job);
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith(
[expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
expect.any(Date)
);
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
// Conditional on the schedule this run claimed, so a concurrent new episode keeps its job
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456);
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }));
});
it("removes the job but does not log a recovery when another process already recorded it", async () => {
it("releases the job but does not log a recovery when another process already recorded it", async () => {
const { service, defaults } = createService({
egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }),
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
});
await service.checkRecovery();
await service.checkRecovery(makeJob());
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
expect(defaults.logger.info).not.toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })
);
@@ -297,21 +394,33 @@ describe("EgressService", () => {
networkService: { requestStatus: statusFor([]) },
});
await service.checkRecovery();
await service.checkRecovery(makeJob());
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled();
});
it("removes the job without probing when the persisted state is no longer degraded", async () => {
it("releases the job without probing when the persisted state is no longer degraded", async () => {
const { service, defaults } = createService();
await service.checkRecovery();
await service.checkRecovery(makeJob());
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
});
it("releases the job without probing when the feature has been disabled mid-episode", async () => {
const { service, defaults } = createService({
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
egressStateRepository: degradedRepository(),
});
await service.checkRecovery(makeJob());
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
});
it("lets errors propagate so the queue records the failure and retries", async () => {
@@ -319,7 +428,7 @@ describe("EgressService", () => {
egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }),
});
await expect(service.checkRecovery()).rejects.toThrow("db down");
await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down");
});
});
});
@@ -9,12 +9,18 @@ const createEgressStateRepo = () => ({
recordProbe: jest.fn(),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
reset: jest.fn(),
});
const createJobsRepo = () => ({
deleteByIdAndType: jest.fn().mockResolvedValue(true),
});
const createService = () => {
const egressStateRepository = createEgressStateRepo();
const service = new EgressStateService(egressStateRepository as any);
return { service, egressStateRepository };
const jobsRepository = createJobsRepo();
const service = new EgressStateService(egressStateRepository as any, jobsRepository as any);
return { service, egressStateRepository, jobsRepository };
};
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
@@ -67,4 +73,28 @@ describe("EgressStateService", () => {
await expect(service.getState()).rejects.toThrow("db down");
});
});
describe("reset", () => {
it("removes any pending recovery job and returns the state to ok", async () => {
const { service, egressStateRepository, jobsRepository } = createService();
const reset = makeState();
(egressStateRepository.reset as jest.Mock).mockResolvedValue(reset);
const result = await service.reset();
expect(jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
expect(result).toEqual(reset);
});
it("does not touch the probe-driven transitions", async () => {
const { service, egressStateRepository } = createService();
(egressStateRepository.reset as jest.Mock).mockResolvedValue(makeState());
await service.reset();
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
});
});
});
@@ -26,8 +26,10 @@ const createMonitorsRepo = () => ({
removeNotificationFromMonitors: jest.fn(),
});
const createSettingsService = (clientHost = "https://app.example.com") => ({
const createSettingsService = (clientHost = "https://app.example.com", egressNotifications: string[] = []) => ({
getSettings: jest.fn().mockReturnValue({ clientHost }),
getDBSettings: jest.fn().mockResolvedValue({ egressNotifications }),
updateDbSettings: jest.fn().mockResolvedValue({ egressNotifications: [] }),
});
const createMessageBuilder = () => ({
@@ -371,5 +373,25 @@ describe("NotificationsService", () => {
expect(result).toBe(deleted);
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
});
it("removes the notification from the egress recovered recipients in app settings", async () => {
const settingsService = createSettingsService("https://app.example.com", ["notif-0", "notif-1"]);
const { service, notificationsRepository } = createService({ settingsService });
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
await service.deleteById("notif-1", "team-1");
expect(settingsService.updateDbSettings).toHaveBeenCalledWith({ egressNotifications: ["notif-0"] });
});
it("leaves app settings alone when the notification is not an egress recipient", async () => {
const settingsService = createSettingsService("https://app.example.com", ["notif-0"]);
const { service, notificationsRepository } = createService({ settingsService });
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
await service.deleteById("notif-1", "team-1");
expect(settingsService.updateDbSettings).not.toHaveBeenCalled();
});
});
});
+24 -1
View File
@@ -41,6 +41,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
const checkService = {
deleteOlderThan: jest.fn().mockResolvedValue(0),
};
const egressService = {
checkRecovery: jest.fn().mockResolvedValue(undefined),
};
const defaults = {
logger: createMockLogger(),
@@ -54,6 +57,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
incidentsRepository,
geoChecksRepository,
dockerLogsRepository,
egressService,
...overrides,
};
@@ -68,7 +72,8 @@ const createHelper = (overrides?: Record<string, unknown>) => {
defaults.checksRepository as any,
defaults.incidentsRepository as any,
defaults.geoChecksRepository as any,
defaults.dockerLogsRepository as any
defaults.dockerLogsRepository as any,
defaults.egressService as any
);
return { helper, defaults };
};
@@ -184,4 +189,22 @@ describe("WorkerHelper", () => {
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
});
});
describe("getEgressRecoveryJob", () => {
it("returns a function that hands the claimed job to the egress service", async () => {
const { helper, defaults } = createHelper();
const job = { id: "egress", type: "egress", refId: null, nextScheduledAt: 1000 } as any;
await helper.getEgressRecoveryJob()(job);
expect(defaults.egressService.checkRecovery).toHaveBeenCalledWith(job);
});
it("lets the egress service's error propagate so the queue records the failure", async () => {
const { helper, defaults } = createHelper();
(defaults.egressService.checkRecovery as jest.Mock).mockRejectedValue(new Error("db down"));
await expect(helper.getEgressRecoveryJob()({ id: "egress" } as any)).rejects.toThrow("db down");
});
});
});
@@ -0,0 +1,49 @@
import { describe, expect, it } from "@jest/globals";
import { parseEgressTarget } from "../../../src/utils/egressTarget.ts";
describe("parseEgressTarget", () => {
it("parses http(s) URLs", () => {
expect(parseEgressTarget("https://example.com/health")).toEqual({ kind: "http", url: "https://example.com/health" });
expect(parseEgressTarget("HTTP://example.com")).toEqual({ kind: "http", url: "HTTP://example.com" });
expect(parseEgressTarget("http://[2606:4700::1111]:8080/")).toEqual({ kind: "http", url: "http://[2606:4700::1111]:8080/" });
});
it("parses host:port with a hostname, IPv4 address, localhost or bracketed IPv6 address", () => {
expect(parseEgressTarget("dns.google:53")).toEqual({ kind: "port", host: "dns.google", port: 53 });
expect(parseEgressTarget("8.8.8.8:53")).toEqual({ kind: "port", host: "8.8.8.8", port: 53 });
expect(parseEgressTarget("localhost:8080")).toEqual({ kind: "port", host: "localhost", port: 8080 });
expect(parseEgressTarget("[2606:4700::1111]:443")).toEqual({ kind: "port", host: "2606:4700::1111", port: 443 });
});
it("parses bare hosts and addresses as ping targets", () => {
expect(parseEgressTarget("1.1.1.1")).toEqual({ kind: "ping", host: "1.1.1.1" });
expect(parseEgressTarget("example.com")).toEqual({ kind: "ping", host: "example.com" });
expect(parseEgressTarget("2606:4700:4700::1111")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
expect(parseEgressTarget("[2606:4700:4700::1111]")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
expect(parseEgressTarget(" 1.1.1.1 ")).toEqual({ kind: "ping", host: "1.1.1.1" });
});
it("rejects anything the probe could not reach", () => {
for (const target of [
"",
"ftp://example.com",
"http://",
"bad host",
"1.1.1.1;rm",
"[1.1.1.1",
"1.1.1.1]",
"[::1",
"::1]",
"[[::1]]",
"[not-an-address]:53",
"[1.1.1.1]:53",
"999.1.1.1",
"1.1.1.1:0",
"1.1.1.1:70000",
"1.1.1.1:",
"host-without-tld",
]) {
expect(parseEgressTarget(target)).toBeNull();
}
});
});
@@ -34,8 +34,30 @@ describe("settingsValidation", () => {
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow();
});
it("accepts IPv6 targets, bare or bracketed with a port", () => {
const targets = ["2606:4700:4700::1111", "[2606:4700:4700::1111]", "[2606:4700:4700::1111]:53", "localhost", "localhost:8080"];
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets }).egressCheckTargets).toEqual(targets);
});
it("rejects targets that are not a host, host:port or http(s) URL", () => {
for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm"]) {
for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm", "http://"]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
}
});
it("rejects malformed addresses that the probe could never reach", () => {
for (const target of [
"[1.1.1.1",
"1.1.1.1]",
"[::1",
"::1]",
"[[::1]]",
"[not-an-address]:53",
"999.1.1.1",
"1.1.1.1:0",
"1.1.1.1:70000",
"[::1]:",
]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
}
});
@@ -59,6 +81,11 @@ describe("settingsValidation", () => {
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
});
it("accepts upper-case hex notification ids", () => {
const id = "64B7F0C2A1D2E3F4A5B6C7D8";
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [id] }).egressNotifications).toEqual([id]);
});
it("rejects notification ids that are not 24-character hex", () => {
for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow();