import { afterEach, describe, expect, it, jest } from "@jest/globals"; import { EgressService } from "../../../src/domain/egress/egress.service.ts"; import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; import type { Job } from "../../../src/domain/jobs/job.type.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── const makeState = (overrides?: Partial): EgressState => ({ id: "egress-1", status: "ok", degradedSince: null, lastRecoveredAt: null, lastProbeAt: null, lastProbeResults: [], createdAt: "2026-01-01T00:00:00.000Z", updatedAt: "2026-01-01T00:00:00.000Z", ...overrides, }); const makeSettings = (overrides?: Record) => ({ egressCheckEnabled: true, egressCheckTargets: ["1.1.1.1", "8.8.8.8"], egressPollIntervalSeconds: 30, egressNotifications: ["notif-1"], ...overrides, }); // The claimed "egress" job row as the queue hands it to the handler const makeJob = (overrides?: Partial): Job => ({ id: "egress", type: "egress", refId: null, isActive: true, nextScheduledAt: 1_000, intervalMs: 30_000, lockedBy: "worker-1", lockedUntil: 61_000, runCount: 0, failCount: 0, lastFinishedAt: null, lastFailReason: null, ...overrides, }); // requestStatus mock that answers per target url const statusFor = (reachableTargets: string[]) => jest.fn().mockImplementation(async (monitor: any) => ({ monitorId: monitor.id, teamId: monitor.teamId, type: monitor.type, status: reachableTargets.includes(monitor.url), code: reachableTargets.includes(monitor.url) ? 200 : 5000, message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed", responseTime: 12, })); const degradedRepository = (overrides?: Record) => ({ findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), markDegraded: jest.fn().mockResolvedValue(null), markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })), reset: jest.fn(), ...overrides, }); const createService = (overrides?: Record) => { const defaults = { settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings()) }, egressStateRepository: { findSingleton: jest.fn().mockResolvedValue(makeState()), recordProbe: jest.fn().mockResolvedValue(makeState()), markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })), reset: jest.fn(), }, jobsRepository: { upsertJob: jest.fn().mockResolvedValue(true), upsertCleanupJob: jest.fn().mockResolvedValue(true), deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true), }, networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) }, proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) }, logger: createMockLogger(), ...overrides, }; const service = new EgressService( defaults.settingsService as any, defaults.egressStateRepository as any, defaults.jobsRepository as any, defaults.networkService as any, defaults.proxyResolver as any, defaults.logger as any ); return { service, defaults }; }; const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) => (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url); // ── Tests ──────────────────────────────────────────────────────────────────── describe("EgressService", () => { afterEach(() => { jest.useRealTimers(); }); // ── probeTargets ────────────────────────────────────────────────────────── describe("probeTargets", () => { it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => { const { service, defaults } = createService(); await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]); const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any); expect(monitors).toEqual([ expect.objectContaining({ type: "ping", url: "1.1.1.1" }), expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }), expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }), expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }), expect.objectContaining({ type: "ping", url: "2606:4700::1111" }), ]); }); it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => { const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128"); const { service, defaults } = createService({ proxyResolver: { resolve } }); await service.probeTargets(["https://example.com/health"]); expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" })); expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), { proxyUrl: "http://proxy.internal:3128", }); }); it("reports reachable only when the provider says status is true", async () => { const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } }); const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]); expect(results).toEqual([ expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }), expect.objectContaining({ target: "8.8.8.8", reachable: false }), ]); }); it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => { const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({ monitorId: monitor.id, teamId: monitor.teamId, type: monitor.type, status: false, code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED", responseTime: 3, })); const { service } = createService({ networkService: { requestStatus } }); const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]); expect(results).toEqual([ expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }), expect.objectContaining({ target: "https://example.com/refused", reachable: false }), expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts ]); }); it("counts an unparseable stored target as unreachable without probing", async () => { const { service, defaults } = createService(); const results = await service.probeTargets(["[1.1.1.1"]); expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); }); it("counts a thrown provider error as unreachable", async () => { const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } }); const results = await service.probeTargets(["1.1.1.1"]); expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]); }); it("counts a hung provider as unreachable once the probe timeout elapses", async () => { jest.useFakeTimers(); const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } }); const pending = service.probeTargets(["1.1.1.1"]); await jest.advanceTimersByTimeAsync(5000); const results = await pending; expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]); }); }); // ── assessAfterFailure ──────────────────────────────────────────────────── describe("assessAfterFailure", () => { it("returns null and does not probe or record when the feature is disabled", async () => { const { service, defaults } = createService({ settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, }); const result = await service.assessAfterFailure(); expect(result).toBeNull(); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); }); it("returns ok and records the probe when any target is reachable", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); const result = await service.assessAfterFailure(); expect(result).toBe("ok"); expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith( [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })], expect.any(Date) ); expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled(); expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled(); }); it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } }); const result = await service.assessAfterFailure(); expect(result).toBe("degraded"); expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith( [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })], expect.any(Date) ); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") })); expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({ id: "egress", type: "egress", refId: null, isActive: true, nextScheduledAt: expect.any(Number), intervalMs: 30_000, }); expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled(); // Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!; const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!; expect(scheduleOrder).toBeLessThan(transitionOrder); }); it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) }, egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockResolvedValue(makeState()), markDegraded: jest.fn().mockResolvedValue(null), }), }); const result = await service.assessAfterFailure(); expect(result).toBe("degraded"); expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 })); expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") })); }); it("does not mark degraded when the recovery job cannot be scheduled", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) }, jobsRepository: { upsertJob: jest.fn(), upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")), deleteGlobalJobIfUnchanged: jest.fn(), }, }); expect(await service.assessAfterFailure()).toBeNull(); expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); }); it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => { const { service, defaults } = createService({ egressStateRepository: degradedRepository() }); const result = await service.assessAfterFailure(); expect(result).toBe("degraded"); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 })); expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled(); }); it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => { const { service, defaults } = createService({ settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) }, networkService: { requestStatus: statusFor([]) }, }); await service.assessAfterFailure(); expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 })); }); it("falls back to the default targets when the configured list is empty", async () => { const { service, defaults } = createService({ settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) }, }); await service.assessAfterFailure(); expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]); }); it("returns null and logs when the assessment itself fails", async () => { const { service, defaults } = createService({ settingsService: { getCachedDBSettings: jest.fn().mockRejectedValue(new Error("db down")) }, }); const result = await service.assessAfterFailure(); expect(result).toBeNull(); expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" })); }); }); // ── shared assessment ───────────────────────────────────────────────────── describe("shared assessment", () => { it("shares one probe between concurrent failures", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]); expect(results).toEqual(["ok", "ok", "ok"]); expect(defaults.settingsService.getCachedDBSettings).toHaveBeenCalledTimes(1); expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); }); it("keeps sharing an in-flight assessment that outlives the reuse window", async () => { jest.useFakeTimers(); const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } }); const first = service.assessAfterFailure(); await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call const second = service.assessAfterFailure(); await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]); expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds }); it("reuses a settled assessment within the reuse window and probes again after it", async () => { jest.useFakeTimers(); const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); await service.assessAfterFailure(); await jest.advanceTimersByTimeAsync(1_000); await service.assessAfterFailure(); expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); await jest.advanceTimersByTimeAsync(5_000); await service.assessAfterFailure(); expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); }); it("reuses a disabled result instead of re-reading settings for every failure", async () => { const getCachedDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })); const { service } = createService({ settingsService: { getCachedDBSettings } }); expect(await service.assessAfterFailure()).toBeNull(); expect(await service.assessAfterFailure()).toBeNull(); expect(getCachedDBSettings).toHaveBeenCalledTimes(1); }); it("does not cache an internal failure", async () => { const getCachedDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings()); const { service, defaults } = createService({ settingsService: { getCachedDBSettings } }); expect(await service.assessAfterFailure()).toBeNull(); expect(await service.assessAfterFailure()).toBe("ok"); expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); }); }); // ── checkRecovery (the "egress" job) ────────────────────────────────────── describe("checkRecovery", () => { it("marks recovered, releases the job and logs when a target becomes reachable", async () => { const { service, defaults } = createService({ egressStateRepository: degradedRepository(), networkService: { requestStatus: statusFor(["1.1.1.1"]) }, }); const job = makeJob({ nextScheduledAt: 123_456 }); await service.checkRecovery(job); expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith( [expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })], expect.any(Date) ); // Conditional on the schedule this run claimed, so a concurrent new episode keeps its job expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })); }); it("releases the job but does not log a recovery when another process already recorded it", async () => { const { service, defaults } = createService({ egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }), networkService: { requestStatus: statusFor(["1.1.1.1"]) }, }); await service.checkRecovery(makeJob()); expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000); expect(defaults.logger.info).not.toHaveBeenCalledWith( expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }) ); }); it("records the probe and keeps the job while every target stays unreachable", async () => { const { service, defaults } = createService({ egressStateRepository: degradedRepository(), networkService: { requestStatus: statusFor([]) }, }); await service.checkRecovery(makeJob()); expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled(); }); it("releases the job without probing when the persisted state is no longer degraded", async () => { const { service, defaults } = createService(); await service.checkRecovery(makeJob()); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000); }); it("releases the job without probing when the feature has been disabled mid-episode", async () => { const { service, defaults } = createService({ settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, egressStateRepository: degradedRepository(), }); await service.checkRecovery(makeJob()); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000); }); it("lets errors propagate so the queue records the failure and retries", async () => { const { service } = createService({ egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }), }); await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down"); }); }); });