Merge pull request #3959 from web-engineer/feat/egress-1-check-service
Egress self-check (1/4): egress state and probe service (#3937)
This commit is contained in:
38 files changed
+1701
-28
No files matched your search
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoEgressStateRepository from "../../src/domain/egress/egress-state.repository.mongo.ts";
|
||||
import { EgressStateModel } from "../../src/domain/egress/egress-state.model.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The transitions here are the guarantees several worker processes rely on: exactly one
|
||||
// caller flips ok -> degraded (and degraded -> ok), and a plain read never writes. Both are
|
||||
// MongoDB filter/update semantics, so they need a live engine.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await EgressStateModel.init(); // unique index on singleton
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await EgressStateModel.deleteMany({});
|
||||
});
|
||||
|
||||
const probe = (target: string, reachable: boolean) => ({ target, reachable, responseTime: 5, message: reachable ? "Success" : "timeout" });
|
||||
const NOW = new Date("2026-01-01T10:00:00.000Z");
|
||||
const LATER = new Date("2026-01-01T10:05:00.000Z");
|
||||
|
||||
describe("MongoEgressStateRepository", () => {
|
||||
const repo = new MongoEgressStateRepository();
|
||||
|
||||
describe("findSingleton", () => {
|
||||
it("creates the ok singleton on first call and returns the same document afterwards", async () => {
|
||||
const first = await repo.findSingleton();
|
||||
const second = await repo.findSingleton();
|
||||
|
||||
expect(first.status).toBe("ok");
|
||||
expect(second.id).toBe(first.id);
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
|
||||
it("does not bump updatedAt on a read", async () => {
|
||||
const created = await repo.findSingleton();
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
const read = await repo.findSingleton();
|
||||
|
||||
expect(read.updatedAt).toBe(created.updatedAt);
|
||||
});
|
||||
|
||||
it("survives two processes racing the first insert", async () => {
|
||||
const [a, b] = await Promise.all([repo.findSingleton(), repo.findSingleton()]);
|
||||
|
||||
expect(a.id).toBe(b.id);
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("markDegraded / markRecovered", () => {
|
||||
it("markDegraded does nothing when the singleton has never been created (callers read it first)", async () => {
|
||||
expect(await repo.markDegraded([probe("1.1.1.1", false)], NOW)).toBeNull();
|
||||
expect(await EgressStateModel.countDocuments()).toBe(0);
|
||||
});
|
||||
|
||||
it("lets exactly one caller perform each transition", async () => {
|
||||
await repo.findSingleton();
|
||||
|
||||
const [d1, d2] = await Promise.all([repo.markDegraded([probe("1.1.1.1", false)], NOW), repo.markDegraded([probe("1.1.1.1", false)], NOW)]);
|
||||
expect([d1, d2].filter((result) => result !== null)).toHaveLength(1);
|
||||
expect((await repo.findSingleton()).status).toBe("degraded");
|
||||
|
||||
const [r1, r2] = await Promise.all([repo.markRecovered([probe("1.1.1.1", true)], LATER), repo.markRecovered([probe("1.1.1.1", true)], LATER)]);
|
||||
expect([r1, r2].filter((result) => result !== null)).toHaveLength(1);
|
||||
const state = await repo.findSingleton();
|
||||
expect(state.status).toBe("ok");
|
||||
expect(state.degradedSince).toBe(NOW.toISOString());
|
||||
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
|
||||
});
|
||||
|
||||
it("markRecovered is a no-op while ok and markDegraded is a no-op while degraded", async () => {
|
||||
await repo.findSingleton();
|
||||
expect(await repo.markRecovered([probe("1.1.1.1", true)], NOW)).toBeNull();
|
||||
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
expect(await repo.markDegraded([probe("1.1.1.1", false)], LATER)).toBeNull();
|
||||
expect((await repo.findSingleton()).degradedSince).toBe(NOW.toISOString());
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordProbe", () => {
|
||||
it("stores the probe without changing the status", async () => {
|
||||
await repo.findSingleton();
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
|
||||
const state = await repo.recordProbe([probe("1.1.1.1", false), probe("8.8.8.8", false)], LATER);
|
||||
|
||||
expect(state.status).toBe("degraded");
|
||||
expect(state.lastProbeAt).toBe(LATER.toISOString());
|
||||
expect(state.lastProbeResults.map((result) => result.target)).toEqual(["1.1.1.1", "8.8.8.8"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reset", () => {
|
||||
it("returns a degraded singleton to ok with no episode and keeps the recovery history", async () => {
|
||||
await repo.findSingleton();
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
await repo.markRecovered([probe("1.1.1.1", true)], LATER);
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], LATER);
|
||||
|
||||
const state = await repo.reset();
|
||||
|
||||
expect(state.status).toBe("ok");
|
||||
expect(state.degradedSince).toBeNull();
|
||||
expect(state.lastProbeAt).toBeNull();
|
||||
expect(state.lastProbeResults).toEqual([]);
|
||||
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
|
||||
});
|
||||
|
||||
it("creates the singleton when none exists", async () => {
|
||||
const state = await repo.reset();
|
||||
|
||||
expect(state.status).toBe("ok");
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -533,4 +533,47 @@ describe("MongoJobsRepository", () => {
|
||||
expect(await repo.countDueBacklog(NOW)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── global rows: egress recovery job ──────────────────────────────────────
|
||||
|
||||
describe("global rows", () => {
|
||||
const repo = new MongoJobsRepository(WORKER_ID);
|
||||
const egressRow = (nextScheduledAt: number) =>
|
||||
seedJob({ _id: "egress", type: "egress" as JobType, refId: null, nextScheduledAt, intervalMs: 30_000 });
|
||||
|
||||
it("deleteGlobalJob removes the row by its canonical _id and leaves other global rows alone", async () => {
|
||||
await egressRow(NOW);
|
||||
await seedJob({ _id: "cleanup-orphaned", type: "cleanup-orphaned" as JobType, refId: null });
|
||||
|
||||
expect(await repo.deleteGlobalJob("egress")).toBe(true);
|
||||
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
expect(await readRow("cleanup-orphaned")).not.toBeNull();
|
||||
expect(await repo.deleteGlobalJob("egress")).toBe(false);
|
||||
});
|
||||
|
||||
it("deleteGlobalJobIfUnchanged removes the row only while nextScheduledAt matches the claimed value", async () => {
|
||||
await egressRow(NOW);
|
||||
|
||||
// Another worker started a new episode and rescheduled the row after this run claimed it
|
||||
await JobModel.updateOne({ _id: "egress" }, { $set: { nextScheduledAt: NOW + 30_000 } });
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW)).toBe(false);
|
||||
expect(await readRow("egress")).not.toBeNull();
|
||||
|
||||
// Nobody touched it: the row goes
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW + 30_000)).toBe(true);
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
});
|
||||
|
||||
it("recordSuccess after a refused conditional delete is a no-op once the lock has moved on", async () => {
|
||||
await egressRow(NOW);
|
||||
const [claimed] = await repo.claimDueBatch("egress", 1, NOW);
|
||||
expect(claimed?.id).toBe("egress");
|
||||
|
||||
// Row deleted by the handler (recovered), then recordSuccess runs from the queue: nothing to update
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", claimed!.nextScheduledAt)).toBe(true);
|
||||
expect(await repo.recordSuccess("egress", claimed!.nextScheduledAt, 30_000, NOW + 1_000)).toBe(false);
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { PingProvider } from "../../../../src/service/network/PingProvider.ts";
|
||||
import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts";
|
||||
import type { Monitor } from "../../../../src/domain/monitors/monitor.type.ts";
|
||||
import * as net from "net";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -27,7 +28,7 @@ const createMockPing = (result?: Partial<{ alive: boolean; time: number | string
|
||||
// ── Contract ─────────────────────────────────────────────────────────────────
|
||||
|
||||
testStatusProviderContract("PingProvider", {
|
||||
create: () => new PingProvider(createMockPing() as any),
|
||||
create: () => new PingProvider(createMockPing() as any, net),
|
||||
supportedType: "ping",
|
||||
unsupportedType: "http",
|
||||
makeMonitor: () => makeMonitor(),
|
||||
@@ -37,7 +38,7 @@ testStatusProviderContract("PingProvider", {
|
||||
|
||||
describe("PingProvider", () => {
|
||||
it("returns success when host is alive", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -55,7 +56,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("returns failure when host is not alive", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -69,7 +70,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("parses string time values", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -77,7 +78,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("defaults responseTime to 0 for non-numeric time", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -85,7 +86,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("defaults status to false when alive is undefined", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -94,7 +95,7 @@ describe("PingProvider", () => {
|
||||
|
||||
it("sanitizes URL by stripping protocol and path", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "https://example.com/path?query=1" }));
|
||||
|
||||
@@ -103,36 +104,54 @@ describe("PingProvider", () => {
|
||||
|
||||
it("sanitizes URL by stripping port", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "example.com:8080" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("example.com");
|
||||
});
|
||||
|
||||
it("keeps a bare IPv6 address intact", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
|
||||
});
|
||||
|
||||
it("unwraps a bracketed IPv6 address and drops its port", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
|
||||
});
|
||||
|
||||
it("throws AppError when url is missing", async () => {
|
||||
const provider = new PingProvider(createMockPing() as any);
|
||||
const provider = new PingProvider(createMockPing() as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor({ url: "" }))).rejects.toThrow("URL is required for ping monitor");
|
||||
});
|
||||
|
||||
it("throws AppError when ping.probe rejects", async () => {
|
||||
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(new Error("Host unreachable")) } };
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("Host unreachable");
|
||||
});
|
||||
|
||||
it("throws AppError when probe returns no response", async () => {
|
||||
const mockPing = { promise: { probe: jest.fn().mockResolvedValue(null) } };
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("No response from ping");
|
||||
});
|
||||
|
||||
it("throws AppError with stringified message for non-Error throws", async () => {
|
||||
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(42) } };
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
|
||||
});
|
||||
|
||||
@@ -9,7 +9,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
// ── Notes ──────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
|
||||
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
|
||||
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and
|
||||
// renews their locks, and registers itself in the worker heartbeat. The repository
|
||||
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
|
||||
// test only the orchestration around it, with the repository fully mocked.
|
||||
@@ -113,6 +113,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const queueWorkersRepository = {
|
||||
upsert: jest.fn<any>().mockResolvedValue(undefined),
|
||||
@@ -226,7 +227,7 @@ describe("DBQueueWorker", () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
|
||||
const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
|
||||
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
|
||||
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"]));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
|
||||
@@ -440,6 +441,20 @@ describe("DBQueueWorker", () => {
|
||||
expect(retentionFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("egress job runs the helper's egress recovery function with the claimed job", async () => {
|
||||
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
|
||||
const egressFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
});
|
||||
|
||||
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
|
||||
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
import { afterEach, describe, expect, it, jest } from "@jest/globals";
|
||||
import { EgressService } from "../../../src/domain/egress/egress.service.ts";
|
||||
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
|
||||
import type { Job } from "../../../src/domain/jobs/job.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
id: "egress-1",
|
||||
status: "ok",
|
||||
degradedSince: null,
|
||||
lastRecoveredAt: null,
|
||||
lastProbeAt: null,
|
||||
lastProbeResults: [],
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const makeSettings = (overrides?: Record<string, unknown>) => ({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: ["notif-1"],
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// The claimed "egress" job row as the queue hands it to the handler
|
||||
const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
id: "egress",
|
||||
type: "egress",
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: 1_000,
|
||||
intervalMs: 30_000,
|
||||
lockedBy: "worker-1",
|
||||
lockedUntil: 61_000,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// requestStatus mock that answers per target url
|
||||
const statusFor = (reachableTargets: string[]) =>
|
||||
jest.fn().mockImplementation(async (monitor: any) => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: reachableTargets.includes(monitor.url),
|
||||
code: reachableTargets.includes(monitor.url) ? 200 : 5000,
|
||||
message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed",
|
||||
responseTime: 12,
|
||||
}));
|
||||
|
||||
const degradedRepository = (overrides?: Record<string, any>) => ({
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
|
||||
markDegraded: jest.fn().mockResolvedValue(null),
|
||||
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
reset: jest.fn(),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const createService = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) },
|
||||
egressStateRepository: {
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState()),
|
||||
recordProbe: jest.fn().mockResolvedValue(makeState()),
|
||||
markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })),
|
||||
reset: jest.fn(),
|
||||
},
|
||||
jobsRepository: {
|
||||
upsertJob: jest.fn().mockResolvedValue(true),
|
||||
upsertCleanupJob: jest.fn().mockResolvedValue(true),
|
||||
deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true),
|
||||
},
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) },
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
|
||||
logger: createMockLogger(),
|
||||
...overrides,
|
||||
};
|
||||
const service = new EgressService(
|
||||
defaults.settingsService as any,
|
||||
defaults.egressStateRepository as any,
|
||||
defaults.jobsRepository as any,
|
||||
defaults.networkService as any,
|
||||
defaults.proxyResolver as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { service, defaults };
|
||||
};
|
||||
|
||||
const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) =>
|
||||
(defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("EgressService", () => {
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
// ── probeTargets ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("probeTargets", () => {
|
||||
it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]);
|
||||
|
||||
const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any);
|
||||
expect(monitors).toEqual([
|
||||
expect.objectContaining({ type: "ping", url: "1.1.1.1" }),
|
||||
expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }),
|
||||
expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }),
|
||||
expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }),
|
||||
expect.objectContaining({ type: "ping", url: "2606:4700::1111" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => {
|
||||
const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128");
|
||||
const { service, defaults } = createService({ proxyResolver: { resolve } });
|
||||
|
||||
await service.probeTargets(["https://example.com/health"]);
|
||||
|
||||
expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" }));
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), {
|
||||
proxyUrl: "http://proxy.internal:3128",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports reachable only when the provider says status is true", async () => {
|
||||
const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } });
|
||||
|
||||
const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }),
|
||||
expect.objectContaining({ target: "8.8.8.8", reachable: false }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => {
|
||||
const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR
|
||||
message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED",
|
||||
responseTime: 3,
|
||||
}));
|
||||
const { service } = createService({ networkService: { requestStatus } });
|
||||
|
||||
const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }),
|
||||
expect.objectContaining({ target: "https://example.com/refused", reachable: false }),
|
||||
expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts
|
||||
]);
|
||||
});
|
||||
|
||||
it("counts an unparseable stored target as unreachable without probing", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
const results = await service.probeTargets(["[1.1.1.1"]);
|
||||
|
||||
expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]);
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("counts a thrown provider error as unreachable", async () => {
|
||||
const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } });
|
||||
|
||||
const results = await service.probeTargets(["1.1.1.1"]);
|
||||
|
||||
expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]);
|
||||
});
|
||||
|
||||
it("counts a hung provider as unreachable once the probe timeout elapses", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
|
||||
|
||||
const pending = service.probeTargets(["1.1.1.1"]);
|
||||
await jest.advanceTimersByTimeAsync(5000);
|
||||
const results = await pending;
|
||||
|
||||
expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── assessAfterFailure ────────────────────────────────────────────────────
|
||||
|
||||
describe("assessAfterFailure", () => {
|
||||
it("returns null and does not probe or record when the feature is disabled", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
|
||||
});
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns ok and records the probe when any target is reachable", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("ok");
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith(
|
||||
[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })],
|
||||
expect.any(Date)
|
||||
);
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith(
|
||||
[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
|
||||
expect.any(Date)
|
||||
);
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
|
||||
id: "egress",
|
||||
type: "egress",
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: expect.any(Number),
|
||||
intervalMs: 30_000,
|
||||
});
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
|
||||
const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
expect(scheduleOrder).toBeLessThan(transitionOrder);
|
||||
});
|
||||
|
||||
it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => {
|
||||
const { service, defaults } = createService({
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
egressStateRepository: degradedRepository({
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState()),
|
||||
markDegraded: jest.fn().mockResolvedValue(null),
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
|
||||
});
|
||||
|
||||
it("does not mark degraded when the recovery job cannot be scheduled", async () => {
|
||||
const { service, defaults } = createService({
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
jobsRepository: {
|
||||
upsertJob: jest.fn(),
|
||||
upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")),
|
||||
deleteGlobalJobIfUnchanged: jest.fn(),
|
||||
},
|
||||
});
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
|
||||
const { service, defaults } = createService({ egressStateRepository: degradedRepository() });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
|
||||
});
|
||||
|
||||
it("falls back to the default targets when the configured list is empty", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
|
||||
});
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]);
|
||||
});
|
||||
|
||||
it("returns null and logs when the assessment itself fails", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getDBSettings: jest.fn().mockRejectedValue(new Error("db down")) },
|
||||
});
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── shared assessment ─────────────────────────────────────────────────────
|
||||
|
||||
describe("shared assessment", () => {
|
||||
it("shares one probe between concurrent failures", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
|
||||
|
||||
expect(results).toEqual(["ok", "ok", "ok"]);
|
||||
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps sharing an in-flight assessment that outlives the reuse window", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
|
||||
|
||||
const first = service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call
|
||||
const second = service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle
|
||||
|
||||
expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]);
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds
|
||||
});
|
||||
|
||||
it("reuses a settled assessment within the reuse window and probes again after it", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
await service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(5_000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("reuses a disabled result instead of re-reading settings for every failure", async () => {
|
||||
const getDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false }));
|
||||
const { service } = createService({ settingsService: { getDBSettings } });
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
|
||||
expect(getDBSettings).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not cache an internal failure", async () => {
|
||||
const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
|
||||
const { service, defaults } = createService({ settingsService: { getDBSettings } });
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(await service.assessAfterFailure()).toBe("ok");
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── checkRecovery (the "egress" job) ──────────────────────────────────────
|
||||
|
||||
describe("checkRecovery", () => {
|
||||
it("marks recovered, releases the job and logs when a target becomes reachable", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository(),
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
|
||||
});
|
||||
const job = makeJob({ nextScheduledAt: 123_456 });
|
||||
|
||||
await service.checkRecovery(job);
|
||||
|
||||
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith(
|
||||
[expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
|
||||
expect.any(Date)
|
||||
);
|
||||
// Conditional on the schedule this run claimed, so a concurrent new episode keeps its job
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456);
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }));
|
||||
});
|
||||
|
||||
it("releases the job but does not log a recovery when another process already recorded it", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }),
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
|
||||
});
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
expect(defaults.logger.info).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })
|
||||
);
|
||||
});
|
||||
|
||||
it("records the probe and keeps the job while every target stays unreachable", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository(),
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("releases the job without probing when the persisted state is no longer degraded", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
});
|
||||
|
||||
it("releases the job without probing when the feature has been disabled mid-episode", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
|
||||
egressStateRepository: degradedRepository(),
|
||||
});
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
});
|
||||
|
||||
it("lets errors propagate so the queue records the failure and retries", async () => {
|
||||
const { service } = createService({
|
||||
egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }),
|
||||
});
|
||||
|
||||
await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { EgressStateService } from "../../../src/domain/egress/egress-state.service.ts";
|
||||
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const createEgressStateRepo = () => ({
|
||||
findSingleton: jest.fn(),
|
||||
recordProbe: jest.fn(),
|
||||
markDegraded: jest.fn(),
|
||||
markRecovered: jest.fn(),
|
||||
reset: jest.fn(),
|
||||
});
|
||||
|
||||
const createJobsRepo = () => ({
|
||||
deleteGlobalJob: jest.fn().mockResolvedValue(true),
|
||||
});
|
||||
|
||||
const createService = () => {
|
||||
const egressStateRepository = createEgressStateRepo();
|
||||
const jobsRepository = createJobsRepo();
|
||||
const service = new EgressStateService(egressStateRepository as any, jobsRepository as any);
|
||||
return { service, egressStateRepository, jobsRepository };
|
||||
};
|
||||
|
||||
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
id: "egress-1",
|
||||
status: "ok",
|
||||
degradedSince: null,
|
||||
lastRecoveredAt: null,
|
||||
lastProbeAt: null,
|
||||
lastProbeResults: [],
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("EgressStateService", () => {
|
||||
describe("getState", () => {
|
||||
it("returns the singleton state from the repository", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
const state = makeState({
|
||||
status: "degraded",
|
||||
degradedSince: "2026-01-01T00:05:00Z",
|
||||
lastProbeAt: "2026-01-01T00:05:00Z",
|
||||
lastProbeResults: [{ target: "1.1.1.1", reachable: false, responseTime: 0, message: "timeout" }],
|
||||
});
|
||||
(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(state);
|
||||
|
||||
const result = await service.getState();
|
||||
|
||||
expect(egressStateRepository.findSingleton).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual(state);
|
||||
});
|
||||
|
||||
it("does not call any transition method", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(makeState());
|
||||
|
||||
await service.getState();
|
||||
|
||||
expect(egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("propagates repository errors", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
(egressStateRepository.findSingleton as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
await expect(service.getState()).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
|
||||
describe("reset", () => {
|
||||
it("removes any pending recovery job and returns the state to ok", async () => {
|
||||
const { service, egressStateRepository, jobsRepository } = createService();
|
||||
const reset = makeState();
|
||||
(egressStateRepository.reset as jest.Mock).mockResolvedValue(reset);
|
||||
|
||||
const result = await service.reset();
|
||||
|
||||
expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress");
|
||||
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual(reset);
|
||||
});
|
||||
|
||||
it("does not touch the probe-driven transitions", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
(egressStateRepository.reset as jest.Mock).mockResolvedValue(makeState());
|
||||
|
||||
await service.reset();
|
||||
|
||||
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -28,6 +28,7 @@ const createMonitorsRepo = () => ({
|
||||
|
||||
const createSettingsService = (clientHost = "https://app.example.com") => ({
|
||||
getSettings: jest.fn().mockReturnValue({ clientHost }),
|
||||
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
|
||||
});
|
||||
|
||||
const createMessageBuilder = () => ({
|
||||
@@ -371,5 +372,26 @@ describe("NotificationsService", () => {
|
||||
expect(result).toBe(deleted);
|
||||
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
|
||||
});
|
||||
|
||||
it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => {
|
||||
const { service, notificationsRepository, settingsService } = createService();
|
||||
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
|
||||
|
||||
await service.deleteById("notif-1", "team-1");
|
||||
|
||||
expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1");
|
||||
const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
expect(deleteOrder).toBeLessThan(cascadeOrder);
|
||||
});
|
||||
|
||||
it("does not touch app settings when the delete is refused", async () => {
|
||||
const { service, notificationsRepository, settingsService } = createService();
|
||||
(notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found"));
|
||||
|
||||
await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found");
|
||||
|
||||
expect(settingsService.removeEgressNotification).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -43,6 +43,7 @@ const createSettingsRepo = () =>
|
||||
create: jest.fn().mockResolvedValue(makeSettings()),
|
||||
findSingleton: jest.fn().mockResolvedValue(makeSettings()),
|
||||
update: jest.fn().mockResolvedValue(makeSettings()),
|
||||
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
|
||||
deleteLegacy: jest.fn().mockResolvedValue(true),
|
||||
}) as unknown as jest.Mocked<ISettingsRepository>;
|
||||
|
||||
@@ -192,4 +193,20 @@ describe("SettingsService", () => {
|
||||
await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized");
|
||||
});
|
||||
});
|
||||
|
||||
describe("removeEgressNotification", () => {
|
||||
it("delegates the pull to the repository", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
|
||||
await service.removeEgressNotification("notif-1");
|
||||
|
||||
expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1");
|
||||
});
|
||||
|
||||
it("throws when repository is not set", async () => {
|
||||
const service = new SettingsService(makeEnv());
|
||||
|
||||
await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -41,6 +41,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
const checkService = {
|
||||
deleteOlderThan: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
const egressService = {
|
||||
checkRecovery: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
@@ -54,6 +57,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
incidentsRepository,
|
||||
geoChecksRepository,
|
||||
dockerLogsRepository,
|
||||
egressService,
|
||||
...overrides,
|
||||
};
|
||||
|
||||
@@ -68,7 +72,8 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
defaults.checksRepository as any,
|
||||
defaults.incidentsRepository as any,
|
||||
defaults.geoChecksRepository as any,
|
||||
defaults.dockerLogsRepository as any
|
||||
defaults.dockerLogsRepository as any,
|
||||
defaults.egressService as any
|
||||
);
|
||||
return { helper, defaults };
|
||||
};
|
||||
@@ -184,4 +189,22 @@ describe("WorkerHelper", () => {
|
||||
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
|
||||
});
|
||||
});
|
||||
|
||||
describe("getEgressRecoveryJob", () => {
|
||||
it("returns a function that hands the claimed job to the egress service", async () => {
|
||||
const { helper, defaults } = createHelper();
|
||||
const job = { id: "egress", type: "egress", refId: null, nextScheduledAt: 1000 } as any;
|
||||
|
||||
await helper.getEgressRecoveryJob()(job);
|
||||
|
||||
expect(defaults.egressService.checkRecovery).toHaveBeenCalledWith(job);
|
||||
});
|
||||
|
||||
it("lets the egress service's error propagate so the queue records the failure", async () => {
|
||||
const { helper, defaults } = createHelper();
|
||||
(defaults.egressService.checkRecovery as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
await expect(helper.getEgressRecoveryJob()({ id: "egress" } as any)).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { parseEgressTarget } from "../../../src/utils/egressTarget.ts";
|
||||
|
||||
describe("parseEgressTarget", () => {
|
||||
it("parses http(s) URLs", () => {
|
||||
expect(parseEgressTarget("https://example.com/health")).toEqual({ kind: "http", url: "https://example.com/health" });
|
||||
expect(parseEgressTarget("HTTP://example.com")).toEqual({ kind: "http", url: "HTTP://example.com" });
|
||||
expect(parseEgressTarget("http://[2606:4700::1111]:8080/")).toEqual({ kind: "http", url: "http://[2606:4700::1111]:8080/" });
|
||||
});
|
||||
|
||||
it("parses host:port with a hostname, IPv4 address, localhost or bracketed IPv6 address", () => {
|
||||
expect(parseEgressTarget("dns.google:53")).toEqual({ kind: "port", host: "dns.google", port: 53 });
|
||||
expect(parseEgressTarget("8.8.8.8:53")).toEqual({ kind: "port", host: "8.8.8.8", port: 53 });
|
||||
expect(parseEgressTarget("localhost:8080")).toEqual({ kind: "port", host: "localhost", port: 8080 });
|
||||
expect(parseEgressTarget("[2606:4700::1111]:443")).toEqual({ kind: "port", host: "2606:4700::1111", port: 443 });
|
||||
});
|
||||
|
||||
it("parses bare hosts and addresses as ping targets", () => {
|
||||
expect(parseEgressTarget("1.1.1.1")).toEqual({ kind: "ping", host: "1.1.1.1" });
|
||||
expect(parseEgressTarget("example.com")).toEqual({ kind: "ping", host: "example.com" });
|
||||
expect(parseEgressTarget("2606:4700:4700::1111")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
|
||||
expect(parseEgressTarget("[2606:4700:4700::1111]")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
|
||||
expect(parseEgressTarget(" 1.1.1.1 ")).toEqual({ kind: "ping", host: "1.1.1.1" });
|
||||
});
|
||||
|
||||
it("rejects anything the probe could not reach", () => {
|
||||
for (const target of [
|
||||
"",
|
||||
"ftp://example.com",
|
||||
"http://",
|
||||
"bad host",
|
||||
"1.1.1.1;rm",
|
||||
"[1.1.1.1",
|
||||
"1.1.1.1]",
|
||||
"[::1",
|
||||
"::1]",
|
||||
"[[::1]]",
|
||||
"[not-an-address]:53",
|
||||
"[1.1.1.1]:53",
|
||||
"999.1.1.1",
|
||||
"1.1.1.1:0",
|
||||
"1.1.1.1:70000",
|
||||
"1.1.1.1:",
|
||||
"host-without-tld",
|
||||
]) {
|
||||
expect(parseEgressTarget(target)).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts";
|
||||
|
||||
const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8";
|
||||
|
||||
describe("settingsValidation", () => {
|
||||
describe("updateAppSettingsBodyValidation egress fields", () => {
|
||||
it("accepts a full set of valid egress settings", () => {
|
||||
const parsed = updateAppSettingsBodyValidation.parse({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: [validNotificationId],
|
||||
});
|
||||
|
||||
expect(parsed.egressCheckEnabled).toBe(true);
|
||||
expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]);
|
||||
expect(parsed.egressPollIntervalSeconds).toBe(30);
|
||||
expect(parsed.egressNotifications).toEqual([validNotificationId]);
|
||||
});
|
||||
|
||||
it("accepts a body with no egress fields", () => {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ checkTTL: 30 })).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts an empty target list (the worker falls back to the default targets)", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: [] }).egressCheckTargets).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects more targets than the maximum", () => {
|
||||
const targets = Array.from({ length: MAX_EGRESS_TARGETS + 1 }, (_, i) => `10.0.0.${i}`);
|
||||
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow();
|
||||
});
|
||||
|
||||
it("accepts IPv6 targets, bare or bracketed with a port", () => {
|
||||
const targets = ["2606:4700:4700::1111", "[2606:4700:4700::1111]", "[2606:4700:4700::1111]:53", "localhost", "localhost:8080"];
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets }).egressCheckTargets).toEqual(targets);
|
||||
});
|
||||
|
||||
it("rejects targets that are not a host, host:port or http(s) URL", () => {
|
||||
for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm", "http://"]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects malformed addresses that the probe could never reach", () => {
|
||||
for (const target of [
|
||||
"[1.1.1.1",
|
||||
"1.1.1.1]",
|
||||
"[::1",
|
||||
"::1]",
|
||||
"[[::1]]",
|
||||
"[not-an-address]:53",
|
||||
"999.1.1.1",
|
||||
"1.1.1.1:0",
|
||||
"1.1.1.1:70000",
|
||||
"[::1]:",
|
||||
]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts the poll interval boundaries", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MIN_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MAX_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects out-of-range and non-integer poll intervals", () => {
|
||||
for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts an empty notification list", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
|
||||
});
|
||||
|
||||
it("accepts upper-case hex notification ids", () => {
|
||||
const id = "64B7F0C2A1D2E3F4A5B6C7D8";
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [id] }).egressNotifications).toEqual([id]);
|
||||
});
|
||||
|
||||
it("rejects notification ids that are not 24-character hex", () => {
|
||||
for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a non-boolean enabled flag", () => {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckEnabled: "yes" })).toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user