Harden the egress recovery job after a second review pass (#3937)

- Schedule the recovery job before recording the degraded transition,
  so a crash between the two leaves a stray row (removed on the job's
  first run) rather than a degraded state nothing is polling
- Count any HTTP response, including 4xx/5xx, as reachable for http
  targets; only a transport failure counts against egress
- Cascade a deleted notification out of egressNotifications with an
  atomic $pull in the settings repository, after the team-scoped delete
- Handle two processes racing the first egress state insert
- Address global job rows by one key (deleteGlobalJob on _id) and
  restore the deleteByIdAndType signature
- Inject net into PingProvider like PortProvider
- Integration tests for MongoEgressStateRepository and the global-row
  delete methods on the jobs repository

🔧
This commit is contained in:
Craig Lawson committed 2026-09-15 08:59:13 +01:00
1 parent 08f104e283
commit b68367b04f
18 files changed
+319 -62

No files matched your search

@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
import { PingProvider } from "../../../../src/service/network/PingProvider.ts";
import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts";
import type { Monitor } from "../../../../src/domain/monitors/monitor.type.ts";
import * as net from "net";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -27,7 +28,7 @@ const createMockPing = (result?: Partial<{ alive: boolean; time: number | string
// ── Contract ─────────────────────────────────────────────────────────────────
testStatusProviderContract("PingProvider", {
create: () => new PingProvider(createMockPing() as any),
create: () => new PingProvider(createMockPing() as any, net),
supportedType: "ping",
unsupportedType: "http",
makeMonitor: () => makeMonitor(),
@@ -37,7 +38,7 @@ testStatusProviderContract("PingProvider", {
describe("PingProvider", () => {
it("returns success when host is alive", async () => {
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any);
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any, net);
const result = await provider.handle(makeMonitor());
@@ -55,7 +56,7 @@ describe("PingProvider", () => {
});
it("returns failure when host is not alive", async () => {
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any);
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any, net);
const result = await provider.handle(makeMonitor());
@@ -69,7 +70,7 @@ describe("PingProvider", () => {
});
it("parses string time values", async () => {
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any);
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any, net);
const result = await provider.handle(makeMonitor());
@@ -77,7 +78,7 @@ describe("PingProvider", () => {
});
it("defaults responseTime to 0 for non-numeric time", async () => {
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any);
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any, net);
const result = await provider.handle(makeMonitor());
@@ -85,7 +86,7 @@ describe("PingProvider", () => {
});
it("defaults status to false when alive is undefined", async () => {
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any);
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any, net);
const result = await provider.handle(makeMonitor());
@@ -94,7 +95,7 @@ describe("PingProvider", () => {
it("sanitizes URL by stripping protocol and path", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any);
const provider = new PingProvider(mockPing as any, net);
await provider.handle(makeMonitor({ url: "https://example.com/path?query=1" }));
@@ -103,7 +104,7 @@ describe("PingProvider", () => {
it("sanitizes URL by stripping port", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any);
const provider = new PingProvider(mockPing as any, net);
await provider.handle(makeMonitor({ url: "example.com:8080" }));
@@ -112,7 +113,7 @@ describe("PingProvider", () => {
it("keeps a bare IPv6 address intact", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any);
const provider = new PingProvider(mockPing as any, net);
await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" }));
@@ -121,7 +122,7 @@ describe("PingProvider", () => {
it("unwraps a bracketed IPv6 address and drops its port", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any);
const provider = new PingProvider(mockPing as any, net);
await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" }));
@@ -129,28 +130,28 @@ describe("PingProvider", () => {
});
it("throws AppError when url is missing", async () => {
const provider = new PingProvider(createMockPing() as any);
const provider = new PingProvider(createMockPing() as any, net);
await expect(provider.handle(makeMonitor({ url: "" }))).rejects.toThrow("URL is required for ping monitor");
});
it("throws AppError when ping.probe rejects", async () => {
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(new Error("Host unreachable")) } };
const provider = new PingProvider(mockPing as any);
const provider = new PingProvider(mockPing as any, net);
await expect(provider.handle(makeMonitor())).rejects.toThrow("Host unreachable");
});
it("throws AppError when probe returns no response", async () => {
const mockPing = { promise: { probe: jest.fn().mockResolvedValue(null) } };
const provider = new PingProvider(mockPing as any);
const provider = new PingProvider(mockPing as any, net);
await expect(provider.handle(makeMonitor())).rejects.toThrow("No response from ping");
});
it("throws AppError with stringified message for non-Error throws", async () => {
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(42) } };
const provider = new PingProvider(mockPing as any);
const provider = new PingProvider(mockPing as any, net);
await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
});
@@ -146,6 +146,27 @@ describe("EgressService", () => {
]);
});
it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => {
const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
status: false,
code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR
message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED",
responseTime: 3,
}));
const { service } = createService({ networkService: { requestStatus } });
const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]);
expect(results).toEqual([
expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }),
expect.objectContaining({ target: "https://example.com/refused", reachable: false }),
expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts
]);
});
it("counts an unparseable stored target as unreachable without probing", async () => {
const { service, defaults } = createService();
@@ -218,7 +239,6 @@ describe("EgressService", () => {
);
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
// The worker that performed the transition sets the schedule
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
id: "egress",
type: "egress",
@@ -228,9 +248,13 @@ describe("EgressService", () => {
intervalMs: 30_000,
});
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!;
const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!;
expect(scheduleOrder).toBeLessThan(transitionOrder);
});
it("only makes sure the job row exists when another worker performed the transition", async () => {
it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => {
const { service, defaults } = createService({
networkService: { requestStatus: statusFor([]) },
egressStateRepository: degradedRepository({
@@ -242,8 +266,22 @@ describe("EgressService", () => {
const result = await service.assessAfterFailure();
expect(result).toBe("degraded");
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
});
it("does not mark degraded when the recovery job cannot be scheduled", async () => {
const { service, defaults } = createService({
networkService: { requestStatus: statusFor([]) },
jobsRepository: {
upsertJob: jest.fn(),
upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")),
deleteGlobalJobIfUnchanged: jest.fn(),
},
});
expect(await service.assessAfterFailure()).toBeNull();
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
});
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
@@ -13,7 +13,7 @@ const createEgressStateRepo = () => ({
});
const createJobsRepo = () => ({
deleteByIdAndType: jest.fn().mockResolvedValue(true),
deleteGlobalJob: jest.fn().mockResolvedValue(true),
});
const createService = () => {
@@ -82,7 +82,7 @@ describe("EgressStateService", () => {
const result = await service.reset();
expect(jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress");
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
expect(result).toEqual(reset);
});
@@ -26,10 +26,9 @@ const createMonitorsRepo = () => ({
removeNotificationFromMonitors: jest.fn(),
});
const createSettingsService = (clientHost = "https://app.example.com", egressNotifications: string[] = []) => ({
const createSettingsService = (clientHost = "https://app.example.com") => ({
getSettings: jest.fn().mockReturnValue({ clientHost }),
getDBSettings: jest.fn().mockResolvedValue({ egressNotifications }),
updateDbSettings: jest.fn().mockResolvedValue({ egressNotifications: [] }),
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
});
const createMessageBuilder = () => ({
@@ -374,24 +373,25 @@ describe("NotificationsService", () => {
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
});
it("removes the notification from the egress recovered recipients in app settings", async () => {
const settingsService = createSettingsService("https://app.example.com", ["notif-0", "notif-1"]);
const { service, notificationsRepository } = createService({ settingsService });
it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => {
const { service, notificationsRepository, settingsService } = createService();
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
await service.deleteById("notif-1", "team-1");
expect(settingsService.updateDbSettings).toHaveBeenCalledWith({ egressNotifications: ["notif-0"] });
expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1");
const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!;
const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!;
expect(deleteOrder).toBeLessThan(cascadeOrder);
});
it("leaves app settings alone when the notification is not an egress recipient", async () => {
const settingsService = createSettingsService("https://app.example.com", ["notif-0"]);
const { service, notificationsRepository } = createService({ settingsService });
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
it("does not touch app settings when the delete is refused", async () => {
const { service, notificationsRepository, settingsService } = createService();
(notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found"));
await service.deleteById("notif-1", "team-1");
await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found");
expect(settingsService.updateDbSettings).not.toHaveBeenCalled();
expect(settingsService.removeEgressNotification).not.toHaveBeenCalled();
});
});
});
@@ -43,6 +43,7 @@ const createSettingsRepo = () =>
create: jest.fn().mockResolvedValue(makeSettings()),
findSingleton: jest.fn().mockResolvedValue(makeSettings()),
update: jest.fn().mockResolvedValue(makeSettings()),
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
deleteLegacy: jest.fn().mockResolvedValue(true),
}) as unknown as jest.Mocked<ISettingsRepository>;
@@ -192,4 +193,20 @@ describe("SettingsService", () => {
await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized");
});
});
describe("removeEgressNotification", () => {
it("delegates the pull to the repository", async () => {
const { service, settingsRepository } = createService();
await service.removeEgressNotification("notif-1");
expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1");
});
it("throws when repository is not set", async () => {
const service = new SettingsService(makeEnv());
await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized");
});
});
});