Harden the egress recovery job after a second review pass (#3937)

- Schedule the recovery job before recording the degraded transition,
  so a crash between the two leaves a stray row (removed on the job's
  first run) rather than a degraded state nothing is polling
- Count any HTTP response, including 4xx/5xx, as reachable for http
  targets; only a transport failure counts against egress
- Cascade a deleted notification out of egressNotifications with an
  atomic $pull in the settings repository, after the team-scoped delete
- Handle two processes racing the first egress state insert
- Address global job rows by one key (deleteGlobalJob on _id) and
  restore the deleteByIdAndType signature
- Inject net into PingProvider like PortProvider
- Integration tests for MongoEgressStateRepository and the global-row
  delete methods on the jobs repository

🔧
This commit is contained in:
Craig Lawson committed 2026-09-15 08:59:13 +01:00
1 parent 08f104e283
commit b68367b04f
18 files changed
+319 -62

No files matched your search

@@ -146,6 +146,27 @@ describe("EgressService", () => {
]);
});
it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => {
const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
status: false,
code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR
message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED",
responseTime: 3,
}));
const { service } = createService({ networkService: { requestStatus } });
const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]);
expect(results).toEqual([
expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }),
expect.objectContaining({ target: "https://example.com/refused", reachable: false }),
expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts
]);
});
it("counts an unparseable stored target as unreachable without probing", async () => {
const { service, defaults } = createService();
@@ -218,7 +239,6 @@ describe("EgressService", () => {
);
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
// The worker that performed the transition sets the schedule
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
id: "egress",
type: "egress",
@@ -228,9 +248,13 @@ describe("EgressService", () => {
intervalMs: 30_000,
});
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!;
const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!;
expect(scheduleOrder).toBeLessThan(transitionOrder);
});
it("only makes sure the job row exists when another worker performed the transition", async () => {
it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => {
const { service, defaults } = createService({
networkService: { requestStatus: statusFor([]) },
egressStateRepository: degradedRepository({
@@ -242,8 +266,22 @@ describe("EgressService", () => {
const result = await service.assessAfterFailure();
expect(result).toBe("degraded");
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
});
it("does not mark degraded when the recovery job cannot be scheduled", async () => {
const { service, defaults } = createService({
networkService: { requestStatus: statusFor([]) },
jobsRepository: {
upsertJob: jest.fn(),
upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")),
deleteGlobalJobIfUnchanged: jest.fn(),
},
});
expect(await service.assessAfterFailure()).toBeNull();
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
});
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
@@ -13,7 +13,7 @@ const createEgressStateRepo = () => ({
});
const createJobsRepo = () => ({
deleteByIdAndType: jest.fn().mockResolvedValue(true),
deleteGlobalJob: jest.fn().mockResolvedValue(true),
});
const createService = () => {
@@ -82,7 +82,7 @@ describe("EgressStateService", () => {
const result = await service.reset();
expect(jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress");
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
expect(result).toEqual(reset);
});
@@ -26,10 +26,9 @@ const createMonitorsRepo = () => ({
removeNotificationFromMonitors: jest.fn(),
});
const createSettingsService = (clientHost = "https://app.example.com", egressNotifications: string[] = []) => ({
const createSettingsService = (clientHost = "https://app.example.com") => ({
getSettings: jest.fn().mockReturnValue({ clientHost }),
getDBSettings: jest.fn().mockResolvedValue({ egressNotifications }),
updateDbSettings: jest.fn().mockResolvedValue({ egressNotifications: [] }),
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
});
const createMessageBuilder = () => ({
@@ -374,24 +373,25 @@ describe("NotificationsService", () => {
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
});
it("removes the notification from the egress recovered recipients in app settings", async () => {
const settingsService = createSettingsService("https://app.example.com", ["notif-0", "notif-1"]);
const { service, notificationsRepository } = createService({ settingsService });
it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => {
const { service, notificationsRepository, settingsService } = createService();
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
await service.deleteById("notif-1", "team-1");
expect(settingsService.updateDbSettings).toHaveBeenCalledWith({ egressNotifications: ["notif-0"] });
expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1");
const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!;
const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!;
expect(deleteOrder).toBeLessThan(cascadeOrder);
});
it("leaves app settings alone when the notification is not an egress recipient", async () => {
const settingsService = createSettingsService("https://app.example.com", ["notif-0"]);
const { service, notificationsRepository } = createService({ settingsService });
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
it("does not touch app settings when the delete is refused", async () => {
const { service, notificationsRepository, settingsService } = createService();
(notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found"));
await service.deleteById("notif-1", "team-1");
await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found");
expect(settingsService.updateDbSettings).not.toHaveBeenCalled();
expect(settingsService.removeEgressNotification).not.toHaveBeenCalled();
});
});
});
@@ -43,6 +43,7 @@ const createSettingsRepo = () =>
create: jest.fn().mockResolvedValue(makeSettings()),
findSingleton: jest.fn().mockResolvedValue(makeSettings()),
update: jest.fn().mockResolvedValue(makeSettings()),
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
deleteLegacy: jest.fn().mockResolvedValue(true),
}) as unknown as jest.Mocked<ISettingsRepository>;
@@ -192,4 +193,20 @@ describe("SettingsService", () => {
await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized");
});
});
describe("removeEgressNotification", () => {
it("delegates the pull to the repository", async () => {
const { service, settingsRepository } = createService();
await service.removeEgressNotification("notif-1");
expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1");
});
it("throws when repository is not set", async () => {
const service = new SettingsService(makeEnv());
await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized");
});
});
});