Harden the egress recovery job after a second review pass (#3937)
- Schedule the recovery job before recording the degraded transition,
so a crash between the two leaves a stray row (removed on the job's
first run) rather than a degraded state nothing is polling
- Count any HTTP response, including 4xx/5xx, as reachable for http
targets; only a transport failure counts against egress
- Cascade a deleted notification out of egressNotifications with an
atomic $pull in the settings repository, after the team-scoped delete
- Handle two processes racing the first egress state insert
- Address global job rows by one key (deleteGlobalJob on _id) and
restore the deleteByIdAndType signature
- Inject net into PingProvider like PortProvider
- Integration tests for MongoEgressStateRepository and the global-row
delete methods on the jobs repository
🔧
This commit is contained in:
1 parent
08f104e283
commit
b68367b04f
18 files changed
+319
-62
No files matched your search
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoEgressStateRepository from "../../src/domain/egress/egress-state.repository.mongo.ts";
|
||||
import { EgressStateModel } from "../../src/domain/egress/egress-state.model.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The transitions here are the guarantees several worker processes rely on: exactly one
|
||||
// caller flips ok -> degraded (and degraded -> ok), and a plain read never writes. Both are
|
||||
// MongoDB filter/update semantics, so they need a live engine.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await EgressStateModel.init(); // unique index on singleton
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await EgressStateModel.deleteMany({});
|
||||
});
|
||||
|
||||
const probe = (target: string, reachable: boolean) => ({ target, reachable, responseTime: 5, message: reachable ? "Success" : "timeout" });
|
||||
const NOW = new Date("2026-01-01T10:00:00.000Z");
|
||||
const LATER = new Date("2026-01-01T10:05:00.000Z");
|
||||
|
||||
describe("MongoEgressStateRepository", () => {
|
||||
const repo = new MongoEgressStateRepository();
|
||||
|
||||
describe("findSingleton", () => {
|
||||
it("creates the ok singleton on first call and returns the same document afterwards", async () => {
|
||||
const first = await repo.findSingleton();
|
||||
const second = await repo.findSingleton();
|
||||
|
||||
expect(first.status).toBe("ok");
|
||||
expect(second.id).toBe(first.id);
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
|
||||
it("does not bump updatedAt on a read", async () => {
|
||||
const created = await repo.findSingleton();
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
const read = await repo.findSingleton();
|
||||
|
||||
expect(read.updatedAt).toBe(created.updatedAt);
|
||||
});
|
||||
|
||||
it("survives two processes racing the first insert", async () => {
|
||||
const [a, b] = await Promise.all([repo.findSingleton(), repo.findSingleton()]);
|
||||
|
||||
expect(a.id).toBe(b.id);
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("markDegraded / markRecovered", () => {
|
||||
it("markDegraded does nothing when the singleton has never been created (callers read it first)", async () => {
|
||||
expect(await repo.markDegraded([probe("1.1.1.1", false)], NOW)).toBeNull();
|
||||
expect(await EgressStateModel.countDocuments()).toBe(0);
|
||||
});
|
||||
|
||||
it("lets exactly one caller perform each transition", async () => {
|
||||
await repo.findSingleton();
|
||||
|
||||
const [d1, d2] = await Promise.all([repo.markDegraded([probe("1.1.1.1", false)], NOW), repo.markDegraded([probe("1.1.1.1", false)], NOW)]);
|
||||
expect([d1, d2].filter((result) => result !== null)).toHaveLength(1);
|
||||
expect((await repo.findSingleton()).status).toBe("degraded");
|
||||
|
||||
const [r1, r2] = await Promise.all([repo.markRecovered([probe("1.1.1.1", true)], LATER), repo.markRecovered([probe("1.1.1.1", true)], LATER)]);
|
||||
expect([r1, r2].filter((result) => result !== null)).toHaveLength(1);
|
||||
const state = await repo.findSingleton();
|
||||
expect(state.status).toBe("ok");
|
||||
expect(state.degradedSince).toBe(NOW.toISOString());
|
||||
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
|
||||
});
|
||||
|
||||
it("markRecovered is a no-op while ok and markDegraded is a no-op while degraded", async () => {
|
||||
await repo.findSingleton();
|
||||
expect(await repo.markRecovered([probe("1.1.1.1", true)], NOW)).toBeNull();
|
||||
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
expect(await repo.markDegraded([probe("1.1.1.1", false)], LATER)).toBeNull();
|
||||
expect((await repo.findSingleton()).degradedSince).toBe(NOW.toISOString());
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordProbe", () => {
|
||||
it("stores the probe without changing the status", async () => {
|
||||
await repo.findSingleton();
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
|
||||
const state = await repo.recordProbe([probe("1.1.1.1", false), probe("8.8.8.8", false)], LATER);
|
||||
|
||||
expect(state.status).toBe("degraded");
|
||||
expect(state.lastProbeAt).toBe(LATER.toISOString());
|
||||
expect(state.lastProbeResults.map((result) => result.target)).toEqual(["1.1.1.1", "8.8.8.8"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reset", () => {
|
||||
it("returns a degraded singleton to ok with no episode and keeps the recovery history", async () => {
|
||||
await repo.findSingleton();
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
await repo.markRecovered([probe("1.1.1.1", true)], LATER);
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], LATER);
|
||||
|
||||
const state = await repo.reset();
|
||||
|
||||
expect(state.status).toBe("ok");
|
||||
expect(state.degradedSince).toBeNull();
|
||||
expect(state.lastProbeAt).toBeNull();
|
||||
expect(state.lastProbeResults).toEqual([]);
|
||||
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
|
||||
});
|
||||
|
||||
it("creates the singleton when none exists", async () => {
|
||||
const state = await repo.reset();
|
||||
|
||||
expect(state.status).toBe("ok");
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -533,4 +533,47 @@ describe("MongoJobsRepository", () => {
|
||||
expect(await repo.countDueBacklog(NOW)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── global rows: egress recovery job ──────────────────────────────────────
|
||||
|
||||
describe("global rows", () => {
|
||||
const repo = new MongoJobsRepository(WORKER_ID);
|
||||
const egressRow = (nextScheduledAt: number) =>
|
||||
seedJob({ _id: "egress", type: "egress" as JobType, refId: null, nextScheduledAt, intervalMs: 30_000 });
|
||||
|
||||
it("deleteGlobalJob removes the row by its canonical _id and leaves other global rows alone", async () => {
|
||||
await egressRow(NOW);
|
||||
await seedJob({ _id: "cleanup-orphaned", type: "cleanup-orphaned" as JobType, refId: null });
|
||||
|
||||
expect(await repo.deleteGlobalJob("egress")).toBe(true);
|
||||
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
expect(await readRow("cleanup-orphaned")).not.toBeNull();
|
||||
expect(await repo.deleteGlobalJob("egress")).toBe(false);
|
||||
});
|
||||
|
||||
it("deleteGlobalJobIfUnchanged removes the row only while nextScheduledAt matches the claimed value", async () => {
|
||||
await egressRow(NOW);
|
||||
|
||||
// Another worker started a new episode and rescheduled the row after this run claimed it
|
||||
await JobModel.updateOne({ _id: "egress" }, { $set: { nextScheduledAt: NOW + 30_000 } });
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW)).toBe(false);
|
||||
expect(await readRow("egress")).not.toBeNull();
|
||||
|
||||
// Nobody touched it: the row goes
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW + 30_000)).toBe(true);
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
});
|
||||
|
||||
it("recordSuccess after a refused conditional delete is a no-op once the lock has moved on", async () => {
|
||||
await egressRow(NOW);
|
||||
const [claimed] = await repo.claimDueBatch("egress", 1, NOW);
|
||||
expect(claimed?.id).toBe("egress");
|
||||
|
||||
// Row deleted by the handler (recovered), then recordSuccess runs from the queue: nothing to update
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", claimed!.nextScheduledAt)).toBe(true);
|
||||
expect(await repo.recordSuccess("egress", claimed!.nextScheduledAt, 30_000, NOW + 1_000)).toBe(false);
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user