Harden the egress recovery job after a second review pass (#3937)
- Schedule the recovery job before recording the degraded transition,
so a crash between the two leaves a stray row (removed on the job's
first run) rather than a degraded state nothing is polling
- Count any HTTP response, including 4xx/5xx, as reachable for http
targets; only a transport failure counts against egress
- Cascade a deleted notification out of egressNotifications with an
atomic $pull in the settings repository, after the team-scoped delete
- Handle two processes racing the first egress state insert
- Address global job rows by one key (deleteGlobalJob on _id) and
restore the deleteByIdAndType signature
- Inject net into PingProvider like PortProvider
- Integration tests for MongoEgressStateRepository and the global-row
delete methods on the jobs repository
🔧
This commit is contained in:
1 parent
08f104e283
commit
b68367b04f
18 files changed
+319
-62
No files matched your search
@@ -146,6 +146,27 @@ describe("EgressService", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => {
|
||||
const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR
|
||||
message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED",
|
||||
responseTime: 3,
|
||||
}));
|
||||
const { service } = createService({ networkService: { requestStatus } });
|
||||
|
||||
const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }),
|
||||
expect.objectContaining({ target: "https://example.com/refused", reachable: false }),
|
||||
expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts
|
||||
]);
|
||||
});
|
||||
|
||||
it("counts an unparseable stored target as unreachable without probing", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
@@ -218,7 +239,6 @@ describe("EgressService", () => {
|
||||
);
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
|
||||
// The worker that performed the transition sets the schedule
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
|
||||
id: "egress",
|
||||
type: "egress",
|
||||
@@ -228,9 +248,13 @@ describe("EgressService", () => {
|
||||
intervalMs: 30_000,
|
||||
});
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
|
||||
const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
expect(scheduleOrder).toBeLessThan(transitionOrder);
|
||||
});
|
||||
|
||||
it("only makes sure the job row exists when another worker performed the transition", async () => {
|
||||
it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => {
|
||||
const { service, defaults } = createService({
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
egressStateRepository: degradedRepository({
|
||||
@@ -242,8 +266,22 @@ describe("EgressService", () => {
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
|
||||
});
|
||||
|
||||
it("does not mark degraded when the recovery job cannot be scheduled", async () => {
|
||||
const { service, defaults } = createService({
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
jobsRepository: {
|
||||
upsertJob: jest.fn(),
|
||||
upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")),
|
||||
deleteGlobalJobIfUnchanged: jest.fn(),
|
||||
},
|
||||
});
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
|
||||
|
||||
@@ -13,7 +13,7 @@ const createEgressStateRepo = () => ({
|
||||
});
|
||||
|
||||
const createJobsRepo = () => ({
|
||||
deleteByIdAndType: jest.fn().mockResolvedValue(true),
|
||||
deleteGlobalJob: jest.fn().mockResolvedValue(true),
|
||||
});
|
||||
|
||||
const createService = () => {
|
||||
@@ -82,7 +82,7 @@ describe("EgressStateService", () => {
|
||||
|
||||
const result = await service.reset();
|
||||
|
||||
expect(jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress");
|
||||
expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress");
|
||||
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual(reset);
|
||||
});
|
||||
|
||||
@@ -26,10 +26,9 @@ const createMonitorsRepo = () => ({
|
||||
removeNotificationFromMonitors: jest.fn(),
|
||||
});
|
||||
|
||||
const createSettingsService = (clientHost = "https://app.example.com", egressNotifications: string[] = []) => ({
|
||||
const createSettingsService = (clientHost = "https://app.example.com") => ({
|
||||
getSettings: jest.fn().mockReturnValue({ clientHost }),
|
||||
getDBSettings: jest.fn().mockResolvedValue({ egressNotifications }),
|
||||
updateDbSettings: jest.fn().mockResolvedValue({ egressNotifications: [] }),
|
||||
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
|
||||
});
|
||||
|
||||
const createMessageBuilder = () => ({
|
||||
@@ -374,24 +373,25 @@ describe("NotificationsService", () => {
|
||||
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
|
||||
});
|
||||
|
||||
it("removes the notification from the egress recovered recipients in app settings", async () => {
|
||||
const settingsService = createSettingsService("https://app.example.com", ["notif-0", "notif-1"]);
|
||||
const { service, notificationsRepository } = createService({ settingsService });
|
||||
it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => {
|
||||
const { service, notificationsRepository, settingsService } = createService();
|
||||
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
|
||||
|
||||
await service.deleteById("notif-1", "team-1");
|
||||
|
||||
expect(settingsService.updateDbSettings).toHaveBeenCalledWith({ egressNotifications: ["notif-0"] });
|
||||
expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1");
|
||||
const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
expect(deleteOrder).toBeLessThan(cascadeOrder);
|
||||
});
|
||||
|
||||
it("leaves app settings alone when the notification is not an egress recipient", async () => {
|
||||
const settingsService = createSettingsService("https://app.example.com", ["notif-0"]);
|
||||
const { service, notificationsRepository } = createService({ settingsService });
|
||||
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
|
||||
it("does not touch app settings when the delete is refused", async () => {
|
||||
const { service, notificationsRepository, settingsService } = createService();
|
||||
(notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found"));
|
||||
|
||||
await service.deleteById("notif-1", "team-1");
|
||||
await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found");
|
||||
|
||||
expect(settingsService.updateDbSettings).not.toHaveBeenCalled();
|
||||
expect(settingsService.removeEgressNotification).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -43,6 +43,7 @@ const createSettingsRepo = () =>
|
||||
create: jest.fn().mockResolvedValue(makeSettings()),
|
||||
findSingleton: jest.fn().mockResolvedValue(makeSettings()),
|
||||
update: jest.fn().mockResolvedValue(makeSettings()),
|
||||
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
|
||||
deleteLegacy: jest.fn().mockResolvedValue(true),
|
||||
}) as unknown as jest.Mocked<ISettingsRepository>;
|
||||
|
||||
@@ -192,4 +193,20 @@ describe("SettingsService", () => {
|
||||
await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized");
|
||||
});
|
||||
});
|
||||
|
||||
describe("removeEgressNotification", () => {
|
||||
it("delegates the pull to the repository", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
|
||||
await service.removeEgressNotification("notif-1");
|
||||
|
||||
expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1");
|
||||
});
|
||||
|
||||
it("throws when repository is not set", async () => {
|
||||
const service = new SettingsService(makeEnv());
|
||||
|
||||
await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized");
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user