Flag checks taken during degraded egress and exclude them from uptime (#3937)

CheckProducer asks EgressService to assess the instance's own connectivity
after a failing check and stamps the result on the check as egressStatus.
CheckEvaluator short-circuits degraded checks so no incident opens and no
spurious recovery fires later. Uptime and down-count aggregations exclude
degraded checks; the response-time series keeps them.

🔧
This commit is contained in:
Craig Lawson committed 2026-09-17 22:06:16 +01:00
1 parent f4864b5455
commit f52acb3f68
11 files changed
+172 -8

No files matched your search

+3 -1
View File
@@ -129,6 +129,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
const egressService = { assessAfterFailure: jest.fn().mockResolvedValue(null) };
const checkProducer = new CheckProducer(
monitorsRepo as any,
@@ -138,9 +139,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
proxyResolver as any,
bufferStub as any,
dockerLogsService as any,
egressService as any,
logger
);
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger);
const notificationReactor = new NotificationReactor(notificationsService as any);
const incidentReactor = new IncidentReactor(incidentService as any);
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
@@ -19,9 +20,10 @@ const createEvaluator = (overrides?: Record<string, any>) => {
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
},
monitorStatusPolicy: new MonitorStatusPolicy(),
logger: createMockLogger(),
...overrides,
};
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
return { evaluator, defaults };
};
@@ -84,4 +86,38 @@ describe("CheckEvaluator", () => {
shouldSendNotification: false,
});
});
// ── degraded egress ───────────────────────────────────────────────────────
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "degraded" } as any;
const monitor = makeMonitor({ status: "up" });
const { evaluator, defaults } = createEvaluator();
const result = await evaluator.evaluate(status, check, monitor);
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
expect(result.monitor).toBe(monitor);
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
expect(result.decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
});
expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
});
it("evaluates a check flagged with egress ok as normal", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "ok" } as any;
const { evaluator, defaults } = createEvaluator();
await evaluator.evaluate(status, check, makeMonitor());
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
});
});
@@ -29,6 +29,7 @@ const createProducer = (overrides?: Record<string, any>) => {
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
buffer: { addToBuffer: jest.fn() },
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
...overrides,
};
const producer = new CheckProducer(
@@ -39,6 +40,7 @@ const createProducer = (overrides?: Record<string, any>) => {
defaults.proxyResolver as any,
defaults.buffer as any,
defaults.dockerLogsService as any,
defaults.egressService as any,
defaults.logger as any
);
return { producer, defaults };
@@ -210,4 +212,62 @@ describe("CheckProducer", () => {
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
});
// ── egress self-check ─────────────────────────────────────────────────────
const failingStatus = { monitorId: "m1", status: false, code: 500, message: "Error" };
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
});
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
const result = await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
expect(result?.check.egressStatus).toBe("degraded");
});
it("flags a failing check as ok when the probe found egress fine", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
});
await producer.produce(makeMonitor());
expect(check.egressStatus).toBe("ok");
});
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check).not.toHaveProperty("egressStatus");
});
});