Flag checks taken during degraded egress and exclude them from uptime (#3937)
CheckProducer asks EgressService to assess the instance's own connectivity
after a failing check and stamps the result on the check as egressStatus.
CheckEvaluator short-circuits degraded checks so no incident opens and no
spurious recovery fires later. Uptime and down-count aggregations exclude
degraded checks; the response-time series keeps them.
🔧
This commit is contained in:
1 parent
f4864b5455
commit
f52acb3f68
11 files changed
+172
-8
No files matched your search
@@ -129,6 +129,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
|
||||
const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
|
||||
const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
|
||||
const egressService = { assessAfterFailure: jest.fn().mockResolvedValue(null) };
|
||||
|
||||
const checkProducer = new CheckProducer(
|
||||
monitorsRepo as any,
|
||||
@@ -138,9 +139,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
proxyResolver as any,
|
||||
bufferStub as any,
|
||||
dockerLogsService as any,
|
||||
egressService as any,
|
||||
logger
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger);
|
||||
|
||||
const notificationReactor = new NotificationReactor(notificationsService as any);
|
||||
const incidentReactor = new IncidentReactor(incidentService as any);
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
@@ -19,9 +20,10 @@ const createEvaluator = (overrides?: Record<string, any>) => {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
logger: createMockLogger(),
|
||||
...overrides,
|
||||
};
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
|
||||
return { evaluator, defaults };
|
||||
};
|
||||
|
||||
@@ -84,4 +86,38 @@ describe("CheckEvaluator", () => {
|
||||
shouldSendNotification: false,
|
||||
});
|
||||
});
|
||||
|
||||
// ── degraded egress ───────────────────────────────────────────────────────
|
||||
|
||||
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "degraded" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
const result = await evaluator.evaluate(status, check, monitor);
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
|
||||
expect(result.monitor).toBe(monitor);
|
||||
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
|
||||
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
|
||||
expect(result.decision).toEqual({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
});
|
||||
expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
|
||||
});
|
||||
|
||||
it("evaluates a check flagged with egress ok as normal", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "ok" } as any;
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
await evaluator.evaluate(status, check, makeMonitor());
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
|
||||
});
|
||||
});
|
||||
@@ -29,6 +29,7 @@ const createProducer = (overrides?: Record<string, any>) => {
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
|
||||
...overrides,
|
||||
};
|
||||
const producer = new CheckProducer(
|
||||
@@ -39,6 +40,7 @@ const createProducer = (overrides?: Record<string, any>) => {
|
||||
defaults.proxyResolver as any,
|
||||
defaults.buffer as any,
|
||||
defaults.dockerLogsService as any,
|
||||
defaults.egressService as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { producer, defaults };
|
||||
@@ -210,4 +212,62 @@ describe("CheckProducer", () => {
|
||||
|
||||
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── egress self-check ─────────────────────────────────────────────────────
|
||||
|
||||
const failingStatus = { monitorId: "m1", status: false, code: 500, message: "Error" };
|
||||
|
||||
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
|
||||
expect(result?.check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("flags a failing check as ok when the probe found egress fine", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(check.egressStatus).toBe("ok");
|
||||
});
|
||||
|
||||
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user