Address review of degraded-egress check persistence
CheckProducer consults egress only for a transport failure: the check
failed, no HTTP status came back (isHttpStatusCode from PR 1), and the
monitor type is one whose failure can be a loss of egress
(EgressAttributableTypes; hardware and docker are excluded). A 4xx/5xx
or a content mismatch is evaluated as usual while the instance is
degraded. A rejected assessment is logged and treated as unknown so the
check is still buffered.
EXCLUDE_DEGRADED_EGRESS_MATCH moves out of the model into
check.query.ts, with an expression form for $group accumulators.
Daily status buckets and docker stats buckets exclude degraded checks
from their up/total counts while keeping them in the response-time
average, matching the uptime facet. toEntity maps egressStatus like its
siblings.
Adds heartbeatJobDegradedEgress integration tests, extends the
repository exclusion tests, and makes the harness egress stub settable.
🔧
This commit is contained in:
1 parent
4f5dba560d
commit
4335e6b7b6
13 files changed
+334
-29
No files matched your search
@@ -14,6 +14,7 @@ import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||
import type { Check } from "../../src/domain/checks/check.type.ts";
|
||||
import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts";
|
||||
import type { EgressStatus } from "../../src/domain/egress/egress.type.ts";
|
||||
|
||||
let checkCounter = 0;
|
||||
|
||||
@@ -79,9 +80,11 @@ export interface HeartbeatTestHarness {
|
||||
bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock };
|
||||
maintenanceWindowsRepo: { findByMonitorId: jest.Mock };
|
||||
messageBuilder: { extractThresholdBreaches: jest.Mock };
|
||||
egressService: { assessAfterFailure: jest.Mock };
|
||||
heartbeatJob: (monitor: Monitor) => Promise<void>;
|
||||
setNextResponse: (status: boolean, code: number) => void;
|
||||
setNextResponseFull: (response: MonitorStatusResponse) => void;
|
||||
setEgressStatus: (status: EgressStatus | null) => void;
|
||||
}
|
||||
|
||||
export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
@@ -129,7 +132,12 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
|
||||
const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
|
||||
const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
|
||||
const egressService = { assessAfterFailure: jest.fn().mockResolvedValue(null) };
|
||||
// Null mirrors the egress check being disabled, so the existing heartbeat suites run unchanged.
|
||||
let nextEgressStatus: EgressStatus | null = null;
|
||||
const egressService = { assessAfterFailure: jest.fn().mockImplementation(() => Promise.resolve(nextEgressStatus)) };
|
||||
const setEgressStatus = (status: EgressStatus | null) => {
|
||||
nextEgressStatus = status;
|
||||
};
|
||||
|
||||
const checkProducer = new CheckProducer(
|
||||
monitorsRepo as any,
|
||||
@@ -169,8 +177,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
bufferStub,
|
||||
maintenanceWindowsRepo,
|
||||
messageBuilder,
|
||||
egressService,
|
||||
heartbeatJob,
|
||||
setNextResponse,
|
||||
setNextResponseFull,
|
||||
setEgressStatus,
|
||||
};
|
||||
}
|
||||
@@ -78,7 +78,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
|
||||
|
||||
// The evaluator runs from this read, so the flag must survive persistence for the short-circuit to fire.
|
||||
expect(checks.map((check) => check.egressStatus)).toEqual(["degraded", "ok", undefined]);
|
||||
expect(checks[2]).not.toHaveProperty("egressStatus");
|
||||
expect(checks[2].egressStatus).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects a value outside the EgressStatuses tuple", async () => {
|
||||
@@ -95,8 +95,9 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
|
||||
|
||||
// 1 up of 2 attributable checks. Counting the degraded pair would give 25%.
|
||||
expect(result.uptimePercentage).toBe(0.5);
|
||||
// The graph still shows the degraded checks as failures.
|
||||
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4 });
|
||||
// The graph still shows the degraded checks as failures, and their response times stay in the averages.
|
||||
expect(result.avgResponseTime).toBe(200);
|
||||
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4, avgResponseTime: 200 });
|
||||
expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 3 });
|
||||
expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 });
|
||||
});
|
||||
@@ -120,7 +121,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
|
||||
expect(summary).toEqual({ totalChecks: 2, downChecks: 1 });
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the daily status buckets and yields no row for a day of only degraded checks", async () => {
|
||||
it("excludes degraded checks from the daily status bucket counts but keeps them in its response-time average", async () => {
|
||||
const otherMonitor = new mongoose.Types.ObjectId();
|
||||
await seedCheck();
|
||||
await seedCheck({ status: false });
|
||||
@@ -129,8 +130,10 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
|
||||
|
||||
const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC");
|
||||
|
||||
// (100 + 100 + 300) / 3 = 167: the degraded check's response time counts, its failure does not.
|
||||
// The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket.
|
||||
expect(buckets).toEqual([
|
||||
{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 },
|
||||
{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 167 },
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -154,6 +157,19 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
|
||||
expect(docker.upChecks.totalChecks).toBe(1);
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the docker bucket counts but keeps them in its response-time average", async () => {
|
||||
const dockerMonitor = new mongoose.Types.ObjectId();
|
||||
const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
|
||||
|
||||
await seedCheck({ metadata: dockerMeta });
|
||||
await seedDegradedFailure({ metadata: dockerMeta });
|
||||
|
||||
const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
|
||||
|
||||
expect(docker.aggregate).toHaveLength(1);
|
||||
expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1, avgResponseTime: 200 });
|
||||
});
|
||||
|
||||
it("keeps degraded checks in the paginated listing so they can be shown as such", async () => {
|
||||
await seedCheck({ status: false });
|
||||
await seedDegradedFailure();
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
import { NETWORK_ERROR } from "../../src/types/network.ts";
|
||||
|
||||
// Egress is only consulted for transport failures, so these heartbeats fail with the network-error
|
||||
// sentinel (timeout / connection refused) rather than an HTTP status code.
|
||||
|
||||
describe("Heartbeat job: degraded egress", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("records failing checks as degraded without changing monitor status, opening an incident or notifying", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Enough failures to cross the 60% threshold if they were counted
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Every check was buffered, flagged, so the repository can exclude it from uptime
|
||||
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
|
||||
expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(3);
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).toMatchObject({ status: false, egressStatus: "degraded" });
|
||||
}
|
||||
|
||||
// Nothing downstream moved
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
expect(storedMonitor.statusWindow).toEqual([true, true, true, true, true]);
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not fire a spurious recovery once egress returns", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Egress is back and the target is fine
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(true, 200);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detects down at the normal threshold when the egress check is disabled", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detects down at the normal threshold when egress is confirmed ok", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("ok");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).toMatchObject({ status: false, egressStatus: "ok" });
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluates HTTP error responses normally while egress is degraded", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// The target answered, so the failure is its own even though the instance is degraded
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, 503);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.incidentsRepo.getAll()[0].statusCode).toBe(503);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluates a failing docker check normally while egress is degraded", async () => {
|
||||
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" });
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps counting real failures taken either side of a degraded spell", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// 2 real failures: window [t, t, t, f, f] = 40% < 60%
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// A degraded spell must not push the window over the threshold
|
||||
h.setEgressStatus("degraded");
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
|
||||
// 3rd real failure: window [t, t, f, f, f] = 60% >= 60%
|
||||
h.setEgressStatus(null);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import { NETWORK_ERROR } from "../../../src/types/network.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
@@ -215,7 +216,8 @@ describe("CheckProducer", () => {
|
||||
|
||||
// ── egress self-check ─────────────────────────────────────────────────────
|
||||
|
||||
const failingStatus = { monitorId: "m1", status: false, code: 500, message: "Error" };
|
||||
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
|
||||
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
|
||||
|
||||
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
@@ -229,6 +231,57 @@ describe("CheckProducer", () => {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("does not consult egress when an HTTP response was received", async () => {
|
||||
for (const status of [
|
||||
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
|
||||
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("consults egress for a timeout or network error", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("never consults egress for a failing docker or hardware check", async () => {
|
||||
for (const type of ["docker", "hardware"] as const) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type }));
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
@@ -258,6 +311,24 @@ describe("CheckProducer", () => {
|
||||
expect(check.egressStatus).toBe("ok");
|
||||
});
|
||||
|
||||
it("buffers the check when the egress assessment rejects", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(result).toEqual({ status: failingStatus, check });
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" })
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
|
||||
Reference in new issue
Block a user