Merge pull request #3963 from web-engineer/feat/egress-2-check-persistence
Egress self-check (2/4): flag checks taken during degraded egress (#3937)
This commit is contained in:
42 files changed
+1319
-74
No files matched your search
@@ -14,6 +14,7 @@ import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||
import type { Check } from "../../src/domain/checks/check.type.ts";
|
||||
import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts";
|
||||
import type { EgressStatus } from "../../src/domain/egress/egress.type.ts";
|
||||
|
||||
let checkCounter = 0;
|
||||
|
||||
@@ -79,9 +80,11 @@ export interface HeartbeatTestHarness {
|
||||
bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock };
|
||||
maintenanceWindowsRepo: { findByMonitorId: jest.Mock };
|
||||
messageBuilder: { extractThresholdBreaches: jest.Mock };
|
||||
egressService: { assessAfterFailure: jest.Mock };
|
||||
heartbeatJob: (monitor: Monitor) => Promise<void>;
|
||||
setNextResponse: (status: boolean, code: number) => void;
|
||||
setNextResponseFull: (response: MonitorStatusResponse) => void;
|
||||
setEgressStatus: (status: EgressStatus | null) => void;
|
||||
}
|
||||
|
||||
export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
@@ -129,6 +132,12 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
|
||||
const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
|
||||
const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
|
||||
// Null mirrors the egress check being disabled, so the existing heartbeat suites run unchanged.
|
||||
let nextEgressStatus: EgressStatus | null = null;
|
||||
const egressService = { assessAfterFailure: jest.fn().mockImplementation(() => Promise.resolve(nextEgressStatus)) };
|
||||
const setEgressStatus = (status: EgressStatus | null) => {
|
||||
nextEgressStatus = status;
|
||||
};
|
||||
|
||||
const checkProducer = new CheckProducer(
|
||||
monitorsRepo as any,
|
||||
@@ -138,9 +147,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
proxyResolver as any,
|
||||
bufferStub as any,
|
||||
dockerLogsService as any,
|
||||
egressService as any,
|
||||
logger
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger);
|
||||
|
||||
const notificationReactor = new NotificationReactor(notificationsService as any);
|
||||
const incidentReactor = new IncidentReactor(incidentService as any);
|
||||
@@ -167,8 +177,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
bufferStub,
|
||||
maintenanceWindowsRepo,
|
||||
messageBuilder,
|
||||
egressService,
|
||||
heartbeatJob,
|
||||
setNextResponse,
|
||||
setNextResponseFull,
|
||||
setEgressStatus,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
|
||||
import { CheckModel } from "../../src/domain/checks/check.model.ts";
|
||||
import type {
|
||||
Check,
|
||||
DockerChecksResult,
|
||||
HardwareChecksResult,
|
||||
PageSpeedChecksResult,
|
||||
UptimeChecksResult,
|
||||
} from "../../src/domain/checks/check.type.ts";
|
||||
import { NETWORK_ERROR } from "../../src/types/network.ts";
|
||||
import type { ILogger } from "../../src/utils/logger.ts";
|
||||
import { createMockLogger } from "../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The guarantee under test — a check flagged egressStatus "degraded" is left out of
|
||||
// every aggregate, so a degraded episode reads as a gap in monitoring, while the check
|
||||
// still appears in listings — lives in $match stages spread into several aggregation
|
||||
// pipelines, so it can only be exercised against a live mongod.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await CheckModel.createCollection(); // timeseries collections must exist before insert
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await CheckModel.deleteMany({});
|
||||
});
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const MONITOR_ID = new mongoose.Types.ObjectId();
|
||||
const TEAM_ID = new mongoose.Types.ObjectId();
|
||||
|
||||
// One shared createdAt keeps every seeded check in the same bucket of the "day" range.
|
||||
const BUCKET_TIME = new Date(Date.now() - 10 * 60 * 1000);
|
||||
|
||||
const seedCheck = (overrides: Record<string, unknown> = {}) =>
|
||||
CheckModel.create({
|
||||
metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "http" },
|
||||
status: true,
|
||||
responseTime: 100,
|
||||
createdAt: BUCKET_TIME,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const seedDegradedFailure = (overrides: Record<string, unknown> = {}) =>
|
||||
seedCheck({ status: false, responseTime: 300, egressStatus: "degraded", ...overrides });
|
||||
|
||||
const utcDate = (date: Date) => date.toISOString().slice(0, 10);
|
||||
|
||||
const makeCheck = (overrides: Partial<Check> = {}): Check =>
|
||||
({
|
||||
metadata: { monitorId: MONITOR_ID.toString(), teamId: TEAM_ID.toString(), type: "http" },
|
||||
status: false,
|
||||
responseTime: 300,
|
||||
statusCode: 500,
|
||||
message: "Error",
|
||||
createdAt: BUCKET_TIME.toISOString(),
|
||||
updatedAt: BUCKET_TIME.toISOString(),
|
||||
...overrides,
|
||||
}) as Check;
|
||||
|
||||
describe("MongoChecksRepository degraded-egress exclusion", () => {
|
||||
let repo: MongoChecksRepository;
|
||||
|
||||
beforeEach(() => {
|
||||
repo = new MongoChecksRepository(createMockLogger() as unknown as ILogger);
|
||||
});
|
||||
|
||||
it("round-trips egressStatus through createChecks and the unevaluated-checks read, leaving it absent when unset", async () => {
|
||||
const created = await repo.createChecks([makeCheck({ egressStatus: "degraded" }), makeCheck({ egressStatus: "ok" }), makeCheck()]);
|
||||
const pending = created.map((check) => ({ checkId: check.id as string, createdAt: new Date(check.createdAt as string).getTime() }));
|
||||
|
||||
const checks = await repo.findUnevaluatedByMonitorId(MONITOR_ID.toString(), pending);
|
||||
|
||||
// The evaluator runs from this read, so the flag must survive persistence for the short-circuit to fire.
|
||||
expect(checks.map((check) => check.egressStatus)).toEqual(["degraded", "ok", undefined]);
|
||||
expect(checks[2].egressStatus).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects a value outside the EgressStatuses tuple", async () => {
|
||||
await expect(seedCheck({ status: false, egressStatus: "unknown" })).rejects.toThrow(/egressStatus/);
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the uptime percentage and from the response-time series alike", async () => {
|
||||
await seedCheck(); // up
|
||||
await seedCheck({ status: false }); // real failure
|
||||
await seedDegradedFailure();
|
||||
await seedDegradedFailure();
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
|
||||
|
||||
// 1 up of 2 attributable checks. Counting the degraded pair would give 25%.
|
||||
expect(result.uptimePercentage).toBe(0.5);
|
||||
// Every series sees the same two checks, so the episode is a gap rather than a dip.
|
||||
expect(result.avgResponseTime).toBe(100);
|
||||
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 2, avgResponseTime: 100 });
|
||||
expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 1 });
|
||||
expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 });
|
||||
});
|
||||
|
||||
it("leaves a bucket out of the response-time series entirely when every check in it was degraded", async () => {
|
||||
await seedDegradedFailure();
|
||||
await seedDegradedFailure();
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
|
||||
|
||||
expect(result.groupedChecks).toEqual([]);
|
||||
expect(result.groupedDownChecks).toEqual([]);
|
||||
expect(result.uptimePercentage).toBe(0);
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the pagespeed series", async () => {
|
||||
await seedCheck({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.9 });
|
||||
await seedDegradedFailure({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.1 });
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "pagespeed" })) as PageSpeedChecksResult;
|
||||
|
||||
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 1, performance: 0.9 });
|
||||
});
|
||||
|
||||
it("does not flag a failure that carries egressStatus ok", async () => {
|
||||
await seedCheck();
|
||||
await seedCheck({ status: false, egressStatus: "ok" });
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
|
||||
|
||||
expect(result.uptimePercentage).toBe(0.5);
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the team summary totals", async () => {
|
||||
await seedCheck();
|
||||
await seedCheck({ status: false });
|
||||
await seedDegradedFailure();
|
||||
|
||||
const summary = await repo.findSummaryByTeamId(TEAM_ID.toString(), "day");
|
||||
|
||||
expect(summary).toEqual({ totalChecks: 2, downChecks: 1 });
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the daily status buckets, dropping a day that saw nothing else", async () => {
|
||||
const otherMonitor = new mongoose.Types.ObjectId();
|
||||
await seedCheck();
|
||||
await seedCheck({ status: false });
|
||||
await seedDegradedFailure();
|
||||
await seedDegradedFailure({ metadata: { monitorId: otherMonitor, teamId: TEAM_ID, type: "http" } });
|
||||
|
||||
const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC");
|
||||
|
||||
// The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket.
|
||||
expect(buckets).toEqual([
|
||||
{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps degraded checks in the paginated listing so they can be shown as such", async () => {
|
||||
await seedCheck({ status: false });
|
||||
await seedDegradedFailure();
|
||||
|
||||
const { checksCount, checks } = await repo.findByMonitorId(MONITOR_ID.toString(), "desc", "day", 0, 10, undefined, "down");
|
||||
|
||||
expect(checksCount).toBe(2);
|
||||
expect(checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
|
||||
});
|
||||
|
||||
it("keeps degraded checks in the team listing and under the resolve filter", async () => {
|
||||
await seedCheck({ status: false, statusCode: NETWORK_ERROR });
|
||||
await seedDegradedFailure({ statusCode: NETWORK_ERROR });
|
||||
|
||||
// The Checks page reads this listing when no single monitor is selected, and a degraded check has the
|
||||
// same shape the resolve filter looks for, so neither may quietly drop it.
|
||||
const team = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "down");
|
||||
const resolve = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "resolve");
|
||||
|
||||
expect(team.checksCount).toBe(2);
|
||||
expect(resolve.checksCount).toBe(2);
|
||||
expect(resolve.checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the docker and hardware aggregates", async () => {
|
||||
const dockerMonitor = new mongoose.Types.ObjectId();
|
||||
const hardwareMonitor = new mongoose.Types.ObjectId();
|
||||
const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
|
||||
const hardwareMeta = { monitorId: hardwareMonitor, teamId: TEAM_ID, type: "hardware" };
|
||||
|
||||
// A Capture agent is reached by an outbound HTTP request and a docker daemon can be reached over TCP,
|
||||
// so both are egress attributable and their checks can carry the flag.
|
||||
await seedCheck({ metadata: dockerMeta });
|
||||
await seedDegradedFailure({ metadata: dockerMeta });
|
||||
await seedCheck({ metadata: hardwareMeta });
|
||||
await seedDegradedFailure({ metadata: hardwareMeta });
|
||||
|
||||
const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
|
||||
const hardware = (await repo.findByDateRangeAndMonitorId(hardwareMonitor.toString(), "day", { type: "hardware" })) as HardwareChecksResult;
|
||||
|
||||
expect(docker.aggregateData.totalChecks).toBe(1);
|
||||
expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1 });
|
||||
expect(hardware.aggregateData.totalChecks).toBe(1);
|
||||
expect(hardware.upChecks.totalChecks).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,191 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
import { NETWORK_ERROR } from "../../src/types/network.ts";
|
||||
|
||||
// Egress is only consulted for transport failures, so these heartbeats fail with the network-error
|
||||
// sentinel (timeout / connection refused) rather than an HTTP status code.
|
||||
|
||||
describe("Heartbeat job: degraded egress", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("records failing checks as degraded without changing monitor status, opening an incident or notifying", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Enough failures to cross the 60% threshold if they were counted
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Every check was buffered, flagged, so the repository can exclude it from uptime
|
||||
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
|
||||
expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(3);
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).toMatchObject({ status: false, egressStatus: "degraded" });
|
||||
}
|
||||
|
||||
// Nothing downstream moved
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
// The window is cleared once as the spell starts, as entering a maintenance window does.
|
||||
expect(storedMonitor.statusWindow).toEqual([]);
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not fire a spurious recovery once egress returns", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Egress is back and the target is fine
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(true, 200);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detects down at the normal threshold when the egress check is disabled", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detects down at the normal threshold when egress is confirmed ok", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("ok");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).toMatchObject({ status: false, egressStatus: "ok" });
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluates HTTP error responses normally while egress is degraded", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// The target answered, so the failure is its own even though the instance is degraded
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, 503);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.incidentsRepo.getAll()[0].statusCode).toBe(503);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluates a failing docker check normally while egress is degraded", async () => {
|
||||
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" });
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("leaves an open incident open when egress drops during it", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// The target goes down first and is detected normally
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
|
||||
// Egress then drops. The monitor must stay down rather than be resolved by the silence.
|
||||
h.setEgressStatus("degraded");
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("starts a fresh status window after a degraded spell", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// 2 real failures: window [t, t, t, f, f] = 40% < 60%
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).statusWindow).toEqual([]);
|
||||
|
||||
// A third real failure would have taken the old window to 60%. Against a window that starts again
|
||||
// at the end of the spell it is one result, and the results either side are never adjacent.
|
||||
h.setEgressStatus(null);
|
||||
await h.heartbeatJob(monitor);
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
|
||||
// The monitor still goes down, once the rebuilt window holds enough real failures to cross.
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -209,6 +209,49 @@ describe("DNSProvider", () => {
|
||||
expect(mockResolverInstance.resolve4).toHaveBeenCalledWith("example.com");
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// The egress check must only blame the instance's own connectivity when nothing answered. Every DNS failure
|
||||
// carries NETWORK_ERROR, so `peerResponded` is the only thing separating an authoritative answer from silence.
|
||||
|
||||
it.each([
|
||||
["ENOTFOUND", "NXDOMAIN: the server answered that the name does not exist"],
|
||||
["ENODATA", "the server answered with no record of that type"],
|
||||
["ESERVFAIL", "the server answered with a failure"],
|
||||
["EREFUSED", "the server declined to answer"],
|
||||
["EFORMERR", "the server rejected the query as malformed"],
|
||||
["ENOTIMP", "the server does not implement the query"],
|
||||
["EBADRESP", "the server replied, badly"],
|
||||
])("reports the peer as having answered for %s", async (code) => {
|
||||
const error = Object.assign(new Error("dns failure"), { code });
|
||||
mockResolverInstance.resolve4.mockRejectedValueOnce(error);
|
||||
const provider = new DNSProvider(createResolver);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEOUT"], ["ECONNREFUSED"], ["ECANCELLED"], [undefined]])("reports the peer as unreachable for %s", async (code) => {
|
||||
const error = code ? Object.assign(new Error("dns failure"), { code }) : new Error("dns failure");
|
||||
mockResolverInstance.resolve4.mockRejectedValueOnce(error);
|
||||
const provider = new DNSProvider(createResolver);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered on success", async () => {
|
||||
const provider = new DNSProvider(createResolver);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("stringifies non-Error throws from outside timeRequest", async () => {
|
||||
mockResolverInstance.setServers.mockImplementationOnce(() => {
|
||||
throw 42;
|
||||
|
||||
@@ -697,4 +697,43 @@ describe("DockerProvider", () => {
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("Error performing Docker request");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// A daemon that answers with an error has been reached; only silence can be the instance's own egress.
|
||||
|
||||
describe("peer reachability", () => {
|
||||
it("reports the peer as having answered when the daemon replies with a status code", async () => {
|
||||
const dockerErr = Object.assign(new Error("base"), { statusCode: 500, json: { message: "engine exploded" } });
|
||||
const { provider } = setup({ ping: jest.fn().mockRejectedValue(dockerErr) });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
|
||||
const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
|
||||
const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered when the daemon is reachable", async () => {
|
||||
const { provider } = setup({});
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -362,6 +362,59 @@ describe("GrpcProvider", () => {
|
||||
[99, "UNKNOWN"],
|
||||
];
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────
|
||||
// NOT_SERVING and a connection failure both report NETWORK_ERROR, and gRPC codes are not HTTP statuses,
|
||||
// so `peerResponded` is what stops the egress check reading a server's answer as a transport failure.
|
||||
|
||||
it("reports the peer as having answered when the server replies NOT_SERVING", async () => {
|
||||
const { provider } = createProvider({ behavior: "not-serving" });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(5000);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([[5], [7], [12], [16], [2]])("reports the peer as having answered for gRPC code %i", async (code) => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
|
||||
close: jest.fn(),
|
||||
};
|
||||
const mocks = createMockGrpc(client as any);
|
||||
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([[4], [14]])("reports the peer as unreachable for gRPC code %i", async (code) => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
|
||||
close: jest.fn(),
|
||||
};
|
||||
const mocks = createMockGrpc(client as any);
|
||||
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as unreachable when the failure carries no gRPC code", async () => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ message: "socket hang up" })),
|
||||
close: jest.fn(),
|
||||
};
|
||||
const mocks = createMockGrpc(client as any);
|
||||
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it.each(statusCodes)("maps gRPC code %i to %s", async (code, expectedName) => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
|
||||
|
||||
@@ -643,4 +643,92 @@ describe("HttpProvider", () => {
|
||||
expect(result.message).toBe("Response code 500 (Internal Server Error)");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// Any HTTP status proves the target answered. Without one, only an errno that requires the peer's own stack
|
||||
// to have replied counts — and never through a proxy, where the refusal may be the proxy's rather than the
|
||||
// target's, which is the instance's own egress path failing.
|
||||
|
||||
describe("peer reachability", () => {
|
||||
it("reports the peer as having answered on a successful response", async () => {
|
||||
mockGot.mockResolvedValue(makeGotResponse());
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered for a failing status code", async () => {
|
||||
const err = new HTTPError("Not Found", { statusCode: 404, body: "", headers: {} }, { phases: { total: 12 } });
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(404);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"], ["CERT_HAS_EXPIRED"], ["DEPTH_ZERO_SELF_SIGNED_CERT"]])(
|
||||
"reports the peer as having answered for %s with no response",
|
||||
async (code) => {
|
||||
const err = new RequestError("request failed");
|
||||
(err as any).code = code;
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
}
|
||||
);
|
||||
|
||||
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["ENETUNREACH"]])("reports the peer as silent for %s", async (code) => {
|
||||
const err = new RequestError("request failed");
|
||||
(err as any).code = code;
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as silent when a proxy refuses the connection", async () => {
|
||||
const err = new RequestError("connect ECONNREFUSED");
|
||||
(err as any).code = "ECONNREFUSED";
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
|
||||
|
||||
// The refusal came from the proxy, which is part of this instance's route out.
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered through a proxy when a status code came back", async () => {
|
||||
const err = new RequestError("Response code 500 (Internal Server Error)");
|
||||
(err as any).code = "ECONNRESET";
|
||||
(err as any).response = { statusCode: 500, body: "", headers: {} };
|
||||
(err as any).timings = { phases: { total: 25 } };
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the peer as silent for a generic error carrying no code", async () => {
|
||||
mockGot.mockRejectedValue(new Error("something broke"));
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -155,4 +155,23 @@ describe("PingProvider", () => {
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
|
||||
it("reports the peer as having answered when an echo reply comes back", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the peer as silent when no echo reply comes back", async () => {
|
||||
// Silence is indistinguishable from the instance losing its egress, so the egress probe decides.
|
||||
const provider = new PingProvider(createMockPing({ alive: false }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -267,4 +267,42 @@ describe("PortProvider", () => {
|
||||
|
||||
await expect(provider.handle(monitor)).rejects.toThrow("42");
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// A check is only recorded as degraded when nothing answered, so a service on the local network that
|
||||
// refuses connections has to stay distinguishable from the instance losing its own egress.
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
|
||||
const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
|
||||
const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as silent on a connection timeout", async () => {
|
||||
const provider = new PortProvider(createTimeoutNet());
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered when the port accepts the connection", async () => {
|
||||
const provider = new PortProvider(createSuccessNet());
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { timeRequest, isStatusUp } from "../../../../src/service/network/utils.ts";
|
||||
import { timeRequest, isStatusUp, peerAnsweredFromError } from "../../../../src/service/network/utils.ts";
|
||||
import { NETWORK_ERROR } from "../../../../src/types/network.ts";
|
||||
|
||||
describe("network utils", () => {
|
||||
@@ -39,6 +39,40 @@ describe("network utils", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// The egress check blames the instance only when nothing answered, so this helper's `true` must never be a
|
||||
// guess: every code in it requires the peer's own stack to have replied.
|
||||
describe("peerAnsweredFromError", () => {
|
||||
it.each([
|
||||
["ECONNREFUSED", "the host refused the connection"],
|
||||
["ECONNRESET", "the host reset an established connection"],
|
||||
["EPIPE", "the host closed a connection we were writing to"],
|
||||
["EPROTO", "a protocol error on a connection that was made"],
|
||||
["CERT_HAS_EXPIRED", "the host presented a certificate"],
|
||||
["DEPTH_ZERO_SELF_SIGNED_CERT", "the host presented a certificate"],
|
||||
["SELF_SIGNED_CERT_IN_CHAIN", "the host presented a certificate"],
|
||||
["UNABLE_TO_GET_ISSUER_CERT_LOCALLY", "the host presented a certificate"],
|
||||
["UNABLE_TO_VERIFY_LEAF_SIGNATURE", "the host presented a certificate"],
|
||||
["ERR_TLS_CERT_ALTNAME_INVALID", "the host presented a certificate"],
|
||||
["CERT_NOT_YET_VALID", "the host presented a certificate"],
|
||||
["ERR_SSL_WRONG_VERSION_NUMBER", "the host spoke back, badly"],
|
||||
])("reports the peer as having answered for %s", (code) => {
|
||||
expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ECONNABORTED"], ["ENETDOWN"]])(
|
||||
"reports the peer as silent for %s",
|
||||
(code) => {
|
||||
expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
it("reports the peer as silent when the error carries no code", () => {
|
||||
expect(peerAnsweredFromError(new Error("Connection timeout"))).toBe(false);
|
||||
expect(peerAnsweredFromError(undefined)).toBe(false);
|
||||
expect(peerAnsweredFromError("not an error")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isStatusUp", () => {
|
||||
it("returns true for 2xx status codes", () => {
|
||||
expect(isStatusUp(200)).toBe(true);
|
||||
|
||||
@@ -191,4 +191,42 @@ describe("WebSocketProvider", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// `ws` rejects a non-101 handshake with a plain Error, so only the errno cases can be told apart from the
|
||||
// instance losing its egress. That leaves a server answering the handshake with an HTTP error indistinct.
|
||||
|
||||
const createErrorWS = (error: Error) =>
|
||||
jest.fn().mockImplementation(() => ({
|
||||
on: jest.fn((event: string, cb: Function) => {
|
||||
if (event === "error") process.nextTick(() => cb(error));
|
||||
}),
|
||||
close: jest.fn(),
|
||||
})) as any;
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
|
||||
const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EHOSTUNREACH"]])("reports the peer as silent for %s", async (code) => {
|
||||
const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered when the socket opens", async () => {
|
||||
const provider = new WebSocketProvider(createMockWS("open"));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
@@ -19,9 +20,10 @@ const createEvaluator = (overrides?: Record<string, any>) => {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
logger: createMockLogger(),
|
||||
...overrides,
|
||||
};
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
|
||||
return { evaluator, defaults };
|
||||
};
|
||||
|
||||
@@ -84,4 +86,38 @@ describe("CheckEvaluator", () => {
|
||||
shouldSendNotification: false,
|
||||
});
|
||||
});
|
||||
|
||||
// ── degraded egress ───────────────────────────────────────────────────────
|
||||
|
||||
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "degraded" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
const result = await evaluator.evaluate(status, check, monitor);
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
|
||||
expect(result.monitor).toBe(monitor);
|
||||
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
|
||||
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
|
||||
expect(result.decision).toEqual({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
});
|
||||
expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
|
||||
});
|
||||
|
||||
it("evaluates a check flagged with egress ok as normal", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "ok" } as any;
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
await evaluator.evaluate(status, check, makeMonitor());
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import { NETWORK_ERROR } from "../../../src/types/network.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
@@ -29,6 +30,7 @@ const createProducer = (overrides?: Record<string, any>) => {
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
|
||||
...overrides,
|
||||
};
|
||||
const producer = new CheckProducer(
|
||||
@@ -39,6 +41,7 @@ const createProducer = (overrides?: Record<string, any>) => {
|
||||
defaults.proxyResolver as any,
|
||||
defaults.buffer as any,
|
||||
defaults.dockerLogsService as any,
|
||||
defaults.egressService as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { producer, defaults };
|
||||
@@ -76,6 +79,31 @@ describe("CheckProducer", () => {
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
|
||||
const monitor = makeMonitor({ status: "maintenance" });
|
||||
const { producer, defaults } = createProducer({
|
||||
monitorsRepository: {
|
||||
updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }),
|
||||
findDockerTlsKeyById: jest.fn().mockResolvedValue(null),
|
||||
},
|
||||
});
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
|
||||
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not touch the status of a monitor that was not in maintenance", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
|
||||
await producer.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("proceeds with the check when the maintenance window is inactive", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
|
||||
@@ -210,4 +238,222 @@ describe("CheckProducer", () => {
|
||||
|
||||
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── egress self-check ─────────────────────────────────────────────────────
|
||||
|
||||
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
|
||||
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
|
||||
|
||||
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("does not consult egress when an HTTP response was received", async () => {
|
||||
for (const status of [
|
||||
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
|
||||
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("consults egress for a timeout or network error", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("never consults egress for a docker daemon on a local socket", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" }));
|
||||
|
||||
// Local IPC never leaves the instance, so it cannot fail through egress.
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
});
|
||||
|
||||
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
|
||||
for (const monitor of [
|
||||
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
|
||||
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
// Both requests leave the instance, so our own loss of egress can be the cause.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not consult egress when the provider reports that the peer answered", async () => {
|
||||
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
|
||||
// them from a connection failure.
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "dns" }));
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "grpc" }));
|
||||
|
||||
// An explicit report from the provider outranks the HTTP-status heuristic.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
|
||||
expect(result?.check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
// ── status window across a spell ─────────────────────────────────────────
|
||||
// Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either
|
||||
// side of a spell are never adjacent.
|
||||
|
||||
it("clears the status window on the first degraded check of a spell", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] });
|
||||
});
|
||||
|
||||
it("does not re-write the status window for later degraded checks in the same spell", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves the status window alone when the probe found egress fine", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("flags a failing check as ok when the probe found egress fine", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(check.egressStatus).toBe("ok");
|
||||
});
|
||||
|
||||
it("buffers the check when the egress assessment rejects", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(result).toEqual({ status: failingStatus, check });
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" })
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,6 @@ const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
const makeSettings = (overrides?: Record<string, unknown>) => ({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: ["notif-1"],
|
||||
...overrides,
|
||||
});
|
||||
@@ -33,7 +32,7 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: 1_000,
|
||||
intervalMs: 30_000,
|
||||
intervalMs: 5_000,
|
||||
lockedBy: "worker-1",
|
||||
lockedUntil: 61_000,
|
||||
runCount: 0,
|
||||
@@ -245,7 +244,7 @@ describe("EgressService", () => {
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: expect.any(Number),
|
||||
intervalMs: 30_000,
|
||||
intervalMs: 5_000,
|
||||
});
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
|
||||
@@ -266,7 +265,7 @@ describe("EgressService", () => {
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
|
||||
});
|
||||
|
||||
@@ -293,21 +292,10 @@ describe("EgressService", () => {
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
|
||||
});
|
||||
|
||||
it("falls back to the default targets when the configured list is empty", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts";
|
||||
import { MAX_EGRESS_TARGETS } from "../../../src/domain/egress/egress.type.ts";
|
||||
|
||||
const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8";
|
||||
|
||||
@@ -10,13 +10,11 @@ describe("settingsValidation", () => {
|
||||
const parsed = updateAppSettingsBodyValidation.parse({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: [validNotificationId],
|
||||
});
|
||||
|
||||
expect(parsed.egressCheckEnabled).toBe(true);
|
||||
expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]);
|
||||
expect(parsed.egressPollIntervalSeconds).toBe(30);
|
||||
expect(parsed.egressNotifications).toEqual([validNotificationId]);
|
||||
});
|
||||
|
||||
@@ -62,21 +60,6 @@ describe("settingsValidation", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts the poll interval boundaries", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MIN_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MAX_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects out-of-range and non-integer poll intervals", () => {
|
||||
for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts an empty notification list", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user