Merge pull request #3963 from web-engineer/feat/egress-2-check-persistence
Egress self-check (2/4): flag checks taken during degraded egress (#3937)
This commit is contained in:
42 files changed
+1319
-74
No files matched your search
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
@@ -19,9 +20,10 @@ const createEvaluator = (overrides?: Record<string, any>) => {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
logger: createMockLogger(),
|
||||
...overrides,
|
||||
};
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
|
||||
return { evaluator, defaults };
|
||||
};
|
||||
|
||||
@@ -84,4 +86,38 @@ describe("CheckEvaluator", () => {
|
||||
shouldSendNotification: false,
|
||||
});
|
||||
});
|
||||
|
||||
// ── degraded egress ───────────────────────────────────────────────────────
|
||||
|
||||
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "degraded" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
const result = await evaluator.evaluate(status, check, monitor);
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
|
||||
expect(result.monitor).toBe(monitor);
|
||||
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
|
||||
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
|
||||
expect(result.decision).toEqual({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
});
|
||||
expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
|
||||
});
|
||||
|
||||
it("evaluates a check flagged with egress ok as normal", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "ok" } as any;
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
await evaluator.evaluate(status, check, makeMonitor());
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import { NETWORK_ERROR } from "../../../src/types/network.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
@@ -29,6 +30,7 @@ const createProducer = (overrides?: Record<string, any>) => {
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
|
||||
...overrides,
|
||||
};
|
||||
const producer = new CheckProducer(
|
||||
@@ -39,6 +41,7 @@ const createProducer = (overrides?: Record<string, any>) => {
|
||||
defaults.proxyResolver as any,
|
||||
defaults.buffer as any,
|
||||
defaults.dockerLogsService as any,
|
||||
defaults.egressService as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { producer, defaults };
|
||||
@@ -76,6 +79,31 @@ describe("CheckProducer", () => {
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
|
||||
const monitor = makeMonitor({ status: "maintenance" });
|
||||
const { producer, defaults } = createProducer({
|
||||
monitorsRepository: {
|
||||
updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }),
|
||||
findDockerTlsKeyById: jest.fn().mockResolvedValue(null),
|
||||
},
|
||||
});
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
|
||||
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not touch the status of a monitor that was not in maintenance", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
|
||||
await producer.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("proceeds with the check when the maintenance window is inactive", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
|
||||
@@ -210,4 +238,222 @@ describe("CheckProducer", () => {
|
||||
|
||||
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── egress self-check ─────────────────────────────────────────────────────
|
||||
|
||||
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
|
||||
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
|
||||
|
||||
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("does not consult egress when an HTTP response was received", async () => {
|
||||
for (const status of [
|
||||
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
|
||||
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("consults egress for a timeout or network error", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("never consults egress for a docker daemon on a local socket", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" }));
|
||||
|
||||
// Local IPC never leaves the instance, so it cannot fail through egress.
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
});
|
||||
|
||||
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
|
||||
for (const monitor of [
|
||||
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
|
||||
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
// Both requests leave the instance, so our own loss of egress can be the cause.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not consult egress when the provider reports that the peer answered", async () => {
|
||||
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
|
||||
// them from a connection failure.
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "dns" }));
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "grpc" }));
|
||||
|
||||
// An explicit report from the provider outranks the HTTP-status heuristic.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
|
||||
expect(result?.check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
// ── status window across a spell ─────────────────────────────────────────
|
||||
// Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either
|
||||
// side of a spell are never adjacent.
|
||||
|
||||
it("clears the status window on the first degraded check of a spell", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] });
|
||||
});
|
||||
|
||||
it("does not re-write the status window for later degraded checks in the same spell", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves the status window alone when the probe found egress fine", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("flags a failing check as ok when the probe found egress fine", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(check.egressStatus).toBe("ok");
|
||||
});
|
||||
|
||||
it("buffers the check when the egress assessment rejects", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(result).toEqual({ status: failingStatus, check });
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" })
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
});
|
||||
@@ -21,7 +21,6 @@ const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
const makeSettings = (overrides?: Record<string, unknown>) => ({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: ["notif-1"],
|
||||
...overrides,
|
||||
});
|
||||
@@ -33,7 +32,7 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: 1_000,
|
||||
intervalMs: 30_000,
|
||||
intervalMs: 5_000,
|
||||
lockedBy: "worker-1",
|
||||
lockedUntil: 61_000,
|
||||
runCount: 0,
|
||||
@@ -245,7 +244,7 @@ describe("EgressService", () => {
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: expect.any(Number),
|
||||
intervalMs: 30_000,
|
||||
intervalMs: 5_000,
|
||||
});
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
|
||||
@@ -266,7 +265,7 @@ describe("EgressService", () => {
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
|
||||
});
|
||||
|
||||
@@ -293,21 +292,10 @@ describe("EgressService", () => {
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
|
||||
});
|
||||
|
||||
it("falls back to the default targets when the configured list is empty", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
|
||||
|
||||
Reference in new issue
Block a user