Address the inline review on the egress check pipeline
Distinguish "the peer answered" from "nothing answered", widen attribution to
every check that leaves the instance, make the maintenance exit explicit, and
fix the recovery poll cadence.
Providers can now say outright whether the peer answered, via peerResponded on
MonitorStatusResponse. Previously the producer inferred it from an HTTP status
code, which two providers do not report:
- GrpcProvider: NOT_SERVING shares NETWORK_ERROR with a connection failure, and
gRPC codes 0-16 are not HTTP statuses, so PERMISSION_DENIED, UNIMPLEMENTED
and the rest were all read as transport failures. Only DEADLINE_EXCEEDED and
UNAVAILABLE, or a failure carrying no gRPC code, now count as unreachable.
- DNSProvider: every resolver error became NETWORK_ERROR, so an authoritative
NXDOMAIN, SERVFAIL or REFUSED was read as unreachable. The c-ares codes where
the server replied are now separated from timeout and connection refused.
Left unset, the HTTP-status heuristic still applies, so other providers are
unchanged.
Attribution now follows whether the check leaves the instance, not the monitor
type. A hardware check is an ordinary outbound HTTP request to the Capture
agent, and a Docker daemon can be reached over TCP, so both are attributable;
only a Docker daemon on a unix socket is local and exempt.
Leaving a maintenance window is now explicit in the producer, alongside the
entry it already owned. It was a side effect of evaluating a check, which the
degraded-egress short-circuit skips, so a monitor whose window ended during an
outage kept reporting "maintenance" for the length of it.
The recovery poll interval is fixed at 5 seconds and no longer configurable.
While degraded, checks are flagged from stored state without probing, so the
interval is the window in which a recovered instance still discards its checks;
the only effect of raising it was a longer blind window. egressPollIntervalSeconds
is removed from the settings, the validator and the OpenAPI spec.
🔧
This commit is contained in:
1 parent
d865103fb5
commit
3dc529733a
17 files changed
+265
-80
No files matched your search
@@ -209,6 +209,49 @@ describe("DNSProvider", () => {
|
||||
expect(mockResolverInstance.resolve4).toHaveBeenCalledWith("example.com");
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// The egress check must only blame the instance's own connectivity when nothing answered. Every DNS failure
|
||||
// carries NETWORK_ERROR, so `peerResponded` is the only thing separating an authoritative answer from silence.
|
||||
|
||||
it.each([
|
||||
["ENOTFOUND", "NXDOMAIN: the server answered that the name does not exist"],
|
||||
["ENODATA", "the server answered with no record of that type"],
|
||||
["ESERVFAIL", "the server answered with a failure"],
|
||||
["EREFUSED", "the server declined to answer"],
|
||||
["EFORMERR", "the server rejected the query as malformed"],
|
||||
["ENOTIMP", "the server does not implement the query"],
|
||||
["EBADRESP", "the server replied, badly"],
|
||||
])("reports the peer as having answered for %s", async (code) => {
|
||||
const error = Object.assign(new Error("dns failure"), { code });
|
||||
mockResolverInstance.resolve4.mockRejectedValueOnce(error);
|
||||
const provider = new DNSProvider(createResolver);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEOUT"], ["ECONNREFUSED"], ["ECANCELLED"], [undefined]])("reports the peer as unreachable for %s", async (code) => {
|
||||
const error = code ? Object.assign(new Error("dns failure"), { code }) : new Error("dns failure");
|
||||
mockResolverInstance.resolve4.mockRejectedValueOnce(error);
|
||||
const provider = new DNSProvider(createResolver);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered on success", async () => {
|
||||
const provider = new DNSProvider(createResolver);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("stringifies non-Error throws from outside timeRequest", async () => {
|
||||
mockResolverInstance.setServers.mockImplementationOnce(() => {
|
||||
throw 42;
|
||||
|
||||
@@ -362,6 +362,59 @@ describe("GrpcProvider", () => {
|
||||
[99, "UNKNOWN"],
|
||||
];
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────
|
||||
// NOT_SERVING and a connection failure both report NETWORK_ERROR, and gRPC codes are not HTTP statuses,
|
||||
// so `peerResponded` is what stops the egress check reading a server's answer as a transport failure.
|
||||
|
||||
it("reports the peer as having answered when the server replies NOT_SERVING", async () => {
|
||||
const { provider } = createProvider({ behavior: "not-serving" });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(5000);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([[5], [7], [12], [16], [2]])("reports the peer as having answered for gRPC code %i", async (code) => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
|
||||
close: jest.fn(),
|
||||
};
|
||||
const mocks = createMockGrpc(client as any);
|
||||
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([[4], [14]])("reports the peer as unreachable for gRPC code %i", async (code) => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
|
||||
close: jest.fn(),
|
||||
};
|
||||
const mocks = createMockGrpc(client as any);
|
||||
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as unreachable when the failure carries no gRPC code", async () => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ message: "socket hang up" })),
|
||||
close: jest.fn(),
|
||||
};
|
||||
const mocks = createMockGrpc(client as any);
|
||||
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it.each(statusCodes)("maps gRPC code %i to %s", async (code, expectedName) => {
|
||||
const client = {
|
||||
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
|
||||
|
||||
@@ -79,6 +79,31 @@ describe("CheckProducer", () => {
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
|
||||
const monitor = makeMonitor({ status: "maintenance" });
|
||||
const { producer, defaults } = createProducer({
|
||||
monitorsRepository: {
|
||||
updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }),
|
||||
findDockerTlsKeyById: jest.fn().mockResolvedValue(null),
|
||||
},
|
||||
});
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
|
||||
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not touch the status of a monitor that was not in maintenance", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
|
||||
await producer.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("proceeds with the check when the maintenance window is inactive", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
|
||||
@@ -265,23 +290,73 @@ describe("CheckProducer", () => {
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("never consults egress for a failing docker or hardware check", async () => {
|
||||
for (const type of ["docker", "hardware"] as const) {
|
||||
it("never consults egress for a docker daemon on a local socket", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" }));
|
||||
|
||||
// Local IPC never leaves the instance, so it cannot fail through egress.
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
});
|
||||
|
||||
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
|
||||
for (const monitor of [
|
||||
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
|
||||
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type }) },
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type }));
|
||||
await producer.produce(monitor);
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
// Both requests leave the instance, so our own loss of egress can be the cause.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not consult egress when the provider reports that the peer answered", async () => {
|
||||
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
|
||||
// them from a connection failure.
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "dns" }));
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "grpc" }));
|
||||
|
||||
// An explicit report from the provider outranks the HTTP-status heuristic.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
|
||||
@@ -21,7 +21,6 @@ const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
const makeSettings = (overrides?: Record<string, unknown>) => ({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: ["notif-1"],
|
||||
...overrides,
|
||||
});
|
||||
@@ -33,7 +32,7 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: 1_000,
|
||||
intervalMs: 30_000,
|
||||
intervalMs: 5_000,
|
||||
lockedBy: "worker-1",
|
||||
lockedUntil: 61_000,
|
||||
runCount: 0,
|
||||
@@ -245,7 +244,7 @@ describe("EgressService", () => {
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: expect.any(Number),
|
||||
intervalMs: 30_000,
|
||||
intervalMs: 5_000,
|
||||
});
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
|
||||
@@ -266,7 +265,7 @@ describe("EgressService", () => {
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
|
||||
});
|
||||
|
||||
@@ -293,21 +292,10 @@ describe("EgressService", () => {
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
|
||||
});
|
||||
|
||||
it("falls back to the default targets when the configured list is empty", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts";
|
||||
import { MAX_EGRESS_TARGETS } from "../../../src/domain/egress/egress.type.ts";
|
||||
|
||||
const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8";
|
||||
|
||||
@@ -10,13 +10,11 @@ describe("settingsValidation", () => {
|
||||
const parsed = updateAppSettingsBodyValidation.parse({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: [validNotificationId],
|
||||
});
|
||||
|
||||
expect(parsed.egressCheckEnabled).toBe(true);
|
||||
expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]);
|
||||
expect(parsed.egressPollIntervalSeconds).toBe(30);
|
||||
expect(parsed.egressNotifications).toEqual([validNotificationId]);
|
||||
});
|
||||
|
||||
@@ -62,21 +60,6 @@ describe("settingsValidation", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts the poll interval boundaries", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MIN_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MAX_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects out-of-range and non-integer poll intervals", () => {
|
||||
for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts an empty notification list", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user