Report whether the peer answered from every provider that can tell
An egress spell masked a service on the surviving network going down: with
`peerResponded` set only by the DNS and gRPC providers, every other failure fell
back to the HTTP-status heuristic, which answers "nothing replied" for anything
that is not speaking HTTP. A refused connection on the local network was read as
the instance's own loss of egress, so the check was recorded as degraded and the
monitor stayed up.
`peerAnsweredFromError` holds the errno codes that require the target's own stack
to have replied — a refusal, a reset, a TLS handshake that got as far as a
certificate. Http, Port, WebSocket and Docker use it; Http also treats any status
code as an answer, and Docker any status from the daemon. Hardware and PageSpeed
go through Http and inherit it.
`true` is the claim that costs something, since it blames the target, so it is
only set on evidence. Two cases stay conservative and are commented as such: a
ping that draws no echo reply, and a gamedig query that fails, neither of which
can be told apart from the instance losing its egress. Nor can a proxied HTTP
request that fails with no response, where the refusal may be the proxy's.
🔧
This commit is contained in:
1 parent
13799af721
commit
b55560e2f1
13 files changed
+318
-7
No files matched your search
@@ -697,4 +697,43 @@ describe("DockerProvider", () => {
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("Error performing Docker request");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// A daemon that answers with an error has been reached; only silence can be the instance's own egress.
|
||||
|
||||
describe("peer reachability", () => {
|
||||
it("reports the peer as having answered when the daemon replies with a status code", async () => {
|
||||
const dockerErr = Object.assign(new Error("base"), { statusCode: 500, json: { message: "engine exploded" } });
|
||||
const { provider } = setup({ ping: jest.fn().mockRejectedValue(dockerErr) });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
|
||||
const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
|
||||
const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered when the daemon is reachable", async () => {
|
||||
const { provider } = setup({});
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -643,4 +643,92 @@ describe("HttpProvider", () => {
|
||||
expect(result.message).toBe("Response code 500 (Internal Server Error)");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// Any HTTP status proves the target answered. Without one, only an errno that requires the peer's own stack
|
||||
// to have replied counts — and never through a proxy, where the refusal may be the proxy's rather than the
|
||||
// target's, which is the instance's own egress path failing.
|
||||
|
||||
describe("peer reachability", () => {
|
||||
it("reports the peer as having answered on a successful response", async () => {
|
||||
mockGot.mockResolvedValue(makeGotResponse());
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered for a failing status code", async () => {
|
||||
const err = new HTTPError("Not Found", { statusCode: 404, body: "", headers: {} }, { phases: { total: 12 } });
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(404);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"], ["CERT_HAS_EXPIRED"], ["DEPTH_ZERO_SELF_SIGNED_CERT"]])(
|
||||
"reports the peer as having answered for %s with no response",
|
||||
async (code) => {
|
||||
const err = new RequestError("request failed");
|
||||
(err as any).code = code;
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
}
|
||||
);
|
||||
|
||||
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["ENETUNREACH"]])("reports the peer as silent for %s", async (code) => {
|
||||
const err = new RequestError("request failed");
|
||||
(err as any).code = code;
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as silent when a proxy refuses the connection", async () => {
|
||||
const err = new RequestError("connect ECONNREFUSED");
|
||||
(err as any).code = "ECONNREFUSED";
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
|
||||
|
||||
// The refusal came from the proxy, which is part of this instance's route out.
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered through a proxy when a status code came back", async () => {
|
||||
const err = new RequestError("Response code 500 (Internal Server Error)");
|
||||
(err as any).code = "ECONNRESET";
|
||||
(err as any).response = { statusCode: 500, body: "", headers: {} };
|
||||
(err as any).timings = { phases: { total: 25 } };
|
||||
mockGot.mockRejectedValue(err);
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the peer as silent for a generic error carrying no code", async () => {
|
||||
mockGot.mockRejectedValue(new Error("something broke"));
|
||||
const { provider } = createProvider();
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -155,4 +155,23 @@ describe("PingProvider", () => {
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
|
||||
it("reports the peer as having answered when an echo reply comes back", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the peer as silent when no echo reply comes back", async () => {
|
||||
// Silence is indistinguishable from the instance losing its egress, so the egress probe decides.
|
||||
const provider = new PingProvider(createMockPing({ alive: false }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -267,4 +267,42 @@ describe("PortProvider", () => {
|
||||
|
||||
await expect(provider.handle(monitor)).rejects.toThrow("42");
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// A check is only recorded as degraded when nothing answered, so a service on the local network that
|
||||
// refuses connections has to stay distinguishable from the instance losing its own egress.
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
|
||||
const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
|
||||
const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as silent on a connection timeout", async () => {
|
||||
const provider = new PortProvider(createTimeoutNet());
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered when the port accepts the connection", async () => {
|
||||
const provider = new PortProvider(createSuccessNet());
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { timeRequest, isStatusUp } from "../../../../src/service/network/utils.ts";
|
||||
import { timeRequest, isStatusUp, peerAnsweredFromError } from "../../../../src/service/network/utils.ts";
|
||||
import { NETWORK_ERROR } from "../../../../src/types/network.ts";
|
||||
|
||||
describe("network utils", () => {
|
||||
@@ -39,6 +39,40 @@ describe("network utils", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// The egress check blames the instance only when nothing answered, so this helper's `true` must never be a
|
||||
// guess: every code in it requires the peer's own stack to have replied.
|
||||
describe("peerAnsweredFromError", () => {
|
||||
it.each([
|
||||
["ECONNREFUSED", "the host refused the connection"],
|
||||
["ECONNRESET", "the host reset an established connection"],
|
||||
["EPIPE", "the host closed a connection we were writing to"],
|
||||
["EPROTO", "a protocol error on a connection that was made"],
|
||||
["CERT_HAS_EXPIRED", "the host presented a certificate"],
|
||||
["DEPTH_ZERO_SELF_SIGNED_CERT", "the host presented a certificate"],
|
||||
["SELF_SIGNED_CERT_IN_CHAIN", "the host presented a certificate"],
|
||||
["UNABLE_TO_GET_ISSUER_CERT_LOCALLY", "the host presented a certificate"],
|
||||
["UNABLE_TO_VERIFY_LEAF_SIGNATURE", "the host presented a certificate"],
|
||||
["ERR_TLS_CERT_ALTNAME_INVALID", "the host presented a certificate"],
|
||||
["CERT_NOT_YET_VALID", "the host presented a certificate"],
|
||||
["ERR_SSL_WRONG_VERSION_NUMBER", "the host spoke back, badly"],
|
||||
])("reports the peer as having answered for %s", (code) => {
|
||||
expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ECONNABORTED"], ["ENETDOWN"]])(
|
||||
"reports the peer as silent for %s",
|
||||
(code) => {
|
||||
expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
it("reports the peer as silent when the error carries no code", () => {
|
||||
expect(peerAnsweredFromError(new Error("Connection timeout"))).toBe(false);
|
||||
expect(peerAnsweredFromError(undefined)).toBe(false);
|
||||
expect(peerAnsweredFromError("not an error")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isStatusUp", () => {
|
||||
it("returns true for 2xx status codes", () => {
|
||||
expect(isStatusUp(200)).toBe(true);
|
||||
|
||||
@@ -191,4 +191,42 @@ describe("WebSocketProvider", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Peer reachability ────────────────────────────────────────────────────
|
||||
// `ws` rejects a non-101 handshake with a plain Error, so only the errno cases can be told apart from the
|
||||
// instance losing its egress. That leaves a server answering the handshake with an HTTP error indistinct.
|
||||
|
||||
const createErrorWS = (error: Error) =>
|
||||
jest.fn().mockImplementation(() => ({
|
||||
on: jest.fn((event: string, cb: Function) => {
|
||||
if (event === "error") process.nextTick(() => cb(error));
|
||||
}),
|
||||
close: jest.fn(),
|
||||
})) as any;
|
||||
|
||||
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
|
||||
const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.status).toBe(false);
|
||||
expect(result.code).toBe(NETWORK_ERROR);
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
|
||||
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EHOSTUNREACH"]])("reports the peer as silent for %s", async (code) => {
|
||||
const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the peer as having answered when the socket opens", async () => {
|
||||
const provider = new WebSocketProvider(createMockWS("open"));
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.peerResponded).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user