Report whether the peer answered from every provider that can tell

An egress spell masked a service on the surviving network going down: with
`peerResponded` set only by the DNS and gRPC providers, every other failure fell
back to the HTTP-status heuristic, which answers "nothing replied" for anything
that is not speaking HTTP. A refused connection on the local network was read as
the instance's own loss of egress, so the check was recorded as degraded and the
monitor stayed up.

`peerAnsweredFromError` holds the errno codes that require the target's own stack
to have replied — a refusal, a reset, a TLS handshake that got as far as a
certificate. Http, Port, WebSocket and Docker use it; Http also treats any status
code as an answer, and Docker any status from the daemon. Hardware and PageSpeed
go through Http and inherit it.

`true` is the claim that costs something, since it blames the target, so it is
only set on evidence. Two cases stay conservative and are commented as such: a
ping that draws no echo reply, and a gamedig query that fails, neither of which
can be told apart from the instance losing its egress. Nor can a proxied HTTP
request that fails with no response, where the refusal may be the proxy's.

🔧

server/src/service/network/DockerProvider.ts

@@ -23,7 +23,7 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
 import { ILogger } from "@/utils/logger.js";
 import { AppError } from "@/utils/AppError.js";
 import Dockerode from "dockerode";
-import { timeRequest } from "@/service/network/utils.js";
+import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import { IEncryptionService } from "@/service/encryption/encryptionService.js";
 import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
@@ -315,12 +315,19 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
 		return rawLogs.filter((log) => log !== null);
 	};
 
-	private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse<DockerStatusPayload> => ({
+	private failed = (
+		monitor: Monitor,
+		code: number,
+		message: string,
+		responseTime: number,
+		peerResponded = false
+	): MonitorStatusResponse<DockerStatusPayload> => ({
 		monitorId: monitor.id,
 		teamId: monitor.teamId,
 		type: monitor.type,
 		status: false,
 		code,
+		peerResponded,
 		message,
 		responseTime,
 		payload: null,
@@ -337,13 +344,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
 			if (error) {
 				let message = "Docker host is unreachable";
 				let code = NETWORK_ERROR;
+				// A status code from the daemon is the daemon answering; otherwise only a refusal or a reset
+				// proves we reached the host, and anything else could be the instance's own loss of egress.
+				let peerResponded = peerAnsweredFromError(error);
 				if (this.isDockerError(error)) {
 					code = error.statusCode ?? NETWORK_ERROR;
 					message = error.json?.message ?? error.reason ?? error.message;
+					peerResponded = peerResponded || error.statusCode !== undefined;
 				} else if (error instanceof Error) {
 					message = error.message;
 				}
-				return this.failed(monitor, code, message, responseTime);
+				return this.failed(monitor, code, message, responseTime, peerResponded);
 			}
 
 			const summaries = await docker.listContainers({ all: true });
@@ -362,6 +373,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
 				type: monitor.type,
 				status: true,
 				code: 200,
+				peerResponded: true,
 				message: "Docker host is reachable",
 				responseTime,
 				payload: { containers, summary, logs },

server/src/service/network/GameProvider.ts

@@ -43,6 +43,9 @@ export class GameProvider implements IStatusProvider<GameStatusPayload> {
 				});
 
 			if (!state) {
+				// gamedig reports a failed query the same way whichever leg failed, so `peerResponded` is left
+				// unset and a failure is put to the egress probe. A game server that stops answering during a
+				// spell is therefore recorded as degraded rather than down.
 				return {
 					monitorId: id,
 					teamId: teamId,

server/src/service/network/HttpProvider.ts

@@ -6,7 +6,7 @@ import { MonitorStatusResponse } from "@/types/network.js";
 import { Agent as HttpsAgent } from "https";
 import { Agent as HttpAgent } from "http";
 import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
-import { isStatusUp } from "@/service/network/utils.js";
+import { isStatusUp, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import CacheableLookup from "cacheable-lookup";
 import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
@@ -107,6 +107,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 				type: monitor.type,
 				status: statusUp,
 				code: statusCode,
+				peerResponded: true,
 				message,
 				responseTime,
 				timings,
@@ -123,6 +124,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 				type: monitor.type,
 				status: false,
 				code: statusCode,
+				peerResponded: true,
 				message: "Response is not JSON",
 				responseTime,
 				timings,
@@ -149,6 +151,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 			type: monitor.type,
 			status: statusUp && matchResult.ok,
 			code: statusCode,
+			peerResponded: true,
 			message: matchResult.ok ? message : matchResult.message,
 			responseTime,
 			timings,
@@ -172,6 +175,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 					type: monitor.type,
 					status: false,
 					code: statusCode ?? NETWORK_ERROR,
+					peerResponded: statusCode !== undefined || (!ctx?.proxyUrl && peerAnsweredFromError(error)),
 					message: message,
 					responseTime,
 					timings: error.timings,
@@ -196,6 +200,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 			type: monitor.type,
 			status: false,
 			code: NETWORK_ERROR,
+			peerResponded: !ctx?.proxyUrl && peerAnsweredFromError(error),
 			message: error instanceof Error ? error.message : String(error),
 			responseTime: 0,
 			payload: null as T,

server/src/service/network/PingProvider.ts

@@ -58,6 +58,9 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
 				type: monitor.type,
 				status: response.alive ?? false,
 				code: response.alive ? 200 : 5000,
+				// An echo reply is the host answering. Its absence is silence, which is indistinguishable
+				// from the instance having lost its egress, so the egress probe decides that case.
+				peerResponded: response.alive ?? false,
 				message: response.alive ? "Success" : "Ping failed",
 				responseTime,
 				payload: response,

server/src/service/network/PortProvider.ts

@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
 import { PortStatusPayload, MonitorStatusResponse } from "@/types/network.js";
 import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
 import { AppError } from "@/utils/AppError.js";
-import { timeRequest } from "@/service/network/utils.js";
+import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import * as net from "net";
 type NetType = typeof net;
@@ -59,6 +59,9 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
 					type: monitor.type,
 					status: false,
 					code: NETWORK_ERROR,
+					// A refused or reset connection came from the host itself; a timeout or an unreachable
+					// network did not, and could equally be the instance's own loss of egress.
+					peerResponded: peerAnsweredFromError(error),
 					message: errorMessage,
 					responseTime: responseTime,
 					timings: undefined,
@@ -72,6 +75,7 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
 				type: monitor.type,
 				status: true,
 				code: 200,
+				peerResponded: true,
 				message: "Port check successful",
 				responseTime: responseTime,
 				timings: undefined,

server/src/service/network/WebSocketProvider.ts

@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
 import { WebSocketStatusPayload, MonitorStatusResponse } from "@/types/network.js";
 import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
 import { AppError } from "@/utils/AppError.js";
-import { timeRequest } from "@/service/network/utils.js";
+import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import type WebSocket from "ws";
 
@@ -63,6 +63,9 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
 					type: monitor.type,
 					status: false,
 					code: NETWORK_ERROR,
+					// `ws` rejects a non-101 handshake with a plain Error carrying only a message, so only the
+					// errno cases — a refusal or a reset — can be told apart from the instance losing egress.
+					peerResponded: peerAnsweredFromError(error),
 					message: errorMessage,
 					responseTime: responseTime,
 					timings: undefined,
@@ -76,6 +79,7 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
 				type: monitor.type,
 				status: true,
 				code: 200,
+				peerResponded: true,
 				message: "WebSocket check successful",
 				responseTime: responseTime,
 				timings: undefined,

server/src/service/network/utils.ts

@@ -16,3 +16,27 @@ export const isStatusUp = (statusCode: number | undefined, customUpCodes: HttpSt
 	if (statusCode === undefined) return false;
 	return (statusCode >= 200 && statusCode < 300) || customUpCodes.includes(statusCode);
 };
+
+// Errno codes that can only arise once the target's own stack has answered: it refused the connection, reset
+// an established one, or got far enough into a TLS handshake to present a certificate. Every other failure —
+// a timeout, an unreachable network, a name that would not resolve — looks the same whether the target is
+// down or the instance has lost its egress, so those are left for the egress probe to decide.
+const PEER_ANSWERED_ERROR_CODES: ReadonlySet<string> = new Set([
+	"ECONNREFUSED",
+	"ECONNRESET",
+	"EPIPE",
+	"EPROTO",
+	"CERT_HAS_EXPIRED",
+	"CERT_NOT_YET_VALID",
+	"DEPTH_ZERO_SELF_SIGNED_CERT",
+	"SELF_SIGNED_CERT_IN_CHAIN",
+	"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
+	"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
+	"ERR_TLS_CERT_ALTNAME_INVALID",
+	"ERR_SSL_WRONG_VERSION_NUMBER",
+]);
+
+export const peerAnsweredFromError = (error: unknown): boolean => {
+	const code = (error as NodeJS.ErrnoException | undefined)?.code;
+	return typeof code === "string" && PEER_ANSWERED_ERROR_CODES.has(code);
+};

server/test/unit/providers/network/dockerProvider.test.ts

@@ -697,4 +697,43 @@ describe("DockerProvider", () => {
 			await expect(provider.handle(makeMonitor())).rejects.toThrow("Error performing Docker request");
 		});
 	});
+
+	// ── Peer reachability ────────────────────────────────────────────────────
+	// A daemon that answers with an error has been reached; only silence can be the instance's own egress.
+
+	describe("peer reachability", () => {
+		it("reports the peer as having answered when the daemon replies with a status code", async () => {
+			const dockerErr = Object.assign(new Error("base"), { statusCode: 500, json: { message: "engine exploded" } });
+			const { provider } = setup({ ping: jest.fn().mockRejectedValue(dockerErr) });
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.status).toBe(false);
+			expect(result.peerResponded).toBe(true);
+		});
+
+		it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
+			const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.peerResponded).toBe(true);
+		});
+
+		it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
+			const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.peerResponded).toBe(false);
+		});
+
+		it("reports the peer as having answered when the daemon is reachable", async () => {
+			const { provider } = setup({});
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.peerResponded).toBe(true);
+		});
+	});
 });

server/test/unit/providers/network/httpProvider.test.ts

@@ -643,4 +643,92 @@ describe("HttpProvider", () => {
 			expect(result.message).toBe("Response code 500 (Internal Server Error)");
 		});
 	});
+
+	// ── Peer reachability ────────────────────────────────────────────────────
+	// Any HTTP status proves the target answered. Without one, only an errno that requires the peer's own stack
+	// to have replied counts — and never through a proxy, where the refusal may be the proxy's rather than the
+	// target's, which is the instance's own egress path failing.
+
+	describe("peer reachability", () => {
+		it("reports the peer as having answered on a successful response", async () => {
+			mockGot.mockResolvedValue(makeGotResponse());
+			const { provider } = createProvider();
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.peerResponded).toBe(true);
+		});
+
+		it("reports the peer as having answered for a failing status code", async () => {
+			const err = new HTTPError("Not Found", { statusCode: 404, body: "", headers: {} }, { phases: { total: 12 } });
+			mockGot.mockRejectedValue(err);
+			const { provider } = createProvider();
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.status).toBe(false);
+			expect(result.code).toBe(404);
+			expect(result.peerResponded).toBe(true);
+		});
+
+		it.each([["ECONNREFUSED"], ["ECONNRESET"], ["CERT_HAS_EXPIRED"], ["DEPTH_ZERO_SELF_SIGNED_CERT"]])(
+			"reports the peer as having answered for %s with no response",
+			async (code) => {
+				const err = new RequestError("request failed");
+				(err as any).code = code;
+				mockGot.mockRejectedValue(err);
+				const { provider } = createProvider();
+
+				const result = await provider.handle(makeMonitor());
+
+				expect(result.code).toBe(NETWORK_ERROR);
+				expect(result.peerResponded).toBe(true);
+			}
+		);
+
+		it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["ENETUNREACH"]])("reports the peer as silent for %s", async (code) => {
+			const err = new RequestError("request failed");
+			(err as any).code = code;
+			mockGot.mockRejectedValue(err);
+			const { provider } = createProvider();
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.peerResponded).toBe(false);
+		});
+
+		it("reports the peer as silent when a proxy refuses the connection", async () => {
+			const err = new RequestError("connect ECONNREFUSED");
+			(err as any).code = "ECONNREFUSED";
+			mockGot.mockRejectedValue(err);
+			const { provider } = createProvider();
+
+			const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
+
+			// The refusal came from the proxy, which is part of this instance's route out.
+			expect(result.peerResponded).toBe(false);
+		});
+
+		it("reports the peer as having answered through a proxy when a status code came back", async () => {
+			const err = new RequestError("Response code 500 (Internal Server Error)");
+			(err as any).code = "ECONNRESET";
+			(err as any).response = { statusCode: 500, body: "", headers: {} };
+			(err as any).timings = { phases: { total: 25 } };
+			mockGot.mockRejectedValue(err);
+			const { provider } = createProvider();
+
+			const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
+
+			expect(result.peerResponded).toBe(true);
+		});
+
+		it("reports the peer as silent for a generic error carrying no code", async () => {
+			mockGot.mockRejectedValue(new Error("something broke"));
+			const { provider } = createProvider();
+
+			const result = await provider.handle(makeMonitor());
+
+			expect(result.peerResponded).toBe(false);
+		});
+	});
 });

server/test/unit/providers/network/pingProvider.test.ts

@@ -155,4 +155,23 @@ describe("PingProvider", () => {
 
 		await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
 	});
+
+	// ── Peer reachability ────────────────────────────────────────────────────
+
+	it("reports the peer as having answered when an echo reply comes back", async () => {
+		const provider = new PingProvider(createMockPing({ alive: true }) as any, net);
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.peerResponded).toBe(true);
+	});
+
+	it("reports the peer as silent when no echo reply comes back", async () => {
+		// Silence is indistinguishable from the instance losing its egress, so the egress probe decides.
+		const provider = new PingProvider(createMockPing({ alive: false }) as any, net);
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.peerResponded).toBe(false);
+	});
 });

server/test/unit/providers/network/portProvider.test.ts

@@ -267,4 +267,42 @@ describe("PortProvider", () => {
 
 		await expect(provider.handle(monitor)).rejects.toThrow("42");
 	});
+
+	// ── Peer reachability ────────────────────────────────────────────────────
+	// A check is only recorded as degraded when nothing answered, so a service on the local network that
+	// refuses connections has to stay distinguishable from the instance losing its own egress.
+
+	it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
+		const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.status).toBe(false);
+		expect(result.code).toBe(NETWORK_ERROR);
+		expect(result.peerResponded).toBe(true);
+	});
+
+	it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
+		const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.peerResponded).toBe(false);
+	});
+
+	it("reports the peer as silent on a connection timeout", async () => {
+		const provider = new PortProvider(createTimeoutNet());
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.peerResponded).toBe(false);
+	});
+
+	it("reports the peer as having answered when the port accepts the connection", async () => {
+		const provider = new PortProvider(createSuccessNet());
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.peerResponded).toBe(true);
+	});
 });

server/test/unit/providers/network/utils.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "@jest/globals";
-import { timeRequest, isStatusUp } from "../../../../src/service/network/utils.ts";
+import { timeRequest, isStatusUp, peerAnsweredFromError } from "../../../../src/service/network/utils.ts";
 import { NETWORK_ERROR } from "../../../../src/types/network.ts";
 
 describe("network utils", () => {
@@ -39,6 +39,40 @@ describe("network utils", () => {
 		});
 	});
 
+	// The egress check blames the instance only when nothing answered, so this helper's `true` must never be a
+	// guess: every code in it requires the peer's own stack to have replied.
+	describe("peerAnsweredFromError", () => {
+		it.each([
+			["ECONNREFUSED", "the host refused the connection"],
+			["ECONNRESET", "the host reset an established connection"],
+			["EPIPE", "the host closed a connection we were writing to"],
+			["EPROTO", "a protocol error on a connection that was made"],
+			["CERT_HAS_EXPIRED", "the host presented a certificate"],
+			["DEPTH_ZERO_SELF_SIGNED_CERT", "the host presented a certificate"],
+			["SELF_SIGNED_CERT_IN_CHAIN", "the host presented a certificate"],
+			["UNABLE_TO_GET_ISSUER_CERT_LOCALLY", "the host presented a certificate"],
+			["UNABLE_TO_VERIFY_LEAF_SIGNATURE", "the host presented a certificate"],
+			["ERR_TLS_CERT_ALTNAME_INVALID", "the host presented a certificate"],
+			["CERT_NOT_YET_VALID", "the host presented a certificate"],
+			["ERR_SSL_WRONG_VERSION_NUMBER", "the host spoke back, badly"],
+		])("reports the peer as having answered for %s", (code) => {
+			expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(true);
+		});
+
+		it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ECONNABORTED"], ["ENETDOWN"]])(
+			"reports the peer as silent for %s",
+			(code) => {
+				expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(false);
+			}
+		);
+
+		it("reports the peer as silent when the error carries no code", () => {
+			expect(peerAnsweredFromError(new Error("Connection timeout"))).toBe(false);
+			expect(peerAnsweredFromError(undefined)).toBe(false);
+			expect(peerAnsweredFromError("not an error")).toBe(false);
+		});
+	});
+
 	describe("isStatusUp", () => {
 		it("returns true for 2xx status codes", () => {
 			expect(isStatusUp(200)).toBe(true);

server/test/unit/providers/network/webSocketProvider.test.ts

@@ -191,4 +191,42 @@ describe("WebSocketProvider", () => {
 			);
 		});
 	});
+
+	// ── Peer reachability ────────────────────────────────────────────────────
+	// `ws` rejects a non-101 handshake with a plain Error, so only the errno cases can be told apart from the
+	// instance losing its egress. That leaves a server answering the handshake with an HTTP error indistinct.
+
+	const createErrorWS = (error: Error) =>
+		jest.fn().mockImplementation(() => ({
+			on: jest.fn((event: string, cb: Function) => {
+				if (event === "error") process.nextTick(() => cb(error));
+			}),
+			close: jest.fn(),
+		})) as any;
+
+	it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
+		const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.status).toBe(false);
+		expect(result.code).toBe(NETWORK_ERROR);
+		expect(result.peerResponded).toBe(true);
+	});
+
+	it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EHOSTUNREACH"]])("reports the peer as silent for %s", async (code) => {
+		const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.peerResponded).toBe(false);
+	});
+
+	it("reports the peer as having answered when the socket opens", async () => {
+		const provider = new WebSocketProvider(createMockWS("open"));
+
+		const result = await provider.handle(makeMonitor());
+
+		expect(result.peerResponded).toBe(true);
+	});
 });