diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index aa8070649..7ea35c13b 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -23,7 +23,7 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js"; import { ILogger } from "@/utils/logger.js"; import { AppError } from "@/utils/AppError.js"; import Dockerode from "dockerode"; -import { timeRequest } from "@/service/network/utils.js"; +import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js"; import { NETWORK_ERROR } from "@/types/network.js"; import { IEncryptionService } from "@/service/encryption/encryptionService.js"; import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js"; @@ -315,12 +315,19 @@ export class DockerProvider implements IStatusProvider { return rawLogs.filter((log) => log !== null); }; - private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse => ({ + private failed = ( + monitor: Monitor, + code: number, + message: string, + responseTime: number, + peerResponded = false + ): MonitorStatusResponse => ({ monitorId: monitor.id, teamId: monitor.teamId, type: monitor.type, status: false, code, + peerResponded, message, responseTime, payload: null, @@ -337,13 +344,17 @@ export class DockerProvider implements IStatusProvider { if (error) { let message = "Docker host is unreachable"; let code = NETWORK_ERROR; + // A status code from the daemon is the daemon answering; otherwise only a refusal or a reset + // proves we reached the host, and anything else could be the instance's own loss of egress. + let peerResponded = peerAnsweredFromError(error); if (this.isDockerError(error)) { code = error.statusCode ?? NETWORK_ERROR; message = error.json?.message ?? error.reason ?? error.message; + peerResponded = peerResponded || error.statusCode !== undefined; } else if (error instanceof Error) { message = error.message; } - return this.failed(monitor, code, message, responseTime); + return this.failed(monitor, code, message, responseTime, peerResponded); } const summaries = await docker.listContainers({ all: true }); @@ -362,6 +373,7 @@ export class DockerProvider implements IStatusProvider { type: monitor.type, status: true, code: 200, + peerResponded: true, message: "Docker host is reachable", responseTime, payload: { containers, summary, logs }, diff --git a/server/src/service/network/GameProvider.ts b/server/src/service/network/GameProvider.ts index 5ccfaee6e..904f304f7 100644 --- a/server/src/service/network/GameProvider.ts +++ b/server/src/service/network/GameProvider.ts @@ -43,6 +43,9 @@ export class GameProvider implements IStatusProvider { }); if (!state) { + // gamedig reports a failed query the same way whichever leg failed, so `peerResponded` is left + // unset and a failure is put to the egress probe. A game server that stops answering during a + // spell is therefore recorded as degraded rather than down. return { monitorId: id, teamId: teamId, diff --git a/server/src/service/network/HttpProvider.ts b/server/src/service/network/HttpProvider.ts index 1ef35f474..9fa705b55 100644 --- a/server/src/service/network/HttpProvider.ts +++ b/server/src/service/network/HttpProvider.ts @@ -6,7 +6,7 @@ import { MonitorStatusResponse } from "@/types/network.js"; import { Agent as HttpsAgent } from "https"; import { Agent as HttpAgent } from "http"; import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js"; -import { isStatusUp } from "@/service/network/utils.js"; +import { isStatusUp, peerAnsweredFromError } from "@/service/network/utils.js"; import { NETWORK_ERROR } from "@/types/network.js"; import CacheableLookup from "cacheable-lookup"; import { HttpProxyAgent, HttpsProxyAgent } from "hpagent"; @@ -107,6 +107,7 @@ export class HttpProvider implements IStatusProvider { type: monitor.type, status: statusUp, code: statusCode, + peerResponded: true, message, responseTime, timings, @@ -123,6 +124,7 @@ export class HttpProvider implements IStatusProvider { type: monitor.type, status: false, code: statusCode, + peerResponded: true, message: "Response is not JSON", responseTime, timings, @@ -149,6 +151,7 @@ export class HttpProvider implements IStatusProvider { type: monitor.type, status: statusUp && matchResult.ok, code: statusCode, + peerResponded: true, message: matchResult.ok ? message : matchResult.message, responseTime, timings, @@ -172,6 +175,7 @@ export class HttpProvider implements IStatusProvider { type: monitor.type, status: false, code: statusCode ?? NETWORK_ERROR, + peerResponded: statusCode !== undefined || (!ctx?.proxyUrl && peerAnsweredFromError(error)), message: message, responseTime, timings: error.timings, @@ -196,6 +200,7 @@ export class HttpProvider implements IStatusProvider { type: monitor.type, status: false, code: NETWORK_ERROR, + peerResponded: !ctx?.proxyUrl && peerAnsweredFromError(error), message: error instanceof Error ? error.message : String(error), responseTime: 0, payload: null as T, diff --git a/server/src/service/network/PingProvider.ts b/server/src/service/network/PingProvider.ts index 7bcde8c7e..7a6abb33a 100644 --- a/server/src/service/network/PingProvider.ts +++ b/server/src/service/network/PingProvider.ts @@ -58,6 +58,9 @@ export class PingProvider implements IStatusProvider { type: monitor.type, status: response.alive ?? false, code: response.alive ? 200 : 5000, + // An echo reply is the host answering. Its absence is silence, which is indistinguishable + // from the instance having lost its egress, so the egress probe decides that case. + peerResponded: response.alive ?? false, message: response.alive ? "Success" : "Ping failed", responseTime, payload: response, diff --git a/server/src/service/network/PortProvider.ts b/server/src/service/network/PortProvider.ts index 692ca40af..bb21d0799 100644 --- a/server/src/service/network/PortProvider.ts +++ b/server/src/service/network/PortProvider.ts @@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js"; import { PortStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js"; import { AppError } from "@/utils/AppError.js"; -import { timeRequest } from "@/service/network/utils.js"; +import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js"; import { NETWORK_ERROR } from "@/types/network.js"; import * as net from "net"; type NetType = typeof net; @@ -59,6 +59,9 @@ export class PortProvider implements IStatusProvider { type: monitor.type, status: false, code: NETWORK_ERROR, + // A refused or reset connection came from the host itself; a timeout or an unreachable + // network did not, and could equally be the instance's own loss of egress. + peerResponded: peerAnsweredFromError(error), message: errorMessage, responseTime: responseTime, timings: undefined, @@ -72,6 +75,7 @@ export class PortProvider implements IStatusProvider { type: monitor.type, status: true, code: 200, + peerResponded: true, message: "Port check successful", responseTime: responseTime, timings: undefined, diff --git a/server/src/service/network/WebSocketProvider.ts b/server/src/service/network/WebSocketProvider.ts index d27106989..be9873d41 100644 --- a/server/src/service/network/WebSocketProvider.ts +++ b/server/src/service/network/WebSocketProvider.ts @@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js"; import { WebSocketStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js"; import { AppError } from "@/utils/AppError.js"; -import { timeRequest } from "@/service/network/utils.js"; +import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js"; import { NETWORK_ERROR } from "@/types/network.js"; import type WebSocket from "ws"; @@ -63,6 +63,9 @@ export class WebSocketProvider implements IStatusProvider= 200 && statusCode < 300) || customUpCodes.includes(statusCode); }; + +// Errno codes that can only arise once the target's own stack has answered: it refused the connection, reset +// an established one, or got far enough into a TLS handshake to present a certificate. Every other failure — +// a timeout, an unreachable network, a name that would not resolve — looks the same whether the target is +// down or the instance has lost its egress, so those are left for the egress probe to decide. +const PEER_ANSWERED_ERROR_CODES: ReadonlySet = new Set([ + "ECONNREFUSED", + "ECONNRESET", + "EPIPE", + "EPROTO", + "CERT_HAS_EXPIRED", + "CERT_NOT_YET_VALID", + "DEPTH_ZERO_SELF_SIGNED_CERT", + "SELF_SIGNED_CERT_IN_CHAIN", + "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", + "ERR_TLS_CERT_ALTNAME_INVALID", + "ERR_SSL_WRONG_VERSION_NUMBER", +]); + +export const peerAnsweredFromError = (error: unknown): boolean => { + const code = (error as NodeJS.ErrnoException | undefined)?.code; + return typeof code === "string" && PEER_ANSWERED_ERROR_CODES.has(code); +}; diff --git a/server/test/unit/providers/network/dockerProvider.test.ts b/server/test/unit/providers/network/dockerProvider.test.ts index 58b2d7903..2259a1e96 100644 --- a/server/test/unit/providers/network/dockerProvider.test.ts +++ b/server/test/unit/providers/network/dockerProvider.test.ts @@ -697,4 +697,43 @@ describe("DockerProvider", () => { await expect(provider.handle(makeMonitor())).rejects.toThrow("Error performing Docker request"); }); }); + + // ── Peer reachability ──────────────────────────────────────────────────── + // A daemon that answers with an error has been reached; only silence can be the instance's own egress. + + describe("peer reachability", () => { + it("reports the peer as having answered when the daemon replies with a status code", async () => { + const dockerErr = Object.assign(new Error("base"), { statusCode: 500, json: { message: "engine exploded" } }); + const { provider } = setup({ ping: jest.fn().mockRejectedValue(dockerErr) }); + + const result = await provider.handle(makeMonitor()); + + expect(result.status).toBe(false); + expect(result.peerResponded).toBe(true); + }); + + it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => { + const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) }); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(true); + }); + + it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => { + const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) }); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(false); + }); + + it("reports the peer as having answered when the daemon is reachable", async () => { + const { provider } = setup({}); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(true); + }); + }); }); diff --git a/server/test/unit/providers/network/httpProvider.test.ts b/server/test/unit/providers/network/httpProvider.test.ts index 4e559ab06..928c46f43 100644 --- a/server/test/unit/providers/network/httpProvider.test.ts +++ b/server/test/unit/providers/network/httpProvider.test.ts @@ -643,4 +643,92 @@ describe("HttpProvider", () => { expect(result.message).toBe("Response code 500 (Internal Server Error)"); }); }); + + // ── Peer reachability ──────────────────────────────────────────────────── + // Any HTTP status proves the target answered. Without one, only an errno that requires the peer's own stack + // to have replied counts — and never through a proxy, where the refusal may be the proxy's rather than the + // target's, which is the instance's own egress path failing. + + describe("peer reachability", () => { + it("reports the peer as having answered on a successful response", async () => { + mockGot.mockResolvedValue(makeGotResponse()); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(true); + }); + + it("reports the peer as having answered for a failing status code", async () => { + const err = new HTTPError("Not Found", { statusCode: 404, body: "", headers: {} }, { phases: { total: 12 } }); + mockGot.mockRejectedValue(err); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor()); + + expect(result.status).toBe(false); + expect(result.code).toBe(404); + expect(result.peerResponded).toBe(true); + }); + + it.each([["ECONNREFUSED"], ["ECONNRESET"], ["CERT_HAS_EXPIRED"], ["DEPTH_ZERO_SELF_SIGNED_CERT"]])( + "reports the peer as having answered for %s with no response", + async (code) => { + const err = new RequestError("request failed"); + (err as any).code = code; + mockGot.mockRejectedValue(err); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor()); + + expect(result.code).toBe(NETWORK_ERROR); + expect(result.peerResponded).toBe(true); + } + ); + + it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["ENETUNREACH"]])("reports the peer as silent for %s", async (code) => { + const err = new RequestError("request failed"); + (err as any).code = code; + mockGot.mockRejectedValue(err); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(false); + }); + + it("reports the peer as silent when a proxy refuses the connection", async () => { + const err = new RequestError("connect ECONNREFUSED"); + (err as any).code = "ECONNREFUSED"; + mockGot.mockRejectedValue(err); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" }); + + // The refusal came from the proxy, which is part of this instance's route out. + expect(result.peerResponded).toBe(false); + }); + + it("reports the peer as having answered through a proxy when a status code came back", async () => { + const err = new RequestError("Response code 500 (Internal Server Error)"); + (err as any).code = "ECONNRESET"; + (err as any).response = { statusCode: 500, body: "", headers: {} }; + (err as any).timings = { phases: { total: 25 } }; + mockGot.mockRejectedValue(err); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" }); + + expect(result.peerResponded).toBe(true); + }); + + it("reports the peer as silent for a generic error carrying no code", async () => { + mockGot.mockRejectedValue(new Error("something broke")); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(false); + }); + }); }); diff --git a/server/test/unit/providers/network/pingProvider.test.ts b/server/test/unit/providers/network/pingProvider.test.ts index 150472925..eaa6227b5 100644 --- a/server/test/unit/providers/network/pingProvider.test.ts +++ b/server/test/unit/providers/network/pingProvider.test.ts @@ -155,4 +155,23 @@ describe("PingProvider", () => { await expect(provider.handle(makeMonitor())).rejects.toThrow("42"); }); + + // ── Peer reachability ──────────────────────────────────────────────────── + + it("reports the peer as having answered when an echo reply comes back", async () => { + const provider = new PingProvider(createMockPing({ alive: true }) as any, net); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(true); + }); + + it("reports the peer as silent when no echo reply comes back", async () => { + // Silence is indistinguishable from the instance losing its egress, so the egress probe decides. + const provider = new PingProvider(createMockPing({ alive: false }) as any, net); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(false); + }); }); diff --git a/server/test/unit/providers/network/portProvider.test.ts b/server/test/unit/providers/network/portProvider.test.ts index fbbfb5767..caaf9ad3d 100644 --- a/server/test/unit/providers/network/portProvider.test.ts +++ b/server/test/unit/providers/network/portProvider.test.ts @@ -267,4 +267,42 @@ describe("PortProvider", () => { await expect(provider.handle(monitor)).rejects.toThrow("42"); }); + + // ── Peer reachability ──────────────────────────────────────────────────── + // A check is only recorded as degraded when nothing answered, so a service on the local network that + // refuses connections has to stay distinguishable from the instance losing its own egress. + + it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => { + const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code }))); + + const result = await provider.handle(makeMonitor()); + + expect(result.status).toBe(false); + expect(result.code).toBe(NETWORK_ERROR); + expect(result.peerResponded).toBe(true); + }); + + it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => { + const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code }))); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(false); + }); + + it("reports the peer as silent on a connection timeout", async () => { + const provider = new PortProvider(createTimeoutNet()); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(false); + }); + + it("reports the peer as having answered when the port accepts the connection", async () => { + const provider = new PortProvider(createSuccessNet()); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(true); + }); }); diff --git a/server/test/unit/providers/network/utils.test.ts b/server/test/unit/providers/network/utils.test.ts index a83e14cff..5a0d33972 100644 --- a/server/test/unit/providers/network/utils.test.ts +++ b/server/test/unit/providers/network/utils.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "@jest/globals"; -import { timeRequest, isStatusUp } from "../../../../src/service/network/utils.ts"; +import { timeRequest, isStatusUp, peerAnsweredFromError } from "../../../../src/service/network/utils.ts"; import { NETWORK_ERROR } from "../../../../src/types/network.ts"; describe("network utils", () => { @@ -39,6 +39,40 @@ describe("network utils", () => { }); }); + // The egress check blames the instance only when nothing answered, so this helper's `true` must never be a + // guess: every code in it requires the peer's own stack to have replied. + describe("peerAnsweredFromError", () => { + it.each([ + ["ECONNREFUSED", "the host refused the connection"], + ["ECONNRESET", "the host reset an established connection"], + ["EPIPE", "the host closed a connection we were writing to"], + ["EPROTO", "a protocol error on a connection that was made"], + ["CERT_HAS_EXPIRED", "the host presented a certificate"], + ["DEPTH_ZERO_SELF_SIGNED_CERT", "the host presented a certificate"], + ["SELF_SIGNED_CERT_IN_CHAIN", "the host presented a certificate"], + ["UNABLE_TO_GET_ISSUER_CERT_LOCALLY", "the host presented a certificate"], + ["UNABLE_TO_VERIFY_LEAF_SIGNATURE", "the host presented a certificate"], + ["ERR_TLS_CERT_ALTNAME_INVALID", "the host presented a certificate"], + ["CERT_NOT_YET_VALID", "the host presented a certificate"], + ["ERR_SSL_WRONG_VERSION_NUMBER", "the host spoke back, badly"], + ])("reports the peer as having answered for %s", (code) => { + expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(true); + }); + + it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ECONNABORTED"], ["ENETDOWN"]])( + "reports the peer as silent for %s", + (code) => { + expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(false); + } + ); + + it("reports the peer as silent when the error carries no code", () => { + expect(peerAnsweredFromError(new Error("Connection timeout"))).toBe(false); + expect(peerAnsweredFromError(undefined)).toBe(false); + expect(peerAnsweredFromError("not an error")).toBe(false); + }); + }); + describe("isStatusUp", () => { it("returns true for 2xx status codes", () => { expect(isStatusUp(200)).toBe(true); diff --git a/server/test/unit/providers/network/webSocketProvider.test.ts b/server/test/unit/providers/network/webSocketProvider.test.ts index b765b2f74..13dc5190b 100644 --- a/server/test/unit/providers/network/webSocketProvider.test.ts +++ b/server/test/unit/providers/network/webSocketProvider.test.ts @@ -191,4 +191,42 @@ describe("WebSocketProvider", () => { ); }); }); + + // ── Peer reachability ──────────────────────────────────────────────────── + // `ws` rejects a non-101 handshake with a plain Error, so only the errno cases can be told apart from the + // instance losing its egress. That leaves a server answering the handshake with an HTTP error indistinct. + + const createErrorWS = (error: Error) => + jest.fn().mockImplementation(() => ({ + on: jest.fn((event: string, cb: Function) => { + if (event === "error") process.nextTick(() => cb(error)); + }), + close: jest.fn(), + })) as any; + + it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => { + const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code }))); + + const result = await provider.handle(makeMonitor()); + + expect(result.status).toBe(false); + expect(result.code).toBe(NETWORK_ERROR); + expect(result.peerResponded).toBe(true); + }); + + it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EHOSTUNREACH"]])("reports the peer as silent for %s", async (code) => { + const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code }))); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(false); + }); + + it("reports the peer as having answered when the socket opens", async () => { + const provider = new WebSocketProvider(createMockWS("open")); + + const result = await provider.handle(makeMonitor()); + + expect(result.peerResponded).toBe(true); + }); });