Report whether the peer answered from every provider that can tell

An egress spell masked a service on the surviving network going down: with
`peerResponded` set only by the DNS and gRPC providers, every other failure fell
back to the HTTP-status heuristic, which answers "nothing replied" for anything
that is not speaking HTTP. A refused connection on the local network was read as
the instance's own loss of egress, so the check was recorded as degraded and the
monitor stayed up.

`peerAnsweredFromError` holds the errno codes that require the target's own stack
to have replied — a refusal, a reset, a TLS handshake that got as far as a
certificate. Http, Port, WebSocket and Docker use it; Http also treats any status
code as an answer, and Docker any status from the daemon. Hardware and PageSpeed
go through Http and inherit it.

`true` is the claim that costs something, since it blames the target, so it is
only set on evidence. Two cases stay conservative and are commented as such: a
ping that draws no echo reply, and a gamedig query that fails, neither of which
can be told apart from the instance losing its egress. Nor can a proxied HTTP
request that fails with no response, where the refusal may be the proxy's.

🔧
This commit is contained in:
Craig Lawson committed 2026-09-19 14:43:11 +01:00
1 parent 13799af721
commit b55560e2f1
13 files changed
+318 -7

No files matched your search

+15 -3
View File
@@ -23,7 +23,7 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { ILogger } from "@/utils/logger.js";
import { AppError } from "@/utils/AppError.js";
import Dockerode from "dockerode";
import { timeRequest } from "@/service/network/utils.js";
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
@@ -315,12 +315,19 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
return rawLogs.filter((log) => log !== null);
};
private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse<DockerStatusPayload> => ({
private failed = (
monitor: Monitor,
code: number,
message: string,
responseTime: number,
peerResponded = false
): MonitorStatusResponse<DockerStatusPayload> => ({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
status: false,
code,
peerResponded,
message,
responseTime,
payload: null,
@@ -337,13 +344,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
if (error) {
let message = "Docker host is unreachable";
let code = NETWORK_ERROR;
// A status code from the daemon is the daemon answering; otherwise only a refusal or a reset
// proves we reached the host, and anything else could be the instance's own loss of egress.
let peerResponded = peerAnsweredFromError(error);
if (this.isDockerError(error)) {
code = error.statusCode ?? NETWORK_ERROR;
message = error.json?.message ?? error.reason ?? error.message;
peerResponded = peerResponded || error.statusCode !== undefined;
} else if (error instanceof Error) {
message = error.message;
}
return this.failed(monitor, code, message, responseTime);
return this.failed(monitor, code, message, responseTime, peerResponded);
}
const summaries = await docker.listContainers({ all: true });
@@ -362,6 +373,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
type: monitor.type,
status: true,
code: 200,
peerResponded: true,
message: "Docker host is reachable",
responseTime,
payload: { containers, summary, logs },
@@ -43,6 +43,9 @@ export class GameProvider implements IStatusProvider<GameStatusPayload> {
});
if (!state) {
// gamedig reports a failed query the same way whichever leg failed, so `peerResponded` is left
// unset and a failure is put to the egress probe. A game server that stops answering during a
// spell is therefore recorded as degraded rather than down.
return {
monitorId: id,
teamId: teamId,
+6 -1
View File
@@ -6,7 +6,7 @@ import { MonitorStatusResponse } from "@/types/network.js";
import { Agent as HttpsAgent } from "https";
import { Agent as HttpAgent } from "http";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { isStatusUp } from "@/service/network/utils.js";
import { isStatusUp, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import CacheableLookup from "cacheable-lookup";
import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
@@ -107,6 +107,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: statusUp,
code: statusCode,
peerResponded: true,
message,
responseTime,
timings,
@@ -123,6 +124,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: false,
code: statusCode,
peerResponded: true,
message: "Response is not JSON",
responseTime,
timings,
@@ -149,6 +151,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: statusUp && matchResult.ok,
code: statusCode,
peerResponded: true,
message: matchResult.ok ? message : matchResult.message,
responseTime,
timings,
@@ -172,6 +175,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: false,
code: statusCode ?? NETWORK_ERROR,
peerResponded: statusCode !== undefined || (!ctx?.proxyUrl && peerAnsweredFromError(error)),
message: message,
responseTime,
timings: error.timings,
@@ -196,6 +200,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: false,
code: NETWORK_ERROR,
peerResponded: !ctx?.proxyUrl && peerAnsweredFromError(error),
message: error instanceof Error ? error.message : String(error),
responseTime: 0,
payload: null as T,
@@ -58,6 +58,9 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
type: monitor.type,
status: response.alive ?? false,
code: response.alive ? 200 : 5000,
// An echo reply is the host answering. Its absence is silence, which is indistinguishable
// from the instance having lost its egress, so the egress probe decides that case.
peerResponded: response.alive ?? false,
message: response.alive ? "Success" : "Ping failed",
responseTime,
payload: response,
+5 -1
View File
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
import { PortStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { AppError } from "@/utils/AppError.js";
import { timeRequest } from "@/service/network/utils.js";
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import * as net from "net";
type NetType = typeof net;
@@ -59,6 +59,9 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
type: monitor.type,
status: false,
code: NETWORK_ERROR,
// A refused or reset connection came from the host itself; a timeout or an unreachable
// network did not, and could equally be the instance's own loss of egress.
peerResponded: peerAnsweredFromError(error),
message: errorMessage,
responseTime: responseTime,
timings: undefined,
@@ -72,6 +75,7 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
type: monitor.type,
status: true,
code: 200,
peerResponded: true,
message: "Port check successful",
responseTime: responseTime,
timings: undefined,
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
import { WebSocketStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { AppError } from "@/utils/AppError.js";
import { timeRequest } from "@/service/network/utils.js";
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import type WebSocket from "ws";
@@ -63,6 +63,9 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
type: monitor.type,
status: false,
code: NETWORK_ERROR,
// `ws` rejects a non-101 handshake with a plain Error carrying only a message, so only the
// errno cases — a refusal or a reset — can be told apart from the instance losing egress.
peerResponded: peerAnsweredFromError(error),
message: errorMessage,
responseTime: responseTime,
timings: undefined,
@@ -76,6 +79,7 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
type: monitor.type,
status: true,
code: 200,
peerResponded: true,
message: "WebSocket check successful",
responseTime: responseTime,
timings: undefined,
+24
View File
@@ -16,3 +16,27 @@ export const isStatusUp = (statusCode: number | undefined, customUpCodes: HttpSt
if (statusCode === undefined) return false;
return (statusCode >= 200 && statusCode < 300) || customUpCodes.includes(statusCode);
};
// Errno codes that can only arise once the target's own stack has answered: it refused the connection, reset
// an established one, or got far enough into a TLS handshake to present a certificate. Every other failure —
// a timeout, an unreachable network, a name that would not resolve — looks the same whether the target is
// down or the instance has lost its egress, so those are left for the egress probe to decide.
const PEER_ANSWERED_ERROR_CODES: ReadonlySet<string> = new Set([
"ECONNREFUSED",
"ECONNRESET",
"EPIPE",
"EPROTO",
"CERT_HAS_EXPIRED",
"CERT_NOT_YET_VALID",
"DEPTH_ZERO_SELF_SIGNED_CERT",
"SELF_SIGNED_CERT_IN_CHAIN",
"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
"ERR_TLS_CERT_ALTNAME_INVALID",
"ERR_SSL_WRONG_VERSION_NUMBER",
]);
export const peerAnsweredFromError = (error: unknown): boolean => {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
return typeof code === "string" && PEER_ANSWERED_ERROR_CODES.has(code);
};