Report whether the peer answered from every provider that can tell
An egress spell masked a service on the surviving network going down: with
`peerResponded` set only by the DNS and gRPC providers, every other failure fell
back to the HTTP-status heuristic, which answers "nothing replied" for anything
that is not speaking HTTP. A refused connection on the local network was read as
the instance's own loss of egress, so the check was recorded as degraded and the
monitor stayed up.
`peerAnsweredFromError` holds the errno codes that require the target's own stack
to have replied — a refusal, a reset, a TLS handshake that got as far as a
certificate. Http, Port, WebSocket and Docker use it; Http also treats any status
code as an answer, and Docker any status from the daemon. Hardware and PageSpeed
go through Http and inherit it.
`true` is the claim that costs something, since it blames the target, so it is
only set on evidence. Two cases stay conservative and are commented as such: a
ping that draws no echo reply, and a gamedig query that fails, neither of which
can be told apart from the instance losing its egress. Nor can a proxied HTTP
request that fails with no response, where the refusal may be the proxy's.
🔧
This commit is contained in:
1 parent
13799af721
commit
b55560e2f1
13 files changed
+318
-7
No files matched your search
@@ -23,7 +23,7 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import Dockerode from "dockerode";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
|
||||
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
@@ -315,12 +315,19 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return rawLogs.filter((log) => log !== null);
|
||||
};
|
||||
|
||||
private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse<DockerStatusPayload> => ({
|
||||
private failed = (
|
||||
monitor: Monitor,
|
||||
code: number,
|
||||
message: string,
|
||||
responseTime: number,
|
||||
peerResponded = false
|
||||
): MonitorStatusResponse<DockerStatusPayload> => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code,
|
||||
peerResponded,
|
||||
message,
|
||||
responseTime,
|
||||
payload: null,
|
||||
@@ -337,13 +344,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
if (error) {
|
||||
let message = "Docker host is unreachable";
|
||||
let code = NETWORK_ERROR;
|
||||
// A status code from the daemon is the daemon answering; otherwise only a refusal or a reset
|
||||
// proves we reached the host, and anything else could be the instance's own loss of egress.
|
||||
let peerResponded = peerAnsweredFromError(error);
|
||||
if (this.isDockerError(error)) {
|
||||
code = error.statusCode ?? NETWORK_ERROR;
|
||||
message = error.json?.message ?? error.reason ?? error.message;
|
||||
peerResponded = peerResponded || error.statusCode !== undefined;
|
||||
} else if (error instanceof Error) {
|
||||
message = error.message;
|
||||
}
|
||||
return this.failed(monitor, code, message, responseTime);
|
||||
return this.failed(monitor, code, message, responseTime, peerResponded);
|
||||
}
|
||||
|
||||
const summaries = await docker.listContainers({ all: true });
|
||||
@@ -362,6 +373,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "Docker host is reachable",
|
||||
responseTime,
|
||||
payload: { containers, summary, logs },
|
||||
|
||||
@@ -43,6 +43,9 @@ export class GameProvider implements IStatusProvider<GameStatusPayload> {
|
||||
});
|
||||
|
||||
if (!state) {
|
||||
// gamedig reports a failed query the same way whichever leg failed, so `peerResponded` is left
|
||||
// unset and a failure is put to the egress probe. A game server that stops answering during a
|
||||
// spell is therefore recorded as degraded rather than down.
|
||||
return {
|
||||
monitorId: id,
|
||||
teamId: teamId,
|
||||
|
||||
@@ -6,7 +6,7 @@ import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { Agent as HttpsAgent } from "https";
|
||||
import { Agent as HttpAgent } from "http";
|
||||
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { isStatusUp } from "@/service/network/utils.js";
|
||||
import { isStatusUp, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import CacheableLookup from "cacheable-lookup";
|
||||
import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
|
||||
@@ -107,6 +107,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: statusUp,
|
||||
code: statusCode,
|
||||
peerResponded: true,
|
||||
message,
|
||||
responseTime,
|
||||
timings,
|
||||
@@ -123,6 +124,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: statusCode,
|
||||
peerResponded: true,
|
||||
message: "Response is not JSON",
|
||||
responseTime,
|
||||
timings,
|
||||
@@ -149,6 +151,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: statusUp && matchResult.ok,
|
||||
code: statusCode,
|
||||
peerResponded: true,
|
||||
message: matchResult.ok ? message : matchResult.message,
|
||||
responseTime,
|
||||
timings,
|
||||
@@ -172,6 +175,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: statusCode ?? NETWORK_ERROR,
|
||||
peerResponded: statusCode !== undefined || (!ctx?.proxyUrl && peerAnsweredFromError(error)),
|
||||
message: message,
|
||||
responseTime,
|
||||
timings: error.timings,
|
||||
@@ -196,6 +200,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: NETWORK_ERROR,
|
||||
peerResponded: !ctx?.proxyUrl && peerAnsweredFromError(error),
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
responseTime: 0,
|
||||
payload: null as T,
|
||||
|
||||
@@ -58,6 +58,9 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: response.alive ?? false,
|
||||
code: response.alive ? 200 : 5000,
|
||||
// An echo reply is the host answering. Its absence is silence, which is indistinguishable
|
||||
// from the instance having lost its egress, so the egress probe decides that case.
|
||||
peerResponded: response.alive ?? false,
|
||||
message: response.alive ? "Success" : "Ping failed",
|
||||
responseTime,
|
||||
payload: response,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import { PortStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import * as net from "net";
|
||||
type NetType = typeof net;
|
||||
@@ -59,6 +59,9 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: NETWORK_ERROR,
|
||||
// A refused or reset connection came from the host itself; a timeout or an unreachable
|
||||
// network did not, and could equally be the instance's own loss of egress.
|
||||
peerResponded: peerAnsweredFromError(error),
|
||||
message: errorMessage,
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
@@ -72,6 +75,7 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "Port check successful",
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import { WebSocketStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import type WebSocket from "ws";
|
||||
|
||||
@@ -63,6 +63,9 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: NETWORK_ERROR,
|
||||
// `ws` rejects a non-101 handshake with a plain Error carrying only a message, so only the
|
||||
// errno cases — a refusal or a reset — can be told apart from the instance losing egress.
|
||||
peerResponded: peerAnsweredFromError(error),
|
||||
message: errorMessage,
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
@@ -76,6 +79,7 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "WebSocket check successful",
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
|
||||
@@ -16,3 +16,27 @@ export const isStatusUp = (statusCode: number | undefined, customUpCodes: HttpSt
|
||||
if (statusCode === undefined) return false;
|
||||
return (statusCode >= 200 && statusCode < 300) || customUpCodes.includes(statusCode);
|
||||
};
|
||||
|
||||
// Errno codes that can only arise once the target's own stack has answered: it refused the connection, reset
|
||||
// an established one, or got far enough into a TLS handshake to present a certificate. Every other failure —
|
||||
// a timeout, an unreachable network, a name that would not resolve — looks the same whether the target is
|
||||
// down or the instance has lost its egress, so those are left for the egress probe to decide.
|
||||
const PEER_ANSWERED_ERROR_CODES: ReadonlySet<string> = new Set([
|
||||
"ECONNREFUSED",
|
||||
"ECONNRESET",
|
||||
"EPIPE",
|
||||
"EPROTO",
|
||||
"CERT_HAS_EXPIRED",
|
||||
"CERT_NOT_YET_VALID",
|
||||
"DEPTH_ZERO_SELF_SIGNED_CERT",
|
||||
"SELF_SIGNED_CERT_IN_CHAIN",
|
||||
"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
|
||||
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
|
||||
"ERR_TLS_CERT_ALTNAME_INVALID",
|
||||
"ERR_SSL_WRONG_VERSION_NUMBER",
|
||||
]);
|
||||
|
||||
export const peerAnsweredFromError = (error: unknown): boolean => {
|
||||
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
||||
return typeof code === "string" && PEER_ANSWERED_ERROR_CODES.has(code);
|
||||
};
|
||||
Reference in new issue
Block a user