Report whether the peer answered from every provider that can tell
An egress spell masked a service on the surviving network going down: with
`peerResponded` set only by the DNS and gRPC providers, every other failure fell
back to the HTTP-status heuristic, which answers "nothing replied" for anything
that is not speaking HTTP. A refused connection on the local network was read as
the instance's own loss of egress, so the check was recorded as degraded and the
monitor stayed up.
`peerAnsweredFromError` holds the errno codes that require the target's own stack
to have replied — a refusal, a reset, a TLS handshake that got as far as a
certificate. Http, Port, WebSocket and Docker use it; Http also treats any status
code as an answer, and Docker any status from the daemon. Hardware and PageSpeed
go through Http and inherit it.
`true` is the claim that costs something, since it blames the target, so it is
only set on evidence. Two cases stay conservative and are commented as such: a
ping that draws no echo reply, and a gamedig query that fails, neither of which
can be told apart from the instance losing its egress. Nor can a proxied HTTP
request that fails with no response, where the refusal may be the proxy's.
🔧
This commit is contained in:
1 parent
13799af721
commit
b55560e2f1
13 files changed
+318
-7
No files matched your search
@@ -23,7 +23,7 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import Dockerode from "dockerode";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
|
||||
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
@@ -315,12 +315,19 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return rawLogs.filter((log) => log !== null);
|
||||
};
|
||||
|
||||
private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse<DockerStatusPayload> => ({
|
||||
private failed = (
|
||||
monitor: Monitor,
|
||||
code: number,
|
||||
message: string,
|
||||
responseTime: number,
|
||||
peerResponded = false
|
||||
): MonitorStatusResponse<DockerStatusPayload> => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code,
|
||||
peerResponded,
|
||||
message,
|
||||
responseTime,
|
||||
payload: null,
|
||||
@@ -337,13 +344,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
if (error) {
|
||||
let message = "Docker host is unreachable";
|
||||
let code = NETWORK_ERROR;
|
||||
// A status code from the daemon is the daemon answering; otherwise only a refusal or a reset
|
||||
// proves we reached the host, and anything else could be the instance's own loss of egress.
|
||||
let peerResponded = peerAnsweredFromError(error);
|
||||
if (this.isDockerError(error)) {
|
||||
code = error.statusCode ?? NETWORK_ERROR;
|
||||
message = error.json?.message ?? error.reason ?? error.message;
|
||||
peerResponded = peerResponded || error.statusCode !== undefined;
|
||||
} else if (error instanceof Error) {
|
||||
message = error.message;
|
||||
}
|
||||
return this.failed(monitor, code, message, responseTime);
|
||||
return this.failed(monitor, code, message, responseTime, peerResponded);
|
||||
}
|
||||
|
||||
const summaries = await docker.listContainers({ all: true });
|
||||
@@ -362,6 +373,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "Docker host is reachable",
|
||||
responseTime,
|
||||
payload: { containers, summary, logs },
|
||||
|
||||
Reference in new issue
Block a user