Merge pull request #3963 from web-engineer/feat/egress-2-check-persistence

Egress self-check (2/4): flag checks taken during degraded egress (#3937)
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-09-20 16:31:06 -07:00
commit 253642dde9
42 files changed
+1319 -74

No files matched your search

-5
View File
@@ -13307,11 +13307,6 @@
},
"maxItems": 10
},
"egressPollIntervalSeconds": {
"type": "integer",
"minimum": 5,
"maximum": 600
},
"egressNotifications": {
"type": "array",
"items": {
@@ -1,5 +1,5 @@
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
import { MAX_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
import { parseEgressTarget } from "@/utils/egressTarget.js";
import { z } from "zod";
@@ -56,7 +56,6 @@ export const updateAppSettingsBodyValidation = z
egressCheckEnabled: z.boolean().optional(),
egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(),
})
.strip()
+2 -1
View File
@@ -129,9 +129,10 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
proxyResolver,
bufferService,
dockerLogsService,
egressService,
logger
);
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy, logger);
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
// ***********************
@@ -1,6 +1,6 @@
import { Schema, model, type Types } from "mongoose";
import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
_id: Types.ObjectId;
@@ -45,7 +45,6 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
globalProxyId: { type: String, default: null },
egressCheckEnabled: { type: Boolean, default: false },
egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
},
{ timestamps: true }
@@ -3,7 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
class MongoSettingsRepository implements ISettingsRepository {
private toEntity = (doc: AppSettingsDocument): Settings => {
@@ -33,7 +33,6 @@ class MongoSettingsRepository implements ISettingsRepository {
globalProxyId: doc.globalProxyId ?? undefined,
egressCheckEnabled: doc.egressCheckEnabled ?? false,
egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
egressNotifications: doc.egressNotifications ?? [],
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
@@ -54,7 +54,6 @@ export interface Settings {
// Egress self-check (see domain/egress)
egressCheckEnabled: boolean;
egressCheckTargets: string[];
egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
createdAt: string;
updatedAt: string;
@@ -1,5 +1,6 @@
import CheckModel from "@/domain/checks/check.model.js";
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
type DateRange = { start: Date; end: Date };
@@ -9,6 +10,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
@@ -16,6 +18,7 @@ export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Pr
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
status: true,
});
return { totalChecks: count };
@@ -28,6 +31,7 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{
@@ -48,6 +52,7 @@ export const getDockerLatestCheck = async (monitorId: string) =>
CheckModel.findOne({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
...EXCLUDE_DEGRADED_EGRESS_MATCH,
})
.sort({ createdAt: -1 })
.select("containers containerSummary createdAt")
@@ -65,6 +70,7 @@ export const getDockerContainerStats = async (
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
"containers.name": containerName,
},
},
@@ -88,6 +94,7 @@ export const getDockerContainerLatestCheck = async (monitorId: string, container
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
"containers.name": containerName,
...EXCLUDE_DEGRADED_EGRESS_MATCH,
})
.sort({ createdAt: -1 })
.select("containers createdAt")
@@ -1,6 +1,7 @@
import { CheckModel } from "@/domain/checks/check.model.js";
import mongoose from "mongoose";
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
type DateRange = { start: Date; end: Date };
type HardwareUpChecks = { totalChecks: number };
@@ -70,6 +71,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
};
@@ -78,6 +80,7 @@ export const getHardwareUpChecks = async (monitorId: string, dates: DateRange):
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
status: true,
});
return { totalChecks: count };
@@ -95,6 +98,7 @@ export const getHardwareStats = async (monitorId: string, dates: DateRange, date
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{ $sort: { createdAt: 1 } },
+6
View File
@@ -1,5 +1,6 @@
import { Schema, model, Types } from "mongoose";
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
import { EgressStatuses } from "@/domain/egress/egress.type.js";
import type {
Check,
CheckAudits,
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
type: containerSummarySchema,
default: undefined,
},
egressStatus: {
type: String,
enum: EgressStatuses,
default: undefined,
},
},
{
timestamps: true,
+7
View File
@@ -0,0 +1,7 @@
import type { EgressStatus } from "@/domain/egress/egress.type.js";
// A check recorded while the instance's own egress was down says nothing about the target, so it is left out of
// every aggregate: uptime percentages, up/down counts and the response-time series alike. A degraded episode
// therefore reads as a gap in monitoring rather than as downtime or as a dip in response time.
// Spread this into the $match or filter of any query that reports statistics. Check listings keep degraded checks.
export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
@@ -16,6 +16,7 @@ import type {
} from "@/domain/checks/check.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
import { ILogger } from "@/utils/logger.js";
@@ -179,6 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
audits: mapAudits(doc.audits),
containers: doc.containers,
containerSummary: doc.containerSummary,
egressStatus: doc.egressStatus,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
};
@@ -348,6 +350,7 @@ class MongoChecksRepository implements IChecksRepository {
const baseMatch = {
"metadata.teamId": new mongoose.Types.ObjectId(teamId),
createdAt: { $gte: getDateForRange(dateRange) },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [totalResult, downResult] = await Promise.all([
@@ -393,6 +396,7 @@ class MongoChecksRepository implements IChecksRepository {
$match: {
"metadata.monitorId": { $in: objectIds },
createdAt: { $gte: windowStart },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{
@@ -473,6 +477,7 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [result] = await CheckModel.aggregate([
{ $match: matchStage },
@@ -638,6 +643,7 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [result] = await CheckModel.aggregate([
+5
View File
@@ -1,4 +1,5 @@
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { Response } from "got";
@@ -116,6 +117,10 @@ export interface Check {
seo?: number;
performance?: number;
audits?: CheckAudits;
// Set on failing checks while the egress check is enabled; absent otherwise.
// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
// "ok": a reliability target was reachable.
egressStatus?: EgressStatus;
createdAt: string;
updatedAt: string;
}
+6 -11
View File
@@ -7,7 +7,7 @@ import type { INetworkService } from "@/service/networkService.js";
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
import type { ILogger } from "@/utils/logger.js";
import {
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
EGRESS_RECOVERY_POLL_SECONDS,
DEFAULT_EGRESS_TARGETS,
type EgressAssessment,
type EgressProbeResult,
@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
const PROBE_MONITOR_ID = "egress-probe";
@@ -136,11 +136,6 @@ export class EgressService implements IEgressService {
return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
};
private toIntervalMs = (seconds: number | undefined): number => {
const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
};
// ****************************
// Event-triggered assessment
// ****************************
@@ -181,7 +176,7 @@ export class EgressService implements IEgressService {
const state = await this.egressStateRepository.findSingleton();
if (state.status === "degraded") {
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
await this.scheduleRecoveryCheck(false);
return "degraded";
}
@@ -195,7 +190,7 @@ export class EgressService implements IEgressService {
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
await this.scheduleRecoveryCheck(true);
const degraded = await this.egressStateRepository.markDegraded(results, now);
this.logger.warn({
message: degraded
@@ -224,8 +219,8 @@ export class EgressService implements IEgressService {
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
// so re-arming while degraded never pushes a pending run back.
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
private scheduleRecoveryCheck = async (reschedule: boolean) => {
const intervalMs = EGRESS_RECOVERY_POLL_SECONDS * 1000;
const seed: JobSeed = {
id: RECOVERY_JOB_ID,
type: RECOVERY_JOB_TYPE,
+5 -3
View File
@@ -9,11 +9,13 @@ export type EgressAssessment = EgressStatus | "disabled";
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
export const MAX_EGRESS_TARGETS = 10;
// While the instance is marked degraded, every failing check is flagged from stored state without probing, so
// this interval is the window in which a recovered instance still discards its checks. It is deliberately fixed
// and not configurable: the only effect of raising it is a longer blind window.
export const EGRESS_RECOVERY_POLL_SECONDS = 5;
// Instance-global singleton persisted in MongoDB so that every worker process and the API
// process share one view of the state (the queue worker may run separately from the API).
export interface EgressState {
@@ -50,6 +50,31 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
// however far away the agent is. `unknown` is excluded because no request is made for it.
export const EgressAttributableTypes = [
"http",
"ping",
"pagespeed",
"hardware",
"docker",
"port",
"game",
"grpc",
"websocket",
"dns",
] as const satisfies readonly MonitorType[];
export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
// A Docker daemon reached over a unix socket is local IPC and cannot fail through egress; one reached over
// TCP or TLS is as remote as any other target. The URL, not the type, settles it.
const isLocalDockerSocket = (monitor: Pick<Monitor, "type" | "url">): boolean =>
monitor.type === "docker" && (monitor.url?.trim().startsWith("unix://") ?? false);
export const isEgressAttributable = (monitor: Pick<Monitor, "type" | "url">): boolean =>
EgressAttributableTypes.some((t) => t === monitor.type) && !isLocalDockerSocket(monitor);
export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
export type MonitorStatus = (typeof MonitorStatuses)[number];
+13
View File
@@ -1,5 +1,6 @@
import { DNSStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { IStatusProvider } from "@/service/network/IStatusProvider.js";
import dns from "dns";
import type { Resolver } from "dns/promises";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { AppError } from "@/utils/AppError.js";
@@ -8,6 +9,16 @@ import { NETWORK_ERROR } from "@/types/network.js";
const SERVICE_NAME = "DNSProvider";
// c-ares codes where the DNS server replied: the name does not exist, holds no such record, or the server
// declined or answered badly. Anything else (timeout, connection refused, a local resolver fault) means we
// never got an answer, so the failure could be our own loss of egress rather than the target's.
const ANSWERED_DNS_CODES: ReadonlySet<string> = new Set([dns.NODATA, dns.FORMERR, dns.SERVFAIL, dns.NOTFOUND, dns.NOTIMP, dns.REFUSED, dns.BADRESP]);
const dnsServerAnswered = (error: unknown): boolean => {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
return typeof code === "string" && ANSWERED_DNS_CODES.has(code);
};
export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
readonly type = "dns";
constructor(private createResolver: () => Resolver) {}
@@ -63,6 +74,7 @@ export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
type: monitor.type,
status: false,
code: NETWORK_ERROR,
peerResponded: dnsServerAnswered(error),
message: error instanceof Error ? error.message : String(error),
responseTime,
payload: {
@@ -81,6 +93,7 @@ export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
type: monitor.type,
status: true,
code: 200,
peerResponded: true,
message: "Success",
responseTime,
payload: {
+15 -3
View File
@@ -23,7 +23,7 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { ILogger } from "@/utils/logger.js";
import { AppError } from "@/utils/AppError.js";
import Dockerode from "dockerode";
import { timeRequest } from "@/service/network/utils.js";
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
@@ -315,12 +315,19 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
return rawLogs.filter((log) => log !== null);
};
private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse<DockerStatusPayload> => ({
private failed = (
monitor: Monitor,
code: number,
message: string,
responseTime: number,
peerResponded = false
): MonitorStatusResponse<DockerStatusPayload> => ({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
status: false,
code,
peerResponded,
message,
responseTime,
payload: null,
@@ -337,13 +344,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
if (error) {
let message = "Docker host is unreachable";
let code = NETWORK_ERROR;
// A status code from the daemon is the daemon answering; otherwise only a refusal or a reset
// proves we reached the host, and anything else could be the instance's own loss of egress.
let peerResponded = peerAnsweredFromError(error);
if (this.isDockerError(error)) {
code = error.statusCode ?? NETWORK_ERROR;
message = error.json?.message ?? error.reason ?? error.message;
peerResponded = peerResponded || error.statusCode !== undefined;
} else if (error instanceof Error) {
message = error.message;
}
return this.failed(monitor, code, message, responseTime);
return this.failed(monitor, code, message, responseTime, peerResponded);
}
const summaries = await docker.listContainers({ all: true });
@@ -362,6 +373,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
type: monitor.type,
status: true,
code: 200,
peerResponded: true,
message: "Docker host is reachable",
responseTime,
payload: { containers, summary, logs },
@@ -43,6 +43,9 @@ export class GameProvider implements IStatusProvider<GameStatusPayload> {
});
if (!state) {
// gamedig reports a failed query the same way whichever leg failed, so `peerResponded` is left
// unset and a failure is put to the egress probe. A game server that stops answering during a
// spell is therefore recorded as degraded rather than down.
return {
monitorId: id,
teamId: teamId,
@@ -25,6 +25,17 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
return type === "grpc";
}
// gRPC codes that mean the call never reached a server. Everything else in 0-16 is the server's own answer,
// including NOT_SERVING, which shares NETWORK_ERROR with a connection failure and so cannot be read from `code`.
// A call that fails with no gRPC code at all never got as far as the protocol.
private peerAnswered(code: number | undefined): boolean {
const UNREACHABLE_GRPC_CODES: ReadonlySet<number> = new Set([
4, // DEADLINE_EXCEEDED
14, // UNAVAILABLE
]);
return typeof code === "number" && !UNREACHABLE_GRPC_CODES.has(code);
}
private getGrpcStatusName(code: number): string {
const statusNames: Record<number, string> = {
0: "OK",
@@ -148,6 +159,7 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
type: type,
status: false,
code: grpcError.grpcCode ?? 5000,
peerResponded: this.peerAnswered(grpcError.grpcCode),
message: grpcError.message ?? "gRPC health check failed",
responseTime,
payload: payload ?? null,
@@ -163,6 +175,8 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
type: type,
status: isServing,
code: isServing ? 200 : 5000,
// The server replied, whatever it said, so this failure is never the instance's own egress.
peerResponded: true,
message: isServing ? `gRPC service healthy (${grpcPayload.servingStatus})` : `gRPC service unhealthy (${grpcPayload.servingStatus})`,
responseTime,
payload: grpcPayload,
+6 -1
View File
@@ -6,7 +6,7 @@ import { MonitorStatusResponse } from "@/types/network.js";
import { Agent as HttpsAgent } from "https";
import { Agent as HttpAgent } from "http";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { isStatusUp } from "@/service/network/utils.js";
import { isStatusUp, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import CacheableLookup from "cacheable-lookup";
import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
@@ -107,6 +107,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: statusUp,
code: statusCode,
peerResponded: true,
message,
responseTime,
timings,
@@ -123,6 +124,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: false,
code: statusCode,
peerResponded: true,
message: "Response is not JSON",
responseTime,
timings,
@@ -149,6 +151,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: statusUp && matchResult.ok,
code: statusCode,
peerResponded: true,
message: matchResult.ok ? message : matchResult.message,
responseTime,
timings,
@@ -172,6 +175,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: false,
code: statusCode ?? NETWORK_ERROR,
peerResponded: statusCode !== undefined || (!ctx?.proxyUrl && peerAnsweredFromError(error)),
message: message,
responseTime,
timings: error.timings,
@@ -196,6 +200,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
type: monitor.type,
status: false,
code: NETWORK_ERROR,
peerResponded: !ctx?.proxyUrl && peerAnsweredFromError(error),
message: error instanceof Error ? error.message : String(error),
responseTime: 0,
payload: null as T,
@@ -58,6 +58,9 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
type: monitor.type,
status: response.alive ?? false,
code: response.alive ? 200 : 5000,
// An echo reply is the host answering. Its absence is silence, which is indistinguishable
// from the instance having lost its egress, so the egress probe decides that case.
peerResponded: response.alive ?? false,
message: response.alive ? "Success" : "Ping failed",
responseTime,
payload: response,
+5 -1
View File
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
import { PortStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { AppError } from "@/utils/AppError.js";
import { timeRequest } from "@/service/network/utils.js";
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import * as net from "net";
type NetType = typeof net;
@@ -59,6 +59,9 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
type: monitor.type,
status: false,
code: NETWORK_ERROR,
// A refused or reset connection came from the host itself; a timeout or an unreachable
// network did not, and could equally be the instance's own loss of egress.
peerResponded: peerAnsweredFromError(error),
message: errorMessage,
responseTime: responseTime,
timings: undefined,
@@ -72,6 +75,7 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
type: monitor.type,
status: true,
code: 200,
peerResponded: true,
message: "Port check successful",
responseTime: responseTime,
timings: undefined,
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
import { WebSocketStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { AppError } from "@/utils/AppError.js";
import { timeRequest } from "@/service/network/utils.js";
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import type WebSocket from "ws";
@@ -63,6 +63,9 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
type: monitor.type,
status: false,
code: NETWORK_ERROR,
// `ws` rejects a non-101 handshake with a plain Error carrying only a message, so only the
// errno cases — a refusal or a reset — can be told apart from the instance losing egress.
peerResponded: peerAnsweredFromError(error),
message: errorMessage,
responseTime: responseTime,
timings: undefined,
@@ -76,6 +79,7 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
type: monitor.type,
status: true,
code: 200,
peerResponded: true,
message: "WebSocket check successful",
responseTime: responseTime,
timings: undefined,
+24
View File
@@ -16,3 +16,27 @@ export const isStatusUp = (statusCode: number | undefined, customUpCodes: HttpSt
if (statusCode === undefined) return false;
return (statusCode >= 200 && statusCode < 300) || customUpCodes.includes(statusCode);
};
// Errno codes that can only arise once the target's own stack has answered: it refused the connection, reset
// an established one, or got far enough into a TLS handshake to present a certificate. Every other failure —
// a timeout, an unreachable network, a name that would not resolve — looks the same whether the target is
// down or the instance has lost its egress, so those are left for the egress probe to decide.
const PEER_ANSWERED_ERROR_CODES: ReadonlySet<string> = new Set([
"ECONNREFUSED",
"ECONNRESET",
"EPIPE",
"EPROTO",
"CERT_HAS_EXPIRED",
"CERT_NOT_YET_VALID",
"DEPTH_ZERO_SELF_SIGNED_CERT",
"SELF_SIGNED_CERT_IN_CHAIN",
"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
"ERR_TLS_CERT_ALTNAME_INVALID",
"ERR_SSL_WRONG_VERSION_NUMBER",
]);
export const peerAnsweredFromError = (error: unknown): boolean => {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
return typeof code === "string" && PEER_ANSWERED_ERROR_CODES.has(code);
};
+6
View File
@@ -33,6 +33,12 @@ export interface MonitorStatusResponse<
status: boolean;
code: number;
message: string;
// Whether the peer answered, for failures where `code` alone cannot say. `code` carries an HTTP status for
// providers that speak HTTP, but DNS and gRPC report their own codes, and gRPC reuses NETWORK_ERROR for a
// server that replied NOT_SERVING. Providers that can tell the two apart set this; the rest leave it unset
// and the HTTP-status heuristic stands. Consumed by the egress check, which must only blame the instance's
// own connectivity when nothing answered.
peerResponded?: boolean;
responseTime?: number;
payload?: T | string | null;
timings?: GotTimings;
+36 -1
View File
@@ -4,17 +4,52 @@ import { Check } from "@/domain/checks/check.type.js";
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
import { IStatusService } from "@/service/statusService.js";
import { MonitorActionDecision } from "@/worker/worker.helper.js";
import { ILogger } from "@/utils/logger.js";
export interface ICheckEvaluator {
evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
}
const SERVICE_NAME = "CheckEvaluator";
export class CheckEvaluator implements ICheckEvaluator {
constructor(
private statusService: IStatusService,
private monitorStatusPolicy: IMonitorStatusPolicy
private monitorStatusPolicy: IMonitorStatusPolicy,
private logger: ILogger
) {}
// A check taken while the instance had no outbound connectivity says nothing about the target.
// It is left out of the status window, running stats and monitor status entirely, so that no incident
// opens for it and no spurious "resolved"/"up" fires once egress returns.
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
this.logger.debug({
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
service: SERVICE_NAME,
method: "evaluate",
});
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
return {
monitor,
status,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
decision,
};
};
evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
if (check.egressStatus === "degraded") {
return this.skipDegradedEgressCheck(status, check, monitor);
}
// ****************************
// Step 3: Evaluate and return result to reactors
// ****************************
+56 -3
View File
@@ -11,6 +11,9 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { isEgressAttributable } from "@/domain/monitors/monitor.type.js";
export interface ICheckProducer {
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -27,6 +30,7 @@ export class CheckProducer implements ICheckProducer {
private proxyResolver: IProxyResolver,
private bufferService: IBufferService,
private dockerLogsService: IDockerLogsService,
private egressService: IEgressService,
private logger: ILogger
) {}
@@ -42,6 +46,31 @@ export class CheckProducer implements ICheckProducer {
return dockerTlsKey ?? undefined;
}
// Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the
// two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx,
// or a 200 with a content mismatch).
private peerAnswered(status: MonitorStatusResponse): boolean {
return status.peerResponded ?? isHttpStatusCode(status.code);
}
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
}
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
try {
return await this.egressService.assessAfterFailure();
} catch (error: unknown) {
this.logger.warn({
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "assessEgress",
});
return null;
}
}
produce = async (monitor: Monitor) => {
if (!monitor.id) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
@@ -65,6 +94,13 @@ export class CheckProducer implements ICheckProducer {
return null;
}
// Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the
// degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is
// evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage.
if (monitor.status === "maintenance") {
monitor = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" });
}
// Step 1b: Acquire status
const proxyUrl = await this.proxyResolver.resolve(monitor);
const dockerTlsKey = await this.resolveDockerTlsKey(monitor);
@@ -78,7 +114,8 @@ export class CheckProducer implements ICheckProducer {
// Step 2: Record
// ****************************
// Step 2a: Create & record a check, return null if fail
// Step 2a: Create & record a check, return null if fail.
// Built before the egress assessment below so that the probe's duration stays out of `createdAt`.
const check = this.checkService.toCheck(status);
if (!check) {
this.logger.warn({
@@ -89,10 +126,26 @@ export class CheckProducer implements ICheckProducer {
});
return null;
}
// Step 2b: Add to buffer
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
if (egressStatus !== null) {
check.egressStatus = egressStatus;
}
// Step 2c: Add to buffer
this.bufferService.addToBuffer(check);
// Step 2c: Handle docker logs
// Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops
// being evaluated, so the results either side of a spell never end up adjacent and cross the threshold
// on stale data. The length guard keeps this to one write per spell rather than one per degraded check,
// and it runs after the check is buffered so that a failed write cannot cost us the check.
if (egressStatus === "degraded" && monitor.statusWindow?.length) {
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] });
}
// Step 2e: Handle docker logs
if (status.type === "docker") {
const dockerLogs = await this.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
for (const dockerLog of dockerLogs) {
+13 -1
View File
@@ -14,6 +14,7 @@ import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
import type { MonitorStatusResponse } from "../../src/types/network.ts";
import type { Check } from "../../src/domain/checks/check.type.ts";
import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts";
import type { EgressStatus } from "../../src/domain/egress/egress.type.ts";
let checkCounter = 0;
@@ -79,9 +80,11 @@ export interface HeartbeatTestHarness {
bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock };
maintenanceWindowsRepo: { findByMonitorId: jest.Mock };
messageBuilder: { extractThresholdBreaches: jest.Mock };
egressService: { assessAfterFailure: jest.Mock };
heartbeatJob: (monitor: Monitor) => Promise<void>;
setNextResponse: (status: boolean, code: number) => void;
setNextResponseFull: (response: MonitorStatusResponse) => void;
setEgressStatus: (status: EgressStatus | null) => void;
}
export function createHeartbeatTestHarness(): HeartbeatTestHarness {
@@ -129,6 +132,12 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
// Null mirrors the egress check being disabled, so the existing heartbeat suites run unchanged.
let nextEgressStatus: EgressStatus | null = null;
const egressService = { assessAfterFailure: jest.fn().mockImplementation(() => Promise.resolve(nextEgressStatus)) };
const setEgressStatus = (status: EgressStatus | null) => {
nextEgressStatus = status;
};
const checkProducer = new CheckProducer(
monitorsRepo as any,
@@ -138,9 +147,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
proxyResolver as any,
bufferStub as any,
dockerLogsService as any,
egressService as any,
logger
);
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger);
const notificationReactor = new NotificationReactor(notificationsService as any);
const incidentReactor = new IncidentReactor(incidentService as any);
@@ -167,8 +177,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
bufferStub,
maintenanceWindowsRepo,
messageBuilder,
egressService,
heartbeatJob,
setNextResponse,
setNextResponseFull,
setEgressStatus,
};
}
@@ -0,0 +1,212 @@
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
import { CheckModel } from "../../src/domain/checks/check.model.ts";
import type {
Check,
DockerChecksResult,
HardwareChecksResult,
PageSpeedChecksResult,
UptimeChecksResult,
} from "../../src/domain/checks/check.type.ts";
import { NETWORK_ERROR } from "../../src/types/network.ts";
import type { ILogger } from "../../src/utils/logger.ts";
import { createMockLogger } from "../helpers/createMockLogger.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The guarantee under test — a check flagged egressStatus "degraded" is left out of
// every aggregate, so a degraded episode reads as a gap in monitoring, while the check
// still appears in listings — lives in $match stages spread into several aggregation
// pipelines, so it can only be exercised against a live mongod.
let mongod: MongoMemoryServer;
beforeAll(async () => {
mongod = await MongoMemoryServer.create();
await mongoose.connect(mongod.getUri());
await CheckModel.createCollection(); // timeseries collections must exist before insert
}, 120_000);
afterAll(async () => {
await mongoose.disconnect();
await mongod.stop();
});
beforeEach(async () => {
await CheckModel.deleteMany({});
});
// ── Helpers ──────────────────────────────────────────────────────────────────
const MONITOR_ID = new mongoose.Types.ObjectId();
const TEAM_ID = new mongoose.Types.ObjectId();
// One shared createdAt keeps every seeded check in the same bucket of the "day" range.
const BUCKET_TIME = new Date(Date.now() - 10 * 60 * 1000);
const seedCheck = (overrides: Record<string, unknown> = {}) =>
CheckModel.create({
metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "http" },
status: true,
responseTime: 100,
createdAt: BUCKET_TIME,
...overrides,
});
const seedDegradedFailure = (overrides: Record<string, unknown> = {}) =>
seedCheck({ status: false, responseTime: 300, egressStatus: "degraded", ...overrides });
const utcDate = (date: Date) => date.toISOString().slice(0, 10);
const makeCheck = (overrides: Partial<Check> = {}): Check =>
({
metadata: { monitorId: MONITOR_ID.toString(), teamId: TEAM_ID.toString(), type: "http" },
status: false,
responseTime: 300,
statusCode: 500,
message: "Error",
createdAt: BUCKET_TIME.toISOString(),
updatedAt: BUCKET_TIME.toISOString(),
...overrides,
}) as Check;
describe("MongoChecksRepository degraded-egress exclusion", () => {
let repo: MongoChecksRepository;
beforeEach(() => {
repo = new MongoChecksRepository(createMockLogger() as unknown as ILogger);
});
it("round-trips egressStatus through createChecks and the unevaluated-checks read, leaving it absent when unset", async () => {
const created = await repo.createChecks([makeCheck({ egressStatus: "degraded" }), makeCheck({ egressStatus: "ok" }), makeCheck()]);
const pending = created.map((check) => ({ checkId: check.id as string, createdAt: new Date(check.createdAt as string).getTime() }));
const checks = await repo.findUnevaluatedByMonitorId(MONITOR_ID.toString(), pending);
// The evaluator runs from this read, so the flag must survive persistence for the short-circuit to fire.
expect(checks.map((check) => check.egressStatus)).toEqual(["degraded", "ok", undefined]);
expect(checks[2].egressStatus).toBeUndefined();
});
it("rejects a value outside the EgressStatuses tuple", async () => {
await expect(seedCheck({ status: false, egressStatus: "unknown" })).rejects.toThrow(/egressStatus/);
});
it("excludes degraded checks from the uptime percentage and from the response-time series alike", async () => {
await seedCheck(); // up
await seedCheck({ status: false }); // real failure
await seedDegradedFailure();
await seedDegradedFailure();
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
// 1 up of 2 attributable checks. Counting the degraded pair would give 25%.
expect(result.uptimePercentage).toBe(0.5);
// Every series sees the same two checks, so the episode is a gap rather than a dip.
expect(result.avgResponseTime).toBe(100);
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 2, avgResponseTime: 100 });
expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 1 });
expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 });
});
it("leaves a bucket out of the response-time series entirely when every check in it was degraded", async () => {
await seedDegradedFailure();
await seedDegradedFailure();
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
expect(result.groupedChecks).toEqual([]);
expect(result.groupedDownChecks).toEqual([]);
expect(result.uptimePercentage).toBe(0);
});
it("excludes degraded checks from the pagespeed series", async () => {
await seedCheck({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.9 });
await seedDegradedFailure({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.1 });
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "pagespeed" })) as PageSpeedChecksResult;
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 1, performance: 0.9 });
});
it("does not flag a failure that carries egressStatus ok", async () => {
await seedCheck();
await seedCheck({ status: false, egressStatus: "ok" });
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
expect(result.uptimePercentage).toBe(0.5);
});
it("excludes degraded checks from the team summary totals", async () => {
await seedCheck();
await seedCheck({ status: false });
await seedDegradedFailure();
const summary = await repo.findSummaryByTeamId(TEAM_ID.toString(), "day");
expect(summary).toEqual({ totalChecks: 2, downChecks: 1 });
});
it("excludes degraded checks from the daily status buckets, dropping a day that saw nothing else", async () => {
const otherMonitor = new mongoose.Types.ObjectId();
await seedCheck();
await seedCheck({ status: false });
await seedDegradedFailure();
await seedDegradedFailure({ metadata: { monitorId: otherMonitor, teamId: TEAM_ID, type: "http" } });
const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC");
// The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket.
expect(buckets).toEqual([
{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 },
]);
});
it("keeps degraded checks in the paginated listing so they can be shown as such", async () => {
await seedCheck({ status: false });
await seedDegradedFailure();
const { checksCount, checks } = await repo.findByMonitorId(MONITOR_ID.toString(), "desc", "day", 0, 10, undefined, "down");
expect(checksCount).toBe(2);
expect(checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
});
it("keeps degraded checks in the team listing and under the resolve filter", async () => {
await seedCheck({ status: false, statusCode: NETWORK_ERROR });
await seedDegradedFailure({ statusCode: NETWORK_ERROR });
// The Checks page reads this listing when no single monitor is selected, and a degraded check has the
// same shape the resolve filter looks for, so neither may quietly drop it.
const team = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "down");
const resolve = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "resolve");
expect(team.checksCount).toBe(2);
expect(resolve.checksCount).toBe(2);
expect(resolve.checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
});
it("excludes degraded checks from the docker and hardware aggregates", async () => {
const dockerMonitor = new mongoose.Types.ObjectId();
const hardwareMonitor = new mongoose.Types.ObjectId();
const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
const hardwareMeta = { monitorId: hardwareMonitor, teamId: TEAM_ID, type: "hardware" };
// A Capture agent is reached by an outbound HTTP request and a docker daemon can be reached over TCP,
// so both are egress attributable and their checks can carry the flag.
await seedCheck({ metadata: dockerMeta });
await seedDegradedFailure({ metadata: dockerMeta });
await seedCheck({ metadata: hardwareMeta });
await seedDegradedFailure({ metadata: hardwareMeta });
const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
const hardware = (await repo.findByDateRangeAndMonitorId(hardwareMonitor.toString(), "day", { type: "hardware" })) as HardwareChecksResult;
expect(docker.aggregateData.totalChecks).toBe(1);
expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1 });
expect(hardware.aggregateData.totalChecks).toBe(1);
expect(hardware.upChecks.totalChecks).toBe(1);
});
});
@@ -0,0 +1,191 @@
import { describe, expect, it, beforeEach } from "@jest/globals";
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
import { NETWORK_ERROR } from "../../src/types/network.ts";
// Egress is only consulted for transport failures, so these heartbeats fail with the network-error
// sentinel (timeout / connection refused) rather than an HTTP status code.
describe("Heartbeat job: degraded egress", () => {
let h: HeartbeatTestHarness;
beforeEach(() => {
h = createHeartbeatTestHarness();
});
it("records failing checks as degraded without changing monitor status, opening an incident or notifying", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
// Enough failures to cross the 60% threshold if they were counted
h.setEgressStatus("degraded");
h.setNextResponse(false, NETWORK_ERROR);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
// Every check was buffered, flagged, so the repository can exclude it from uptime
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(3);
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
expect(check).toMatchObject({ status: false, egressStatus: "degraded" });
}
// Nothing downstream moved
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
expect(storedMonitor.status).toBe("up");
// The window is cleared once as the spell starts, as entering a maintenance window does.
expect(storedMonitor.statusWindow).toEqual([]);
expect(h.incidentsRepo.getAll()).toHaveLength(0);
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
});
it("does not fire a spurious recovery once egress returns", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
h.setEgressStatus("degraded");
h.setNextResponse(false, NETWORK_ERROR);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
// Egress is back and the target is fine
h.setEgressStatus(null);
h.setNextResponse(true, 200);
await h.heartbeatJob(monitor);
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
expect(h.incidentsRepo.getAll()).toHaveLength(0);
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
});
it("detects down at the normal threshold when the egress check is disabled", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
h.setEgressStatus(null);
h.setNextResponse(false, NETWORK_ERROR);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
expect(check).not.toHaveProperty("egressStatus");
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
});
it("detects down at the normal threshold when egress is confirmed ok", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
h.setEgressStatus("ok");
h.setNextResponse(false, NETWORK_ERROR);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
expect(check).toMatchObject({ status: false, egressStatus: "ok" });
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
});
it("evaluates HTTP error responses normally while egress is degraded", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
// The target answered, so the failure is its own even though the instance is degraded
h.setEgressStatus("degraded");
h.setNextResponse(false, 503);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
expect(check).not.toHaveProperty("egressStatus");
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
expect(h.incidentsRepo.getAll()[0].statusCode).toBe(503);
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
});
it("evaluates a failing docker check normally while egress is degraded", async () => {
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" });
h.monitorsRepo.seed(monitor);
h.setEgressStatus("degraded");
h.setNextResponse(false, NETWORK_ERROR);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
expect(check).not.toHaveProperty("egressStatus");
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
});
it("leaves an open incident open when egress drops during it", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
// The target goes down first and is detected normally
h.setEgressStatus(null);
h.setNextResponse(false, NETWORK_ERROR);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
// Egress then drops. The monitor must stay down rather than be resolved by the silence.
h.setEgressStatus("degraded");
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
expect(h.notificationsService.handleNotifications).toHaveBeenCalledTimes(1);
});
it("starts a fresh status window after a degraded spell", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
// 2 real failures: window [t, t, t, f, f] = 40% < 60%
h.setNextResponse(false, NETWORK_ERROR);
await h.heartbeatJob(monitor);
await h.heartbeatJob(monitor);
h.setEgressStatus("degraded");
await h.heartbeatJob(monitor);
await h.heartbeatJob(monitor);
expect((await h.monitorsRepo.findById("mon-1", "team-1")).statusWindow).toEqual([]);
// A third real failure would have taken the old window to 60%. Against a window that starts again
// at the end of the spell it is one result, and the results either side are never adjacent.
h.setEgressStatus(null);
await h.heartbeatJob(monitor);
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
expect(h.incidentsRepo.getAll()).toHaveLength(0);
// The monitor still goes down, once the rebuilt window holds enough real failures to cross.
for (let i = 0; i < 4; i++) {
await h.heartbeatJob(monitor);
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
});
});
@@ -209,6 +209,49 @@ describe("DNSProvider", () => {
expect(mockResolverInstance.resolve4).toHaveBeenCalledWith("example.com");
});
// ── Peer reachability ────────────────────────────────────────────────────
// The egress check must only blame the instance's own connectivity when nothing answered. Every DNS failure
// carries NETWORK_ERROR, so `peerResponded` is the only thing separating an authoritative answer from silence.
it.each([
["ENOTFOUND", "NXDOMAIN: the server answered that the name does not exist"],
["ENODATA", "the server answered with no record of that type"],
["ESERVFAIL", "the server answered with a failure"],
["EREFUSED", "the server declined to answer"],
["EFORMERR", "the server rejected the query as malformed"],
["ENOTIMP", "the server does not implement the query"],
["EBADRESP", "the server replied, badly"],
])("reports the peer as having answered for %s", async (code) => {
const error = Object.assign(new Error("dns failure"), { code });
mockResolverInstance.resolve4.mockRejectedValueOnce(error);
const provider = new DNSProvider(createResolver);
const result = await provider.handle(makeMonitor());
expect(result.status).toBe(false);
expect(result.code).toBe(NETWORK_ERROR);
expect(result.peerResponded).toBe(true);
});
it.each([["ETIMEOUT"], ["ECONNREFUSED"], ["ECANCELLED"], [undefined]])("reports the peer as unreachable for %s", async (code) => {
const error = code ? Object.assign(new Error("dns failure"), { code }) : new Error("dns failure");
mockResolverInstance.resolve4.mockRejectedValueOnce(error);
const provider = new DNSProvider(createResolver);
const result = await provider.handle(makeMonitor());
expect(result.status).toBe(false);
expect(result.peerResponded).toBe(false);
});
it("reports the peer as having answered on success", async () => {
const provider = new DNSProvider(createResolver);
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
it("stringifies non-Error throws from outside timeRequest", async () => {
mockResolverInstance.setServers.mockImplementationOnce(() => {
throw 42;
@@ -697,4 +697,43 @@ describe("DockerProvider", () => {
await expect(provider.handle(makeMonitor())).rejects.toThrow("Error performing Docker request");
});
});
// ── Peer reachability ────────────────────────────────────────────────────
// A daemon that answers with an error has been reached; only silence can be the instance's own egress.
describe("peer reachability", () => {
it("reports the peer as having answered when the daemon replies with a status code", async () => {
const dockerErr = Object.assign(new Error("base"), { statusCode: 500, json: { message: "engine exploded" } });
const { provider } = setup({ ping: jest.fn().mockRejectedValue(dockerErr) });
const result = await provider.handle(makeMonitor());
expect(result.status).toBe(false);
expect(result.peerResponded).toBe(true);
});
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
const { provider } = setup({ ping: jest.fn().mockRejectedValue(Object.assign(new Error("connect failed"), { code })) });
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
it("reports the peer as having answered when the daemon is reachable", async () => {
const { provider } = setup({});
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
});
});
@@ -362,6 +362,59 @@ describe("GrpcProvider", () => {
[99, "UNKNOWN"],
];
// ── Peer reachability ────────────────────────────────────────────────
// NOT_SERVING and a connection failure both report NETWORK_ERROR, and gRPC codes are not HTTP statuses,
// so `peerResponded` is what stops the egress check reading a server's answer as a transport failure.
it("reports the peer as having answered when the server replies NOT_SERVING", async () => {
const { provider } = createProvider({ behavior: "not-serving" });
const result = await provider.handle(makeMonitor());
expect(result.status).toBe(false);
expect(result.code).toBe(5000);
expect(result.peerResponded).toBe(true);
});
it.each([[5], [7], [12], [16], [2]])("reports the peer as having answered for gRPC code %i", async (code) => {
const client = {
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
close: jest.fn(),
};
const mocks = createMockGrpc(client as any);
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
it.each([[4], [14]])("reports the peer as unreachable for gRPC code %i", async (code) => {
const client = {
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
close: jest.fn(),
};
const mocks = createMockGrpc(client as any);
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
it("reports the peer as unreachable when the failure carries no gRPC code", async () => {
const client = {
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ message: "socket hang up" })),
close: jest.fn(),
};
const mocks = createMockGrpc(client as any);
const provider = new GrpcProvider(mocks.grpc as any, mocks.protoLoader as any);
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
it.each(statusCodes)("maps gRPC code %i to %s", async (code, expectedName) => {
const client = {
Check: jest.fn((_r: any, _o: any, cb: Function) => cb({ code, details: "test" })),
@@ -643,4 +643,92 @@ describe("HttpProvider", () => {
expect(result.message).toBe("Response code 500 (Internal Server Error)");
});
});
// ── Peer reachability ────────────────────────────────────────────────────
// Any HTTP status proves the target answered. Without one, only an errno that requires the peer's own stack
// to have replied counts — and never through a proxy, where the refusal may be the proxy's rather than the
// target's, which is the instance's own egress path failing.
describe("peer reachability", () => {
it("reports the peer as having answered on a successful response", async () => {
mockGot.mockResolvedValue(makeGotResponse());
const { provider } = createProvider();
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
it("reports the peer as having answered for a failing status code", async () => {
const err = new HTTPError("Not Found", { statusCode: 404, body: "", headers: {} }, { phases: { total: 12 } });
mockGot.mockRejectedValue(err);
const { provider } = createProvider();
const result = await provider.handle(makeMonitor());
expect(result.status).toBe(false);
expect(result.code).toBe(404);
expect(result.peerResponded).toBe(true);
});
it.each([["ECONNREFUSED"], ["ECONNRESET"], ["CERT_HAS_EXPIRED"], ["DEPTH_ZERO_SELF_SIGNED_CERT"]])(
"reports the peer as having answered for %s with no response",
async (code) => {
const err = new RequestError("request failed");
(err as any).code = code;
mockGot.mockRejectedValue(err);
const { provider } = createProvider();
const result = await provider.handle(makeMonitor());
expect(result.code).toBe(NETWORK_ERROR);
expect(result.peerResponded).toBe(true);
}
);
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["ENETUNREACH"]])("reports the peer as silent for %s", async (code) => {
const err = new RequestError("request failed");
(err as any).code = code;
mockGot.mockRejectedValue(err);
const { provider } = createProvider();
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
it("reports the peer as silent when a proxy refuses the connection", async () => {
const err = new RequestError("connect ECONNREFUSED");
(err as any).code = "ECONNREFUSED";
mockGot.mockRejectedValue(err);
const { provider } = createProvider();
const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
// The refusal came from the proxy, which is part of this instance's route out.
expect(result.peerResponded).toBe(false);
});
it("reports the peer as having answered through a proxy when a status code came back", async () => {
const err = new RequestError("Response code 500 (Internal Server Error)");
(err as any).code = "ECONNRESET";
(err as any).response = { statusCode: 500, body: "", headers: {} };
(err as any).timings = { phases: { total: 25 } };
mockGot.mockRejectedValue(err);
const { provider } = createProvider();
const result = await provider.handle(makeMonitor(), { proxyUrl: "http://proxy.example.com:8080" });
expect(result.peerResponded).toBe(true);
});
it("reports the peer as silent for a generic error carrying no code", async () => {
mockGot.mockRejectedValue(new Error("something broke"));
const { provider } = createProvider();
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
});
});
@@ -155,4 +155,23 @@ describe("PingProvider", () => {
await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
});
// ── Peer reachability ────────────────────────────────────────────────────
it("reports the peer as having answered when an echo reply comes back", async () => {
const provider = new PingProvider(createMockPing({ alive: true }) as any, net);
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
it("reports the peer as silent when no echo reply comes back", async () => {
// Silence is indistinguishable from the instance losing its egress, so the egress probe decides.
const provider = new PingProvider(createMockPing({ alive: false }) as any, net);
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
});
@@ -267,4 +267,42 @@ describe("PortProvider", () => {
await expect(provider.handle(monitor)).rejects.toThrow("42");
});
// ── Peer reachability ────────────────────────────────────────────────────
// A check is only recorded as degraded when nothing answered, so a service on the local network that
// refuses connections has to stay distinguishable from the instance losing its own egress.
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
const result = await provider.handle(makeMonitor());
expect(result.status).toBe(false);
expect(result.code).toBe(NETWORK_ERROR);
expect(result.peerResponded).toBe(true);
});
it.each([["ETIMEDOUT"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ENOTFOUND"]])("reports the peer as silent for %s", async (code) => {
const provider = new PortProvider(createErrorNet(Object.assign(new Error("connect failed"), { code })));
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
it("reports the peer as silent on a connection timeout", async () => {
const provider = new PortProvider(createTimeoutNet());
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
it("reports the peer as having answered when the port accepts the connection", async () => {
const provider = new PortProvider(createSuccessNet());
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
});
@@ -1,5 +1,5 @@
import { describe, expect, it } from "@jest/globals";
import { timeRequest, isStatusUp } from "../../../../src/service/network/utils.ts";
import { timeRequest, isStatusUp, peerAnsweredFromError } from "../../../../src/service/network/utils.ts";
import { NETWORK_ERROR } from "../../../../src/types/network.ts";
describe("network utils", () => {
@@ -39,6 +39,40 @@ describe("network utils", () => {
});
});
// The egress check blames the instance only when nothing answered, so this helper's `true` must never be a
// guess: every code in it requires the peer's own stack to have replied.
describe("peerAnsweredFromError", () => {
it.each([
["ECONNREFUSED", "the host refused the connection"],
["ECONNRESET", "the host reset an established connection"],
["EPIPE", "the host closed a connection we were writing to"],
["EPROTO", "a protocol error on a connection that was made"],
["CERT_HAS_EXPIRED", "the host presented a certificate"],
["DEPTH_ZERO_SELF_SIGNED_CERT", "the host presented a certificate"],
["SELF_SIGNED_CERT_IN_CHAIN", "the host presented a certificate"],
["UNABLE_TO_GET_ISSUER_CERT_LOCALLY", "the host presented a certificate"],
["UNABLE_TO_VERIFY_LEAF_SIGNATURE", "the host presented a certificate"],
["ERR_TLS_CERT_ALTNAME_INVALID", "the host presented a certificate"],
["CERT_NOT_YET_VALID", "the host presented a certificate"],
["ERR_SSL_WRONG_VERSION_NUMBER", "the host spoke back, badly"],
])("reports the peer as having answered for %s", (code) => {
expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(true);
});
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EAI_AGAIN"], ["EHOSTUNREACH"], ["ENETUNREACH"], ["ECONNABORTED"], ["ENETDOWN"]])(
"reports the peer as silent for %s",
(code) => {
expect(peerAnsweredFromError(Object.assign(new Error("failed"), { code }))).toBe(false);
}
);
it("reports the peer as silent when the error carries no code", () => {
expect(peerAnsweredFromError(new Error("Connection timeout"))).toBe(false);
expect(peerAnsweredFromError(undefined)).toBe(false);
expect(peerAnsweredFromError("not an error")).toBe(false);
});
});
describe("isStatusUp", () => {
it("returns true for 2xx status codes", () => {
expect(isStatusUp(200)).toBe(true);
@@ -191,4 +191,42 @@ describe("WebSocketProvider", () => {
);
});
});
// ── Peer reachability ────────────────────────────────────────────────────
// `ws` rejects a non-101 handshake with a plain Error, so only the errno cases can be told apart from the
// instance losing its egress. That leaves a server answering the handshake with an HTTP error indistinct.
const createErrorWS = (error: Error) =>
jest.fn().mockImplementation(() => ({
on: jest.fn((event: string, cb: Function) => {
if (event === "error") process.nextTick(() => cb(error));
}),
close: jest.fn(),
})) as any;
it.each([["ECONNREFUSED"], ["ECONNRESET"]])("reports the peer as having answered for %s", async (code) => {
const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
const result = await provider.handle(makeMonitor());
expect(result.status).toBe(false);
expect(result.code).toBe(NETWORK_ERROR);
expect(result.peerResponded).toBe(true);
});
it.each([["ETIMEDOUT"], ["ENOTFOUND"], ["EHOSTUNREACH"]])("reports the peer as silent for %s", async (code) => {
const provider = new WebSocketProvider(createErrorWS(Object.assign(new Error("connect failed"), { code })));
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(false);
});
it("reports the peer as having answered when the socket opens", async () => {
const provider = new WebSocketProvider(createMockWS("open"));
const result = await provider.handle(makeMonitor());
expect(result.peerResponded).toBe(true);
});
});
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
@@ -19,9 +20,10 @@ const createEvaluator = (overrides?: Record<string, any>) => {
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
},
monitorStatusPolicy: new MonitorStatusPolicy(),
logger: createMockLogger(),
...overrides,
};
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
return { evaluator, defaults };
};
@@ -84,4 +86,38 @@ describe("CheckEvaluator", () => {
shouldSendNotification: false,
});
});
// ── degraded egress ───────────────────────────────────────────────────────
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "degraded" } as any;
const monitor = makeMonitor({ status: "up" });
const { evaluator, defaults } = createEvaluator();
const result = await evaluator.evaluate(status, check, monitor);
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
expect(result.monitor).toBe(monitor);
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
expect(result.decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
});
expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
});
it("evaluates a check flagged with egress ok as normal", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "ok" } as any;
const { evaluator, defaults } = createEvaluator();
await evaluator.evaluate(status, check, makeMonitor());
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
});
});
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
import { NETWORK_ERROR } from "../../../src/types/network.ts";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
@@ -29,6 +30,7 @@ const createProducer = (overrides?: Record<string, any>) => {
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
buffer: { addToBuffer: jest.fn() },
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
...overrides,
};
const producer = new CheckProducer(
@@ -39,6 +41,7 @@ const createProducer = (overrides?: Record<string, any>) => {
defaults.proxyResolver as any,
defaults.buffer as any,
defaults.dockerLogsService as any,
defaults.egressService as any,
defaults.logger as any
);
return { producer, defaults };
@@ -76,6 +79,31 @@ describe("CheckProducer", () => {
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
const monitor = makeMonitor({ status: "maintenance" });
const { producer, defaults } = createProducer({
monitorsRepository: {
updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }),
findDockerTlsKeyById: jest.fn().mockResolvedValue(null),
},
});
await producer.produce(monitor);
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
});
it("does not touch the status of a monitor that was not in maintenance", async () => {
const { producer, defaults } = createProducer();
await producer.produce(makeMonitor({ status: "up" }));
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("proceeds with the check when the maintenance window is inactive", async () => {
const { producer, defaults } = createProducer({
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
@@ -210,4 +238,222 @@ describe("CheckProducer", () => {
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
});
// ── egress self-check ─────────────────────────────────────────────────────
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
});
it("does not consult egress when an HTTP response was received", async () => {
for (const status of [
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
]) {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
}
});
it("consults egress for a timeout or network error", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
});
it("never consults egress for a docker daemon on a local socket", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" }));
// Local IPC never leaves the instance, so it cannot fail through egress.
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
});
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
for (const monitor of [
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
]) {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(monitor);
// Both requests leave the instance, so our own loss of egress can be the cause.
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
}
});
it("does not consult egress when the provider reports that the peer answered", async () => {
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
// them from a connection failure.
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ type: "dns" }));
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
expect(check).not.toHaveProperty("egressStatus");
});
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ type: "grpc" }));
// An explicit report from the provider outranks the HTTP-status heuristic.
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
});
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
const result = await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check.egressStatus).toBe("degraded");
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
expect(result?.check.egressStatus).toBe("degraded");
});
// ── status window across a spell ─────────────────────────────────────────
// Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either
// side of a spell are never adjacent.
it("clears the status window on the first degraded check of a spell", async () => {
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] });
});
it("does not re-write the status window for later degraded checks in the same spell", async () => {
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
});
await producer.produce(makeMonitor({ statusWindow: [] }));
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("leaves the status window alone when the probe found egress fine", async () => {
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
});
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("flags a failing check as ok when the probe found egress fine", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
});
await producer.produce(makeMonitor());
expect(check.egressStatus).toBe("ok");
});
it("buffers the check when the egress assessment rejects", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) },
});
const result = await producer.produce(makeMonitor());
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
expect(check).not.toHaveProperty("egressStatus");
expect(result).toEqual({ status: failingStatus, check });
expect(defaults.logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" })
);
});
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn().mockReturnValue(check) },
});
await producer.produce(makeMonitor());
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
expect(check).not.toHaveProperty("egressStatus");
});
});
@@ -21,7 +21,6 @@ const makeState = (overrides?: Partial<EgressState>): EgressState => ({
const makeSettings = (overrides?: Record<string, unknown>) => ({
egressCheckEnabled: true,
egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
egressPollIntervalSeconds: 30,
egressNotifications: ["notif-1"],
...overrides,
});
@@ -33,7 +32,7 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
refId: null,
isActive: true,
nextScheduledAt: 1_000,
intervalMs: 30_000,
intervalMs: 5_000,
lockedBy: "worker-1",
lockedUntil: 61_000,
runCount: 0,
@@ -245,7 +244,7 @@ describe("EgressService", () => {
refId: null,
isActive: true,
nextScheduledAt: expect.any(Number),
intervalMs: 30_000,
intervalMs: 5_000,
});
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
@@ -266,7 +265,7 @@ describe("EgressService", () => {
const result = await service.assessAfterFailure();
expect(result).toBe("degraded");
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
});
@@ -293,21 +292,10 @@ describe("EgressService", () => {
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 5_000 }));
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
const { service, defaults } = createService({
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
networkService: { requestStatus: statusFor([]) },
});
await service.assessAfterFailure();
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
});
it("falls back to the default targets when the configured list is empty", async () => {
const { service, defaults } = createService({
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
@@ -1,6 +1,6 @@
import { describe, expect, it } from "@jest/globals";
import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts";
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts";
import { MAX_EGRESS_TARGETS } from "../../../src/domain/egress/egress.type.ts";
const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8";
@@ -10,13 +10,11 @@ describe("settingsValidation", () => {
const parsed = updateAppSettingsBodyValidation.parse({
egressCheckEnabled: true,
egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"],
egressPollIntervalSeconds: 30,
egressNotifications: [validNotificationId],
});
expect(parsed.egressCheckEnabled).toBe(true);
expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]);
expect(parsed.egressPollIntervalSeconds).toBe(30);
expect(parsed.egressNotifications).toEqual([validNotificationId]);
});
@@ -62,21 +60,6 @@ describe("settingsValidation", () => {
}
});
it("accepts the poll interval boundaries", () => {
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
MIN_EGRESS_POLL_INTERVAL_SECONDS
);
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
MAX_EGRESS_POLL_INTERVAL_SECONDS
);
});
it("rejects out-of-range and non-integer poll intervals", () => {
for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow();
}
});
it("accepts an empty notification list", () => {
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
});