Merge pull request #3963 from web-engineer/feat/egress-2-check-persistence

Egress self-check (2/4): flag checks taken during degraded egress (#3937)

server/src/api/validation/settingsValidation.ts

@@ -1,5 +1,5 @@
 import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
-import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
+import { MAX_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
 import { parseEgressTarget } from "@/utils/egressTarget.js";
 import { z } from "zod";
 
@@ -56,7 +56,6 @@ export const updateAppSettingsBodyValidation = z
 
 		egressCheckEnabled: z.boolean().optional(),
 		egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
-		egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
 		egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(),
 	})
 	.strip()

server/src/config/services.worker.ts

@@ -129,9 +129,10 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
 		proxyResolver,
 		bufferService,
 		dockerLogsService,
+		egressService,
 		logger
 	);
-	const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
+	const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy, logger);
 	const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
 
 	// ***********************

server/src/domain/app-settings/app-settings.model.ts

@@ -1,6 +1,6 @@
 import { Schema, model, type Types } from "mongoose";
 import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
-import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
+import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
 
 interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
 	_id: Types.ObjectId;
@@ -45,7 +45,6 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
 		globalProxyId: { type: String, default: null },
 		egressCheckEnabled: { type: Boolean, default: false },
 		egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
-		egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
 		egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
 	},
 	{ timestamps: true }

server/src/domain/app-settings/app-settings.repository.mongo.ts

@@ -3,7 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
 import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
 import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
 import { toStringId, toDateString } from "@/utils/mongoMappers.js";
-import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
+import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
 
 class MongoSettingsRepository implements ISettingsRepository {
 	private toEntity = (doc: AppSettingsDocument): Settings => {
@@ -33,7 +33,6 @@ class MongoSettingsRepository implements ISettingsRepository {
 			globalProxyId: doc.globalProxyId ?? undefined,
 			egressCheckEnabled: doc.egressCheckEnabled ?? false,
 			egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
-			egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
 			egressNotifications: doc.egressNotifications ?? [],
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),

server/src/domain/app-settings/app-settings.type.ts

@@ -54,7 +54,6 @@ export interface Settings {
 	// Egress self-check (see domain/egress)
 	egressCheckEnabled: boolean;
 	egressCheckTargets: string[];
-	egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
 	egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
 	createdAt: string;
 	updatedAt: string;

server/src/domain/checks/check.docker.aggregation.ts

@@ -1,5 +1,6 @@
 import CheckModel from "@/domain/checks/check.model.js";
 import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
+import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
 import mongoose from "mongoose";
 
 type DateRange = { start: Date; end: Date };
@@ -9,6 +10,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "docker",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	});
 
 export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
@@ -16,6 +18,7 @@ export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Pr
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "docker",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 		status: true,
 	});
 	return { totalChecks: count };
@@ -28,6 +31,7 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
 				"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 				"metadata.type": "docker",
 				createdAt: { $gte: dates.start, $lte: dates.end },
+				...EXCLUDE_DEGRADED_EGRESS_MATCH,
 			},
 		},
 		{
@@ -48,6 +52,7 @@ export const getDockerLatestCheck = async (monitorId: string) =>
 	CheckModel.findOne({
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "docker",
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	})
 		.sort({ createdAt: -1 })
 		.select("containers containerSummary createdAt")
@@ -65,6 +70,7 @@ export const getDockerContainerStats = async (
 				"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 				"metadata.type": "docker",
 				createdAt: { $gte: dates.start, $lte: dates.end },
+				...EXCLUDE_DEGRADED_EGRESS_MATCH,
 				"containers.name": containerName,
 			},
 		},
@@ -88,6 +94,7 @@ export const getDockerContainerLatestCheck = async (monitorId: string, container
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "docker",
 		"containers.name": containerName,
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	})
 		.sort({ createdAt: -1 })
 		.select("containers createdAt")

server/src/domain/checks/check.hardware.aggregations.ts

@@ -1,6 +1,7 @@
 import { CheckModel } from "@/domain/checks/check.model.js";
 import mongoose from "mongoose";
 import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
+import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
 
 type DateRange = { start: Date; end: Date };
 type HardwareUpChecks = { totalChecks: number };
@@ -70,6 +71,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "hardware",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	});
 };
 
@@ -78,6 +80,7 @@ export const getHardwareUpChecks = async (monitorId: string, dates: DateRange):
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "hardware",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 		status: true,
 	});
 	return { totalChecks: count };
@@ -95,6 +98,7 @@ export const getHardwareStats = async (monitorId: string, dates: DateRange, date
 				"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 				"metadata.type": "hardware",
 				createdAt: { $gte: dates.start, $lte: dates.end },
+				...EXCLUDE_DEGRADED_EGRESS_MATCH,
 			},
 		},
 		{ $sort: { createdAt: 1 } },

server/src/domain/checks/check.model.ts

@@ -1,5 +1,6 @@
 import { Schema, model, Types } from "mongoose";
 import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
+import { EgressStatuses } from "@/domain/egress/egress.type.js";
 import type {
 	Check,
 	CheckAudits,
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
 			type: containerSummarySchema,
 			default: undefined,
 		},
+		egressStatus: {
+			type: String,
+			enum: EgressStatuses,
+			default: undefined,
+		},
 	},
 	{
 		timestamps: true,

server/src/domain/checks/check.query.ts

@@ -0,0 +1,7 @@
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
+
+// A check recorded while the instance's own egress was down says nothing about the target, so it is left out of
+// every aggregate: uptime percentages, up/down counts and the response-time series alike. A degraded episode
+// therefore reads as a gap in monitoring rather than as downtime or as a dip in response time.
+// Spread this into the $match or filter of any query that reports statistics. Check listings keep degraded checks.
+export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;

server/src/domain/checks/check.repository.mongo.ts

@@ -16,6 +16,7 @@ import type {
 } from "@/domain/checks/check.type.js";
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
 import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
+import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
 import mongoose from "mongoose";
 import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
 import { ILogger } from "@/utils/logger.js";
@@ -179,6 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
 			audits: mapAudits(doc.audits),
 			containers: doc.containers,
 			containerSummary: doc.containerSummary,
+			egressStatus: doc.egressStatus,
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
 		};
@@ -348,6 +350,7 @@ class MongoChecksRepository implements IChecksRepository {
 		const baseMatch = {
 			"metadata.teamId": new mongoose.Types.ObjectId(teamId),
 			createdAt: { $gte: getDateForRange(dateRange) },
+			...EXCLUDE_DEGRADED_EGRESS_MATCH,
 		};
 
 		const [totalResult, downResult] = await Promise.all([
@@ -393,6 +396,7 @@ class MongoChecksRepository implements IChecksRepository {
 				$match: {
 					"metadata.monitorId": { $in: objectIds },
 					createdAt: { $gte: windowStart },
+					...EXCLUDE_DEGRADED_EGRESS_MATCH,
 				},
 			},
 			{
@@ -473,6 +477,7 @@ class MongoChecksRepository implements IChecksRepository {
 		const matchStage = {
 			"metadata.monitorId": monitorObjectId,
 			createdAt: { $gte: startDate, $lte: endDate },
+			...EXCLUDE_DEGRADED_EGRESS_MATCH,
 		};
 		const [result] = await CheckModel.aggregate([
 			{ $match: matchStage },
@@ -638,6 +643,7 @@ class MongoChecksRepository implements IChecksRepository {
 		const matchStage = {
 			"metadata.monitorId": monitorObjectId,
 			createdAt: { $gte: startDate, $lte: endDate },
+			...EXCLUDE_DEGRADED_EGRESS_MATCH,
 		};
 
 		const [result] = await CheckModel.aggregate([

server/src/domain/checks/check.type.ts

@@ -1,4 +1,5 @@
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
 import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
 import type { Response } from "got";
 
@@ -116,6 +117,10 @@ export interface Check {
 	seo?: number;
 	performance?: number;
 	audits?: CheckAudits;
+	// Set on failing checks while the egress check is enabled; absent otherwise.
+	// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
+	// "ok": a reliability target was reachable.
+	egressStatus?: EgressStatus;
 	createdAt: string;
 	updatedAt: string;
 }

server/src/domain/egress/egress.service.ts

@@ -7,7 +7,7 @@ import type { INetworkService } from "@/service/networkService.js";
 import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
 import type { ILogger } from "@/utils/logger.js";
 import {
-	DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
+	EGRESS_RECOVERY_POLL_SECONDS,
 	DEFAULT_EGRESS_TARGETS,
 	type EgressAssessment,
 	type EgressProbeResult,
@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
 const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
 
 // HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
-const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
+export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
 
 // Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
 const PROBE_MONITOR_ID = "egress-probe";
@@ -136,11 +136,6 @@ export class EgressService implements IEgressService {
 		return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
 	};
 
-	private toIntervalMs = (seconds: number | undefined): number => {
-		const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
-		return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
-	};
-
 	// ****************************
 	// Event-triggered assessment
 	// ****************************
@@ -181,7 +176,7 @@ export class EgressService implements IEgressService {
 			const state = await this.egressStateRepository.findSingleton();
 			if (state.status === "degraded") {
 				// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
-				await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
+				await this.scheduleRecoveryCheck(false);
 				return "degraded";
 			}
 
@@ -195,7 +190,7 @@ export class EgressService implements IEgressService {
 
 			// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
 			// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
-			await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
+			await this.scheduleRecoveryCheck(true);
 			const degraded = await this.egressStateRepository.markDegraded(results, now);
 			this.logger.warn({
 				message: degraded
@@ -224,8 +219,8 @@ export class EgressService implements IEgressService {
 
 	// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
 	// so re-arming while degraded never pushes a pending run back.
-	private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
-		const intervalMs = this.toIntervalMs(pollIntervalSeconds);
+	private scheduleRecoveryCheck = async (reschedule: boolean) => {
+		const intervalMs = EGRESS_RECOVERY_POLL_SECONDS * 1000;
 		const seed: JobSeed = {
 			id: RECOVERY_JOB_ID,
 			type: RECOVERY_JOB_TYPE,

server/src/domain/egress/egress.type.ts

@@ -9,11 +9,13 @@ export type EgressAssessment = EgressStatus | "disabled";
 // Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
 // http(s):// URLs are fetched. Any one target being reachable means egress is fine.
 export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
-export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
-export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
-export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
 export const MAX_EGRESS_TARGETS = 10;
 
+// While the instance is marked degraded, every failing check is flagged from stored state without probing, so
+// this interval is the window in which a recovered instance still discards its checks. It is deliberately fixed
+// and not configurable: the only effect of raising it is a longer blind window.
+export const EGRESS_RECOVERY_POLL_SECONDS = 5;
+
 // Instance-global singleton persisted in MongoDB so that every worker process and the API
 // process share one view of the state (the queue worker may run separately from the API).
 export interface EgressState {

server/src/domain/monitors/monitor.type.ts

@@ -50,6 +50,31 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
 export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
 export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
 
+// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
+// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
+// however far away the agent is. `unknown` is excluded because no request is made for it.
+export const EgressAttributableTypes = [
+	"http",
+	"ping",
+	"pagespeed",
+	"hardware",
+	"docker",
+	"port",
+	"game",
+	"grpc",
+	"websocket",
+	"dns",
+] as const satisfies readonly MonitorType[];
+export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
+
+// A Docker daemon reached over a unix socket is local IPC and cannot fail through egress; one reached over
+// TCP or TLS is as remote as any other target. The URL, not the type, settles it.
+const isLocalDockerSocket = (monitor: Pick<Monitor, "type" | "url">): boolean =>
+	monitor.type === "docker" && (monitor.url?.trim().startsWith("unix://") ?? false);
+
+export const isEgressAttributable = (monitor: Pick<Monitor, "type" | "url">): boolean =>
+	EgressAttributableTypes.some((t) => t === monitor.type) && !isLocalDockerSocket(monitor);
+
 export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
 export type MonitorStatus = (typeof MonitorStatuses)[number];
 

server/src/service/network/DNSProvider.ts

@@ -1,5 +1,6 @@
 import { DNSStatusPayload, MonitorStatusResponse } from "@/types/network.js";
 import { IStatusProvider } from "@/service/network/IStatusProvider.js";
+import dns from "dns";
 import type { Resolver } from "dns/promises";
 import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
 import { AppError } from "@/utils/AppError.js";
@@ -8,6 +9,16 @@ import { NETWORK_ERROR } from "@/types/network.js";
 
 const SERVICE_NAME = "DNSProvider";
 
+// c-ares codes where the DNS server replied: the name does not exist, holds no such record, or the server
+// declined or answered badly. Anything else (timeout, connection refused, a local resolver fault) means we
+// never got an answer, so the failure could be our own loss of egress rather than the target's.
+const ANSWERED_DNS_CODES: ReadonlySet<string> = new Set([dns.NODATA, dns.FORMERR, dns.SERVFAIL, dns.NOTFOUND, dns.NOTIMP, dns.REFUSED, dns.BADRESP]);
+
+const dnsServerAnswered = (error: unknown): boolean => {
+	const code = (error as NodeJS.ErrnoException | undefined)?.code;
+	return typeof code === "string" && ANSWERED_DNS_CODES.has(code);
+};
+
 export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
 	readonly type = "dns";
 	constructor(private createResolver: () => Resolver) {}
@@ -63,6 +74,7 @@ export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
 					type: monitor.type,
 					status: false,
 					code: NETWORK_ERROR,
+					peerResponded: dnsServerAnswered(error),
 					message: error instanceof Error ? error.message : String(error),
 					responseTime,
 					payload: {
@@ -81,6 +93,7 @@ export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
 				type: monitor.type,
 				status: true,
 				code: 200,
+				peerResponded: true,
 				message: "Success",
 				responseTime,
 				payload: {

server/src/service/network/DockerProvider.ts

@@ -23,7 +23,7 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
 import { ILogger } from "@/utils/logger.js";
 import { AppError } from "@/utils/AppError.js";
 import Dockerode from "dockerode";
-import { timeRequest } from "@/service/network/utils.js";
+import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import { IEncryptionService } from "@/service/encryption/encryptionService.js";
 import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
@@ -315,12 +315,19 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
 		return rawLogs.filter((log) => log !== null);
 	};
 
-	private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse<DockerStatusPayload> => ({
+	private failed = (
+		monitor: Monitor,
+		code: number,
+		message: string,
+		responseTime: number,
+		peerResponded = false
+	): MonitorStatusResponse<DockerStatusPayload> => ({
 		monitorId: monitor.id,
 		teamId: monitor.teamId,
 		type: monitor.type,
 		status: false,
 		code,
+		peerResponded,
 		message,
 		responseTime,
 		payload: null,
@@ -337,13 +344,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
 			if (error) {
 				let message = "Docker host is unreachable";
 				let code = NETWORK_ERROR;
+				// A status code from the daemon is the daemon answering; otherwise only a refusal or a reset
+				// proves we reached the host, and anything else could be the instance's own loss of egress.
+				let peerResponded = peerAnsweredFromError(error);
 				if (this.isDockerError(error)) {
 					code = error.statusCode ?? NETWORK_ERROR;
 					message = error.json?.message ?? error.reason ?? error.message;
+					peerResponded = peerResponded || error.statusCode !== undefined;
 				} else if (error instanceof Error) {
 					message = error.message;
 				}
-				return this.failed(monitor, code, message, responseTime);
+				return this.failed(monitor, code, message, responseTime, peerResponded);
 			}
 
 			const summaries = await docker.listContainers({ all: true });
@@ -362,6 +373,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
 				type: monitor.type,
 				status: true,
 				code: 200,
+				peerResponded: true,
 				message: "Docker host is reachable",
 				responseTime,
 				payload: { containers, summary, logs },

server/src/service/network/GameProvider.ts

@@ -43,6 +43,9 @@ export class GameProvider implements IStatusProvider<GameStatusPayload> {
 				});
 
 			if (!state) {
+				// gamedig reports a failed query the same way whichever leg failed, so `peerResponded` is left
+				// unset and a failure is put to the egress probe. A game server that stops answering during a
+				// spell is therefore recorded as degraded rather than down.
 				return {
 					monitorId: id,
 					teamId: teamId,

server/src/service/network/GrpcProvider.ts

@@ -25,6 +25,17 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
 		return type === "grpc";
 	}
 
+	// gRPC codes that mean the call never reached a server. Everything else in 0-16 is the server's own answer,
+	// including NOT_SERVING, which shares NETWORK_ERROR with a connection failure and so cannot be read from `code`.
+	// A call that fails with no gRPC code at all never got as far as the protocol.
+	private peerAnswered(code: number | undefined): boolean {
+		const UNREACHABLE_GRPC_CODES: ReadonlySet<number> = new Set([
+			4, // DEADLINE_EXCEEDED
+			14, // UNAVAILABLE
+		]);
+		return typeof code === "number" && !UNREACHABLE_GRPC_CODES.has(code);
+	}
+
 	private getGrpcStatusName(code: number): string {
 		const statusNames: Record<number, string> = {
 			0: "OK",
@@ -148,6 +159,7 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
 					type: type,
 					status: false,
 					code: grpcError.grpcCode ?? 5000,
+					peerResponded: this.peerAnswered(grpcError.grpcCode),
 					message: grpcError.message ?? "gRPC health check failed",
 					responseTime,
 					payload: payload ?? null,
@@ -163,6 +175,8 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
 				type: type,
 				status: isServing,
 				code: isServing ? 200 : 5000,
+				// The server replied, whatever it said, so this failure is never the instance's own egress.
+				peerResponded: true,
 				message: isServing ? `gRPC service healthy (${grpcPayload.servingStatus})` : `gRPC service unhealthy (${grpcPayload.servingStatus})`,
 				responseTime,
 				payload: grpcPayload,

server/src/service/network/HttpProvider.ts

@@ -6,7 +6,7 @@ import { MonitorStatusResponse } from "@/types/network.js";
 import { Agent as HttpsAgent } from "https";
 import { Agent as HttpAgent } from "http";
 import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
-import { isStatusUp } from "@/service/network/utils.js";
+import { isStatusUp, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import CacheableLookup from "cacheable-lookup";
 import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
@@ -107,6 +107,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 				type: monitor.type,
 				status: statusUp,
 				code: statusCode,
+				peerResponded: true,
 				message,
 				responseTime,
 				timings,
@@ -123,6 +124,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 				type: monitor.type,
 				status: false,
 				code: statusCode,
+				peerResponded: true,
 				message: "Response is not JSON",
 				responseTime,
 				timings,
@@ -149,6 +151,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 			type: monitor.type,
 			status: statusUp && matchResult.ok,
 			code: statusCode,
+			peerResponded: true,
 			message: matchResult.ok ? message : matchResult.message,
 			responseTime,
 			timings,
@@ -172,6 +175,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 					type: monitor.type,
 					status: false,
 					code: statusCode ?? NETWORK_ERROR,
+					peerResponded: statusCode !== undefined || (!ctx?.proxyUrl && peerAnsweredFromError(error)),
 					message: message,
 					responseTime,
 					timings: error.timings,
@@ -196,6 +200,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 			type: monitor.type,
 			status: false,
 			code: NETWORK_ERROR,
+			peerResponded: !ctx?.proxyUrl && peerAnsweredFromError(error),
 			message: error instanceof Error ? error.message : String(error),
 			responseTime: 0,
 			payload: null as T,

server/src/service/network/PingProvider.ts

@@ -58,6 +58,9 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
 				type: monitor.type,
 				status: response.alive ?? false,
 				code: response.alive ? 200 : 5000,
+				// An echo reply is the host answering. Its absence is silence, which is indistinguishable
+				// from the instance having lost its egress, so the egress probe decides that case.
+				peerResponded: response.alive ?? false,
 				message: response.alive ? "Success" : "Ping failed",
 				responseTime,
 				payload: response,

server/src/service/network/PortProvider.ts

@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
 import { PortStatusPayload, MonitorStatusResponse } from "@/types/network.js";
 import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
 import { AppError } from "@/utils/AppError.js";
-import { timeRequest } from "@/service/network/utils.js";
+import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import * as net from "net";
 type NetType = typeof net;
@@ -59,6 +59,9 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
 					type: monitor.type,
 					status: false,
 					code: NETWORK_ERROR,
+					// A refused or reset connection came from the host itself; a timeout or an unreachable
+					// network did not, and could equally be the instance's own loss of egress.
+					peerResponded: peerAnsweredFromError(error),
 					message: errorMessage,
 					responseTime: responseTime,
 					timings: undefined,
@@ -72,6 +75,7 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
 				type: monitor.type,
 				status: true,
 				code: 200,
+				peerResponded: true,
 				message: "Port check successful",
 				responseTime: responseTime,
 				timings: undefined,

server/src/service/network/WebSocketProvider.ts

@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
 import { WebSocketStatusPayload, MonitorStatusResponse } from "@/types/network.js";
 import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
 import { AppError } from "@/utils/AppError.js";
-import { timeRequest } from "@/service/network/utils.js";
+import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
 import { NETWORK_ERROR } from "@/types/network.js";
 import type WebSocket from "ws";
 
@@ -63,6 +63,9 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
 					type: monitor.type,
 					status: false,
 					code: NETWORK_ERROR,
+					// `ws` rejects a non-101 handshake with a plain Error carrying only a message, so only the
+					// errno cases — a refusal or a reset — can be told apart from the instance losing egress.
+					peerResponded: peerAnsweredFromError(error),
 					message: errorMessage,
 					responseTime: responseTime,
 					timings: undefined,
@@ -76,6 +79,7 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
 				type: monitor.type,
 				status: true,
 				code: 200,
+				peerResponded: true,
 				message: "WebSocket check successful",
 				responseTime: responseTime,
 				timings: undefined,

server/src/service/network/utils.ts

@@ -16,3 +16,27 @@ export const isStatusUp = (statusCode: number | undefined, customUpCodes: HttpSt
 	if (statusCode === undefined) return false;
 	return (statusCode >= 200 && statusCode < 300) || customUpCodes.includes(statusCode);
 };
+
+// Errno codes that can only arise once the target's own stack has answered: it refused the connection, reset
+// an established one, or got far enough into a TLS handshake to present a certificate. Every other failure —
+// a timeout, an unreachable network, a name that would not resolve — looks the same whether the target is
+// down or the instance has lost its egress, so those are left for the egress probe to decide.
+const PEER_ANSWERED_ERROR_CODES: ReadonlySet<string> = new Set([
+	"ECONNREFUSED",
+	"ECONNRESET",
+	"EPIPE",
+	"EPROTO",
+	"CERT_HAS_EXPIRED",
+	"CERT_NOT_YET_VALID",
+	"DEPTH_ZERO_SELF_SIGNED_CERT",
+	"SELF_SIGNED_CERT_IN_CHAIN",
+	"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
+	"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
+	"ERR_TLS_CERT_ALTNAME_INVALID",
+	"ERR_SSL_WRONG_VERSION_NUMBER",
+]);
+
+export const peerAnsweredFromError = (error: unknown): boolean => {
+	const code = (error as NodeJS.ErrnoException | undefined)?.code;
+	return typeof code === "string" && PEER_ANSWERED_ERROR_CODES.has(code);
+};

server/src/types/network.ts

@@ -33,6 +33,12 @@ export interface MonitorStatusResponse<
 	status: boolean;
 	code: number;
 	message: string;
+	// Whether the peer answered, for failures where `code` alone cannot say. `code` carries an HTTP status for
+	// providers that speak HTTP, but DNS and gRPC report their own codes, and gRPC reuses NETWORK_ERROR for a
+	// server that replied NOT_SERVING. Providers that can tell the two apart set this; the rest leave it unset
+	// and the HTTP-status heuristic stands. Consumed by the egress check, which must only blame the instance's
+	// own connectivity when nothing answered.
+	peerResponded?: boolean;
 	responseTime?: number;
 	payload?: T | string | null;
 	timings?: GotTimings;

server/src/worker/worker.check-evaluator.ts

@@ -4,17 +4,52 @@ import { Check } from "@/domain/checks/check.type.js";
 import { Monitor } from "@/domain/monitors/monitor.type.js";
 import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
 import { IStatusService } from "@/service/statusService.js";
+import { MonitorActionDecision } from "@/worker/worker.helper.js";
+import { ILogger } from "@/utils/logger.js";
 
 export interface ICheckEvaluator {
 	evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
 }
 
+const SERVICE_NAME = "CheckEvaluator";
+
 export class CheckEvaluator implements ICheckEvaluator {
 	constructor(
 		private statusService: IStatusService,
-		private monitorStatusPolicy: IMonitorStatusPolicy
+		private monitorStatusPolicy: IMonitorStatusPolicy,
+		private logger: ILogger
 	) {}
+
+	// A check taken while the instance had no outbound connectivity says nothing about the target.
+	// It is left out of the status window, running stats and monitor status entirely, so that no incident
+	// opens for it and no spurious "resolved"/"up" fires once egress returns.
+	private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
+		this.logger.debug({
+			message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
+			service: SERVICE_NAME,
+			method: "evaluate",
+		});
+		const decision: MonitorActionDecision = {
+			shouldCreateIncident: false,
+			shouldResolveIncident: false,
+			shouldSendNotification: false,
+			incidentReason: null,
+			notificationReason: null,
+		};
+		return {
+			monitor,
+			status,
+			check,
+			statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
+			decision,
+		};
+	};
+
 	evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
+		if (check.egressStatus === "degraded") {
+			return this.skipDegradedEgressCheck(status, check, monitor);
+		}
+
 		// ****************************
 		// Step 3:  Evaluate and return result to reactors
 		// ****************************

server/src/worker/worker.check-producer.ts

@@ -11,6 +11,9 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
 import { IProxyResolver } from "@/service/network/ProxyResolver.js";
 import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
+import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
+import { isEgressAttributable } from "@/domain/monitors/monitor.type.js";
 
 export interface ICheckProducer {
 	produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -27,6 +30,7 @@ export class CheckProducer implements ICheckProducer {
 		private proxyResolver: IProxyResolver,
 		private bufferService: IBufferService,
 		private dockerLogsService: IDockerLogsService,
+		private egressService: IEgressService,
 		private logger: ILogger
 	) {}
 
@@ -42,6 +46,31 @@ export class CheckProducer implements ICheckProducer {
 		return dockerTlsKey ?? undefined;
 	}
 
+	// Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the
+	// two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx,
+	// or a 200 with a content mismatch).
+	private peerAnswered(status: MonitorStatusResponse): boolean {
+		return status.peerResponded ?? isHttpStatusCode(status.code);
+	}
+
+	private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
+		return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
+	}
+
+	// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
+	private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
+		try {
+			return await this.egressService.assessAfterFailure();
+		} catch (error: unknown) {
+			this.logger.warn({
+				message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
+				service: SERVICE_NAME,
+				method: "assessEgress",
+			});
+			return null;
+		}
+	}
+
 	produce = async (monitor: Monitor) => {
 		if (!monitor.id) {
 			throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
@@ -65,6 +94,13 @@ export class CheckProducer implements ICheckProducer {
 			return null;
 		}
 
+		// Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the
+		// degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is
+		// evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage.
+		if (monitor.status === "maintenance") {
+			monitor = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" });
+		}
+
 		// Step 1b: Acquire status
 		const proxyUrl = await this.proxyResolver.resolve(monitor);
 		const dockerTlsKey = await this.resolveDockerTlsKey(monitor);
@@ -78,7 +114,8 @@ export class CheckProducer implements ICheckProducer {
 		// Step 2: Record
 		// ****************************
 
-		// Step 2a:  Create & record a check, return null if fail
+		// Step 2a:  Create & record a check, return null if fail.
+		// Built before the egress assessment below so that the probe's duration stays out of `createdAt`.
 		const check = this.checkService.toCheck(status);
 		if (!check) {
 			this.logger.warn({
@@ -89,10 +126,26 @@ export class CheckProducer implements ICheckProducer {
 			});
 			return null;
 		}
-		// Step 2b: Add to buffer
+
+		// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
+		// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
+		const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
+		if (egressStatus !== null) {
+			check.egressStatus = egressStatus;
+		}
+
+		// Step 2c: Add to buffer
 		this.bufferService.addToBuffer(check);
 
-		// Step 2c: Handle docker logs
+		// Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops
+		// being evaluated, so the results either side of a spell never end up adjacent and cross the threshold
+		// on stale data. The length guard keeps this to one write per spell rather than one per degraded check,
+		// and it runs after the check is buffered so that a failed write cannot cost us the check.
+		if (egressStatus === "degraded" && monitor.statusWindow?.length) {
+			await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] });
+		}
+
+		// Step 2e: Handle docker logs
 		if (status.type === "docker") {
 			const dockerLogs = await this.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
 			for (const dockerLog of dockerLogs) {