Address review of degraded-egress check persistence

CheckProducer consults egress only for a transport failure: the check
failed, no HTTP status came back (isHttpStatusCode from PR 1), and the
monitor type is one whose failure can be a loss of egress
(EgressAttributableTypes; hardware and docker are excluded). A 4xx/5xx
or a content mismatch is evaluated as usual while the instance is
degraded. A rejected assessment is logged and treated as unknown so the
check is still buffered.

EXCLUDE_DEGRADED_EGRESS_MATCH moves out of the model into
check.query.ts, with an expression form for $group accumulators.
Daily status buckets and docker stats buckets exclude degraded checks
from their up/total counts while keeping them in the response-time
average, matching the uptime facet. toEntity maps egressStatus like its
siblings.

Adds heartbeatJobDegradedEgress integration tests, extends the
repository exclusion tests, and makes the harness egress stub settable.

🔧

server/src/domain/checks/check.docker.aggregation.ts

@@ -1,5 +1,6 @@
-import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
+import CheckModel from "@/domain/checks/check.model.js";
 import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
+import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
 import mongoose from "mongoose";
 
 type DateRange = { start: Date; end: Date };
@@ -35,8 +36,9 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
 			$group: {
 				_id: { $dateToString: { format: dateString, date: "$createdAt" } },
 				avgResponseTime: { $avg: "$responseTime" },
-				upCount: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
-				totalCount: { $sum: 1 },
+				// Counts exclude degraded-egress checks like aggregateData.totalChecks does; the response-time average keeps them.
+				upCount: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
+				totalCount: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
 				avgRunning: { $avg: "$containerSummary.running" },
 				avgTotal: { $avg: "$containerSummary.total" },
 				avgUnhealthy: { $avg: "$containerSummary.unhealthy" },

server/src/domain/checks/check.hardware.aggregations.ts

@@ -1,6 +1,7 @@
-import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
+import { CheckModel } from "@/domain/checks/check.model.js";
 import mongoose from "mongoose";
 import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
+import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
 
 type DateRange = { start: Date; end: Date };
 type HardwareUpChecks = { totalChecks: number };

server/src/domain/checks/check.model.ts

@@ -1,6 +1,6 @@
 import { Schema, model, Types } from "mongoose";
 import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
-import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js";
+import { EgressStatuses } from "@/domain/egress/egress.type.js";
 import type {
 	Check,
 	CheckAudits,
@@ -365,10 +365,6 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
 
 const CheckModel = model<CheckDocument>("Check", CheckSchema);
 
-// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
-// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query.
-const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
-
 export type { CheckDocument, CheckMetadataDocument };
-export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH };
+export { CheckModel, containerSummarySchema };
 export default CheckModel;

server/src/domain/checks/check.query.ts

@@ -0,0 +1,8 @@
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
+
+// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
+// are left out of every uptime percentage and up/down count. Response-time series keep them.
+// Spread this into the $match / filter of a query, or use the expression form inside a $group accumulator.
+export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
+
+export const IS_NOT_DEGRADED_EGRESS_EXPR = { $ne: ["$egressStatus", "degraded" satisfies EgressStatus] } as const;

server/src/domain/checks/check.repository.mongo.ts

@@ -15,7 +15,8 @@ import type {
 	HardwareCheckStats,
 } from "@/domain/checks/check.type.js";
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
-import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js";
+import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
+import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
 import mongoose from "mongoose";
 import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
 import { ILogger } from "@/utils/logger.js";
@@ -179,7 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
 			audits: mapAudits(doc.audits),
 			containers: doc.containers,
 			containerSummary: doc.containerSummary,
-			...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }),
+			egressStatus: doc.egressStatus,
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
 		};
@@ -395,7 +396,6 @@ class MongoChecksRepository implements IChecksRepository {
 				$match: {
 					"metadata.monitorId": { $in: objectIds },
 					createdAt: { $gte: windowStart },
-					...EXCLUDE_DEGRADED_EGRESS_MATCH,
 				},
 			},
 			{
@@ -404,11 +404,14 @@ class MongoChecksRepository implements IChecksRepository {
 						monitorId: "$metadata.monitorId",
 						day: { $dateTrunc: { date: "$createdAt", unit: "day", timezone } },
 					},
-					totalChecks: { $sum: 1 },
-					upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
+					// Counts exclude degraded-egress checks; the response-time average keeps them.
+					totalChecks: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
+					upChecks: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
 					avgResponseTime: { $avg: "$responseTime" },
 				},
 			},
+			// A day with nothing attributable to the target renders as an empty bar; the client divides by totalChecks for the tooltip.
+			{ $match: { totalChecks: { $gt: 0 } } },
 			{ $sort: { "_id.day": 1 } },
 			{
 				$project: {

server/src/domain/checks/check.type.ts

@@ -1,7 +1,7 @@
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
 import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
 import type { Response } from "got";
-import type { EgressStatus } from "@/domain/egress/egress.type.js";
 
 export const CHECK_TTL_SENTINEL = 366;
 
@@ -117,8 +117,9 @@ export interface Check {
 	seo?: number;
 	performance?: number;
 	audits?: CheckAudits;
-	// Set only on checks produced while the reliability probe ran (a failing check).
-	// "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target.
+	// Set on failing checks while the egress check is enabled; absent otherwise.
+	// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
+	// "ok": a reliability target was reachable.
 	egressStatus?: EgressStatus;
 	createdAt: string;
 	updatedAt: string;

server/src/domain/egress/egress.service.ts

@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
 const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
 
 // HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
-const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
+export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
 
 // Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
 const PROBE_MONITOR_ID = "egress-probe";

server/src/domain/monitors/monitor.type.ts

@@ -50,6 +50,21 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
 export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
 export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
 
+// Types whose transport failure can be the instance's own loss of egress. Hardware (Capture agent) and docker
+// (local socket or host) failures are local, so they are never attributed to egress.
+export const EgressAttributableTypes = [
+	"http",
+	"ping",
+	"pagespeed",
+	"port",
+	"game",
+	"grpc",
+	"websocket",
+	"dns",
+] as const satisfies readonly MonitorType[];
+export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
+export const isEgressAttributable = (type: MonitorType): type is EgressAttributableType => EgressAttributableTypes.some((t) => t === type);
+
 export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
 export type MonitorStatus = (typeof MonitorStatuses)[number];
 

server/src/worker/worker.check-producer.ts

@@ -11,7 +11,9 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
 import { IProxyResolver } from "@/service/network/ProxyResolver.js";
 import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
-import { IEgressService } from "@/domain/egress/egress.service.js";
+import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
+import { isEgressAttributable } from "@/domain/monitors/monitor.type.js";
 
 export interface ICheckProducer {
 	produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -44,6 +46,26 @@ export class CheckProducer implements ICheckProducer {
 		return dockerTlsKey ?? undefined;
 	}
 
+	// Only a failure to reach the target at all can be the instance's own fault. Any HTTP response (4xx, 5xx,
+	// or a 200 with a content mismatch) proves the target was reached, and hardware/docker failures are local.
+	private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
+		return status.status === false && isEgressAttributable(monitor.type) && !isHttpStatusCode(status.code);
+	}
+
+	// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
+	private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
+		try {
+			return await this.egressService.assessAfterFailure();
+		} catch (error: unknown) {
+			this.logger.warn({
+				message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
+				service: SERVICE_NAME,
+				method: "assessEgress",
+			});
+			return null;
+		}
+	}
+
 	produce = async (monitor: Monitor) => {
 		if (!monitor.id) {
 			throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
@@ -76,9 +98,9 @@ export class CheckProducer implements ICheckProducer {
 			throw new Error("No network response");
 		}
 
-		// Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target.
+		// Step 1c: On a transport failure, ask whether the instance itself can reach anything before blaming the target.
 		// Null means the egress check is disabled (or failed internally) and the check is treated as usual.
-		const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null;
+		const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
 
 		// ****************************
 		// Step 2: Record

server/test/helpers/heartbeatTestHarness.ts

@@ -14,6 +14,7 @@ import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
 import type { MonitorStatusResponse } from "../../src/types/network.ts";
 import type { Check } from "../../src/domain/checks/check.type.ts";
 import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts";
+import type { EgressStatus } from "../../src/domain/egress/egress.type.ts";
 
 let checkCounter = 0;
 
@@ -79,9 +80,11 @@ export interface HeartbeatTestHarness {
 	bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock };
 	maintenanceWindowsRepo: { findByMonitorId: jest.Mock };
 	messageBuilder: { extractThresholdBreaches: jest.Mock };
+	egressService: { assessAfterFailure: jest.Mock };
 	heartbeatJob: (monitor: Monitor) => Promise<void>;
 	setNextResponse: (status: boolean, code: number) => void;
 	setNextResponseFull: (response: MonitorStatusResponse) => void;
+	setEgressStatus: (status: EgressStatus | null) => void;
 }
 
 export function createHeartbeatTestHarness(): HeartbeatTestHarness {
@@ -129,7 +132,12 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
 	const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
 	const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
 	const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
-	const egressService = { assessAfterFailure: jest.fn().mockResolvedValue(null) };
+	// Null mirrors the egress check being disabled, so the existing heartbeat suites run unchanged.
+	let nextEgressStatus: EgressStatus | null = null;
+	const egressService = { assessAfterFailure: jest.fn().mockImplementation(() => Promise.resolve(nextEgressStatus)) };
+	const setEgressStatus = (status: EgressStatus | null) => {
+		nextEgressStatus = status;
+	};
 
 	const checkProducer = new CheckProducer(
 		monitorsRepo as any,
@@ -169,8 +177,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
 		bufferStub,
 		maintenanceWindowsRepo,
 		messageBuilder,
+		egressService,
 		heartbeatJob,
 		setNextResponse,
 		setNextResponseFull,
+		setEgressStatus,
 	};
 }

server/test/integration/checksRepositoryEgressExclusion.test.ts

@@ -78,7 +78,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
 
 		// The evaluator runs from this read, so the flag must survive persistence for the short-circuit to fire.
 		expect(checks.map((check) => check.egressStatus)).toEqual(["degraded", "ok", undefined]);
-		expect(checks[2]).not.toHaveProperty("egressStatus");
+		expect(checks[2].egressStatus).toBeUndefined();
 	});
 
 	it("rejects a value outside the EgressStatuses tuple", async () => {
@@ -95,8 +95,9 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
 
 		// 1 up of 2 attributable checks. Counting the degraded pair would give 25%.
 		expect(result.uptimePercentage).toBe(0.5);
-		// The graph still shows the degraded checks as failures.
-		expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4 });
+		// The graph still shows the degraded checks as failures, and their response times stay in the averages.
+		expect(result.avgResponseTime).toBe(200);
+		expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4, avgResponseTime: 200 });
 		expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 3 });
 		expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 });
 	});
@@ -120,7 +121,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
 		expect(summary).toEqual({ totalChecks: 2, downChecks: 1 });
 	});
 
-	it("excludes degraded checks from the daily status buckets and yields no row for a day of only degraded checks", async () => {
+	it("excludes degraded checks from the daily status bucket counts but keeps them in its response-time average", async () => {
 		const otherMonitor = new mongoose.Types.ObjectId();
 		await seedCheck();
 		await seedCheck({ status: false });
@@ -129,8 +130,10 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
 
 		const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC");
 
+		// (100 + 100 + 300) / 3 = 167: the degraded check's response time counts, its failure does not.
+		// The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket.
 		expect(buckets).toEqual([
-			{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 },
+			{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 167 },
 		]);
 	});
 
@@ -154,6 +157,19 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
 		expect(docker.upChecks.totalChecks).toBe(1);
 	});
 
+	it("excludes degraded checks from the docker bucket counts but keeps them in its response-time average", async () => {
+		const dockerMonitor = new mongoose.Types.ObjectId();
+		const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
+
+		await seedCheck({ metadata: dockerMeta });
+		await seedDegradedFailure({ metadata: dockerMeta });
+
+		const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
+
+		expect(docker.aggregate).toHaveLength(1);
+		expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1, avgResponseTime: 200 });
+	});
+
 	it("keeps degraded checks in the paginated listing so they can be shown as such", async () => {
 		await seedCheck({ status: false });
 		await seedDegradedFailure();

server/test/integration/heartbeatJobDegradedEgress.test.ts

@@ -0,0 +1,160 @@
+import { describe, expect, it, beforeEach } from "@jest/globals";
+import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
+import { NETWORK_ERROR } from "../../src/types/network.ts";
+
+// Egress is only consulted for transport failures, so these heartbeats fail with the network-error
+// sentinel (timeout / connection refused) rather than an HTTP status code.
+
+describe("Heartbeat job: degraded egress", () => {
+	let h: HeartbeatTestHarness;
+
+	beforeEach(() => {
+		h = createHeartbeatTestHarness();
+	});
+
+	it("records failing checks as degraded without changing monitor status, opening an incident or notifying", async () => {
+		const monitor = makeMonitor();
+		h.monitorsRepo.seed(monitor);
+
+		// Enough failures to cross the 60% threshold if they were counted
+		h.setEgressStatus("degraded");
+		h.setNextResponse(false, NETWORK_ERROR);
+		for (let i = 0; i < 3; i++) {
+			await h.heartbeatJob(monitor);
+		}
+
+		// Every check was buffered, flagged, so the repository can exclude it from uptime
+		expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
+		expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(3);
+		for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
+			expect(check).toMatchObject({ status: false, egressStatus: "degraded" });
+		}
+
+		// Nothing downstream moved
+		const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
+		expect(storedMonitor.status).toBe("up");
+		expect(storedMonitor.statusWindow).toEqual([true, true, true, true, true]);
+		expect(h.incidentsRepo.getAll()).toHaveLength(0);
+		expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
+	});
+
+	it("does not fire a spurious recovery once egress returns", async () => {
+		const monitor = makeMonitor();
+		h.monitorsRepo.seed(monitor);
+
+		h.setEgressStatus("degraded");
+		h.setNextResponse(false, NETWORK_ERROR);
+		for (let i = 0; i < 3; i++) {
+			await h.heartbeatJob(monitor);
+		}
+
+		// Egress is back and the target is fine
+		h.setEgressStatus(null);
+		h.setNextResponse(true, 200);
+		await h.heartbeatJob(monitor);
+
+		expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
+		expect(h.incidentsRepo.getAll()).toHaveLength(0);
+		expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
+	});
+
+	it("detects down at the normal threshold when the egress check is disabled", async () => {
+		const monitor = makeMonitor();
+		h.monitorsRepo.seed(monitor);
+
+		h.setEgressStatus(null);
+		h.setNextResponse(false, NETWORK_ERROR);
+		for (let i = 0; i < 3; i++) {
+			await h.heartbeatJob(monitor);
+		}
+
+		expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
+		for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
+			expect(check).not.toHaveProperty("egressStatus");
+		}
+		expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
+		expect(h.incidentsRepo.getAll()).toHaveLength(1);
+		expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
+	});
+
+	it("detects down at the normal threshold when egress is confirmed ok", async () => {
+		const monitor = makeMonitor();
+		h.monitorsRepo.seed(monitor);
+
+		h.setEgressStatus("ok");
+		h.setNextResponse(false, NETWORK_ERROR);
+		for (let i = 0; i < 3; i++) {
+			await h.heartbeatJob(monitor);
+		}
+
+		for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
+			expect(check).toMatchObject({ status: false, egressStatus: "ok" });
+		}
+		expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
+		expect(h.incidentsRepo.getAll()).toHaveLength(1);
+		expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
+	});
+
+	it("evaluates HTTP error responses normally while egress is degraded", async () => {
+		const monitor = makeMonitor();
+		h.monitorsRepo.seed(monitor);
+
+		// The target answered, so the failure is its own even though the instance is degraded
+		h.setEgressStatus("degraded");
+		h.setNextResponse(false, 503);
+		for (let i = 0; i < 3; i++) {
+			await h.heartbeatJob(monitor);
+		}
+
+		expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
+		for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
+			expect(check).not.toHaveProperty("egressStatus");
+		}
+		expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
+		expect(h.incidentsRepo.getAll()).toHaveLength(1);
+		expect(h.incidentsRepo.getAll()[0].statusCode).toBe(503);
+		expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
+	});
+
+	it("evaluates a failing docker check normally while egress is degraded", async () => {
+		const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" });
+		h.monitorsRepo.seed(monitor);
+
+		h.setEgressStatus("degraded");
+		h.setNextResponse(false, NETWORK_ERROR);
+		for (let i = 0; i < 3; i++) {
+			await h.heartbeatJob(monitor);
+		}
+
+		expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
+		for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
+			expect(check).not.toHaveProperty("egressStatus");
+		}
+		expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
+		expect(h.incidentsRepo.getAll()).toHaveLength(1);
+	});
+
+	it("keeps counting real failures taken either side of a degraded spell", async () => {
+		const monitor = makeMonitor();
+		h.monitorsRepo.seed(monitor);
+
+		// 2 real failures: window [t, t, t, f, f] = 40% < 60%
+		h.setNextResponse(false, NETWORK_ERROR);
+		await h.heartbeatJob(monitor);
+		await h.heartbeatJob(monitor);
+
+		// A degraded spell must not push the window over the threshold
+		h.setEgressStatus("degraded");
+		await h.heartbeatJob(monitor);
+		await h.heartbeatJob(monitor);
+		expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
+		expect(h.incidentsRepo.getAll()).toHaveLength(0);
+
+		// 3rd real failure: window [t, t, f, f, f] = 60% >= 60%
+		h.setEgressStatus(null);
+		await h.heartbeatJob(monitor);
+
+		expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
+		expect(h.incidentsRepo.getAll()).toHaveLength(1);
+	});
+});

server/test/unit/services/checkProducer.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
 import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
 import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
 import { createMockLogger } from "../../helpers/createMockLogger.ts";
+import { NETWORK_ERROR } from "../../../src/types/network.ts";
 
 const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
 	({
@@ -215,7 +216,8 @@ describe("CheckProducer", () => {
 
 	// ── egress self-check ─────────────────────────────────────────────────────
 
-	const failingStatus = { monitorId: "m1", status: false, code: 500, message: "Error" };
+	// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
+	const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
 
 	it("does not consult the egress service for a successful check and leaves the field unset", async () => {
 		const check: Record<string, unknown> = { id: "check-1" };
@@ -229,6 +231,57 @@ describe("CheckProducer", () => {
 		expect(check).not.toHaveProperty("egressStatus");
 	});
 
+	it("does not consult egress when an HTTP response was received", async () => {
+		for (const status of [
+			{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
+			{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
+		]) {
+			const check: Record<string, unknown> = { id: "check-1" };
+			const { producer, defaults } = createProducer({
+				networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
+				checkService: { toCheck: jest.fn().mockReturnValue(check) },
+				egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
+			});
+
+			await producer.produce(makeMonitor());
+
+			expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
+			expect(check).not.toHaveProperty("egressStatus");
+			expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
+		}
+	});
+
+	it("consults egress for a timeout or network error", async () => {
+		const check: Record<string, unknown> = { id: "check-1" };
+		const { producer, defaults } = createProducer({
+			networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
+			checkService: { toCheck: jest.fn().mockReturnValue(check) },
+			egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
+		});
+
+		await producer.produce(makeMonitor());
+
+		expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
+		expect(check.egressStatus).toBe("degraded");
+	});
+
+	it("never consults egress for a failing docker or hardware check", async () => {
+		for (const type of ["docker", "hardware"] as const) {
+			const check: Record<string, unknown> = { id: "check-1" };
+			const { producer, defaults } = createProducer({
+				networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type }) },
+				checkService: { toCheck: jest.fn().mockReturnValue(check) },
+				egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
+			});
+
+			await producer.produce(makeMonitor({ type }));
+
+			expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
+			expect(check).not.toHaveProperty("egressStatus");
+			expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
+		}
+	});
+
 	it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
 		const check: Record<string, unknown> = { id: "check-1" };
 		const { producer, defaults } = createProducer({
@@ -258,6 +311,24 @@ describe("CheckProducer", () => {
 		expect(check.egressStatus).toBe("ok");
 	});
 
+	it("buffers the check when the egress assessment rejects", async () => {
+		const check: Record<string, unknown> = { id: "check-1" };
+		const { producer, defaults } = createProducer({
+			networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
+			checkService: { toCheck: jest.fn().mockReturnValue(check) },
+			egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) },
+		});
+
+		const result = await producer.produce(makeMonitor());
+
+		expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
+		expect(check).not.toHaveProperty("egressStatus");
+		expect(result).toEqual({ status: failingStatus, check });
+		expect(defaults.logger.warn).toHaveBeenCalledWith(
+			expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" })
+		);
+	});
+
 	it("leaves the field unset on a failing check when the egress check is disabled", async () => {
 		const check: Record<string, unknown> = { id: "check-1" };
 		const { producer, defaults } = createProducer({