Flag checks taken during degraded egress and exclude them from uptime (#3937)

CheckProducer asks EgressService to assess the instance's own connectivity
after a failing check and stamps the result on the check as egressStatus.
CheckEvaluator short-circuits degraded checks so no incident opens and no
spurious recovery fires later. Uptime and down-count aggregations exclude
degraded checks; the response-time series keeps them.

🔧

server/src/config/services.worker.ts

@@ -129,9 +129,10 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
 		proxyResolver,
 		bufferService,
 		dockerLogsService,
+		egressService,
 		logger
 	);
-	const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
+	const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy, logger);
 	const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
 
 	// ***********************

server/src/domain/checks/check.docker.aggregation.ts

@@ -1,4 +1,4 @@
-import CheckModel from "@/domain/checks/check.model.js";
+import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
 import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
 import mongoose from "mongoose";
 
@@ -9,6 +9,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "docker",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	});
 
 export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {

server/src/domain/checks/check.hardware.aggregations.ts

@@ -1,4 +1,4 @@
-import { CheckModel } from "@/domain/checks/check.model.js";
+import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
 import mongoose from "mongoose";
 import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
 
@@ -70,6 +70,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "hardware",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	});
 };
 

server/src/domain/checks/check.model.ts

@@ -1,5 +1,6 @@
 import { Schema, model, Types } from "mongoose";
 import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
+import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js";
 import type {
 	Check,
 	CheckAudits,
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
 			type: containerSummarySchema,
 			default: undefined,
 		},
+		egressStatus: {
+			type: String,
+			enum: EgressStatuses,
+			default: undefined,
+		},
 	},
 	{
 		timestamps: true,
@@ -359,6 +365,10 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
 
 const CheckModel = model<CheckDocument>("Check", CheckSchema);
 
+// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
+// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query.
+const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
+
 export type { CheckDocument, CheckMetadataDocument };
-export { CheckModel, containerSummarySchema };
+export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH };
 export default CheckModel;

server/src/domain/checks/check.repository.mongo.ts

@@ -15,7 +15,7 @@ import type {
 	HardwareCheckStats,
 } from "@/domain/checks/check.type.js";
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
-import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
+import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js";
 import mongoose from "mongoose";
 import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
 import { ILogger } from "@/utils/logger.js";
@@ -179,6 +179,7 @@ class MongoChecksRepository implements IChecksRepository {
 			audits: mapAudits(doc.audits),
 			containers: doc.containers,
 			containerSummary: doc.containerSummary,
+			...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }),
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
 		};
@@ -348,6 +349,7 @@ class MongoChecksRepository implements IChecksRepository {
 		const baseMatch = {
 			"metadata.teamId": new mongoose.Types.ObjectId(teamId),
 			createdAt: { $gte: getDateForRange(dateRange) },
+			...EXCLUDE_DEGRADED_EGRESS_MATCH,
 		};
 
 		const [totalResult, downResult] = await Promise.all([
@@ -393,6 +395,7 @@ class MongoChecksRepository implements IChecksRepository {
 				$match: {
 					"metadata.monitorId": { $in: objectIds },
 					createdAt: { $gte: windowStart },
+					...EXCLUDE_DEGRADED_EGRESS_MATCH,
 				},
 			},
 			{
@@ -480,6 +483,8 @@ class MongoChecksRepository implements IChecksRepository {
 			{
 				$facet: {
 					uptimePercentage: [
+						// Response-time series (groupedUpChecks / groupedDownChecks) deliberately keep degraded checks so they still show as failures on the graph.
+						{ $match: EXCLUDE_DEGRADED_EGRESS_MATCH },
 						{
 							$group: {
 								_id: null,

server/src/domain/checks/check.type.ts

@@ -1,6 +1,7 @@
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
 import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
 import type { Response } from "got";
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
 
 export const CHECK_TTL_SENTINEL = 366;
 
@@ -116,6 +117,9 @@ export interface Check {
 	seo?: number;
 	performance?: number;
 	audits?: CheckAudits;
+	// Set only on checks produced while the reliability probe ran (a failing check).
+	// "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target.
+	egressStatus?: EgressStatus;
 	createdAt: string;
 	updatedAt: string;
 }

server/src/worker/worker.check-evaluator.ts

@@ -4,17 +4,52 @@ import { Check } from "@/domain/checks/check.type.js";
 import { Monitor } from "@/domain/monitors/monitor.type.js";
 import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
 import { IStatusService } from "@/service/statusService.js";
+import { MonitorActionDecision } from "@/worker/worker.helper.js";
+import { ILogger } from "@/utils/logger.js";
 
 export interface ICheckEvaluator {
 	evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
 }
 
+const SERVICE_NAME = "CheckEvaluator";
+
 export class CheckEvaluator implements ICheckEvaluator {
 	constructor(
 		private statusService: IStatusService,
-		private monitorStatusPolicy: IMonitorStatusPolicy
+		private monitorStatusPolicy: IMonitorStatusPolicy,
+		private logger: ILogger
 	) {}
+
+	// A check taken while the instance had no outbound connectivity says nothing about the target.
+	// It is left out of the status window, running stats and monitor status entirely, so that no incident
+	// opens for it and no spurious "resolved"/"up" fires once egress returns.
+	private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
+		this.logger.debug({
+			message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
+			service: SERVICE_NAME,
+			method: "evaluate",
+		});
+		const decision: MonitorActionDecision = {
+			shouldCreateIncident: false,
+			shouldResolveIncident: false,
+			shouldSendNotification: false,
+			incidentReason: null,
+			notificationReason: null,
+		};
+		return {
+			monitor,
+			status,
+			check,
+			statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
+			decision,
+		};
+	};
+
 	evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
+		if (check.egressStatus === "degraded") {
+			return this.skipDegradedEgressCheck(status, check, monitor);
+		}
+
 		// ****************************
 		// Step 3:  Evaluate and return result to reactors
 		// ****************************

server/src/worker/worker.check-producer.ts

@@ -11,6 +11,7 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
 import { IProxyResolver } from "@/service/network/ProxyResolver.js";
 import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
+import { IEgressService } from "@/domain/egress/egress.service.js";
 
 export interface ICheckProducer {
 	produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -27,6 +28,7 @@ export class CheckProducer implements ICheckProducer {
 		private proxyResolver: IProxyResolver,
 		private bufferService: IBufferService,
 		private dockerLogsService: IDockerLogsService,
+		private egressService: IEgressService,
 		private logger: ILogger
 	) {}
 
@@ -74,6 +76,10 @@ export class CheckProducer implements ICheckProducer {
 			throw new Error("No network response");
 		}
 
+		// Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target.
+		// Null means the egress check is disabled (or failed internally) and the check is treated as usual.
+		const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null;
+
 		// ****************************
 		// Step 2: Record
 		// ****************************
@@ -89,6 +95,9 @@ export class CheckProducer implements ICheckProducer {
 			});
 			return null;
 		}
+		if (egressStatus !== null) {
+			check.egressStatus = egressStatus;
+		}
 		// Step 2b: Add to buffer
 		this.bufferService.addToBuffer(check);
 

server/test/helpers/heartbeatTestHarness.ts

@@ -129,6 +129,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
 	const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
 	const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
 	const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
+	const egressService = { assessAfterFailure: jest.fn().mockResolvedValue(null) };
 
 	const checkProducer = new CheckProducer(
 		monitorsRepo as any,
@@ -138,9 +139,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
 		proxyResolver as any,
 		bufferStub as any,
 		dockerLogsService as any,
+		egressService as any,
 		logger
 	);
-	const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
+	const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger);
 
 	const notificationReactor = new NotificationReactor(notificationsService as any);
 	const incidentReactor = new IncidentReactor(incidentService as any);

server/test/unit/services/checkEvaluator.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
 import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
 import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
 import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
+import { createMockLogger } from "../../helpers/createMockLogger.ts";
 
 const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
 	({
@@ -19,9 +20,10 @@ const createEvaluator = (overrides?: Record<string, any>) => {
 			updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
 		},
 		monitorStatusPolicy: new MonitorStatusPolicy(),
+		logger: createMockLogger(),
 		...overrides,
 	};
-	const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
+	const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
 	return { evaluator, defaults };
 };
 
@@ -84,4 +86,38 @@ describe("CheckEvaluator", () => {
 			shouldSendNotification: false,
 		});
 	});
+
+	// ── degraded egress ───────────────────────────────────────────────────────
+
+	it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
+		const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
+		const check = { id: "check-1", egressStatus: "degraded" } as any;
+		const monitor = makeMonitor({ status: "up" });
+		const { evaluator, defaults } = createEvaluator();
+
+		const result = await evaluator.evaluate(status, check, monitor);
+
+		expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
+		expect(result.monitor).toBe(monitor);
+		expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
+		expect(result.statusChange.timestamp).toEqual(expect.any(Number));
+		expect(result.decision).toEqual({
+			shouldCreateIncident: false,
+			shouldResolveIncident: false,
+			shouldSendNotification: false,
+			incidentReason: null,
+			notificationReason: null,
+		});
+		expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
+	});
+
+	it("evaluates a check flagged with egress ok as normal", async () => {
+		const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
+		const check = { id: "check-1", egressStatus: "ok" } as any;
+		const { evaluator, defaults } = createEvaluator();
+
+		await evaluator.evaluate(status, check, makeMonitor());
+
+		expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
+	});
 });

server/test/unit/services/checkProducer.test.ts

@@ -29,6 +29,7 @@ const createProducer = (overrides?: Record<string, any>) => {
 		proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
 		buffer: { addToBuffer: jest.fn() },
 		dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
+		egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
 		...overrides,
 	};
 	const producer = new CheckProducer(
@@ -39,6 +40,7 @@ const createProducer = (overrides?: Record<string, any>) => {
 		defaults.proxyResolver as any,
 		defaults.buffer as any,
 		defaults.dockerLogsService as any,
+		defaults.egressService as any,
 		defaults.logger as any
 	);
 	return { producer, defaults };
@@ -210,4 +212,62 @@ describe("CheckProducer", () => {
 
 		expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
 	});
+
+	// ── egress self-check ─────────────────────────────────────────────────────
+
+	const failingStatus = { monitorId: "m1", status: false, code: 500, message: "Error" };
+
+	it("does not consult the egress service for a successful check and leaves the field unset", async () => {
+		const check: Record<string, unknown> = { id: "check-1" };
+		const { producer, defaults } = createProducer({
+			checkService: { toCheck: jest.fn().mockReturnValue(check) },
+		});
+
+		await producer.produce(makeMonitor());
+
+		expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
+		expect(check).not.toHaveProperty("egressStatus");
+	});
+
+	it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
+		const check: Record<string, unknown> = { id: "check-1" };
+		const { producer, defaults } = createProducer({
+			networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
+			checkService: { toCheck: jest.fn().mockReturnValue(check) },
+			egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
+		});
+
+		const result = await producer.produce(makeMonitor());
+
+		expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
+		expect(check.egressStatus).toBe("degraded");
+		expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
+		expect(result?.check.egressStatus).toBe("degraded");
+	});
+
+	it("flags a failing check as ok when the probe found egress fine", async () => {
+		const check: Record<string, unknown> = { id: "check-1" };
+		const { producer } = createProducer({
+			networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
+			checkService: { toCheck: jest.fn().mockReturnValue(check) },
+			egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
+		});
+
+		await producer.produce(makeMonitor());
+
+		expect(check.egressStatus).toBe("ok");
+	});
+
+	it("leaves the field unset on a failing check when the egress check is disabled", async () => {
+		const check: Record<string, unknown> = { id: "check-1" };
+		const { producer, defaults } = createProducer({
+			networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
+			checkService: { toCheck: jest.fn().mockReturnValue(check) },
+		});
+
+		await producer.produce(makeMonitor());
+
+		expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
+		expect(check).not.toHaveProperty("egressStatus");
+	});
 });