Flag checks taken during degraded egress and exclude them from uptime (#3937)

CheckProducer asks EgressService to assess the instance's own connectivity
after a failing check and stamps the result on the check as egressStatus.
CheckEvaluator short-circuits degraded checks so no incident opens and no
spurious recovery fires later. Uptime and down-count aggregations exclude
degraded checks; the response-time series keeps them.

🔧

server/src/config/services.worker.ts

@@ -129,9 +129,10 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
 		proxyResolver,
 		bufferService,
 		dockerLogsService,
+		egressService,
 		logger
 	);
-	const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
+	const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy, logger);
 	const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
 
 	// ***********************

server/src/domain/checks/check.docker.aggregation.ts

@@ -1,4 +1,4 @@
-import CheckModel from "@/domain/checks/check.model.js";
+import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
 import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
 import mongoose from "mongoose";
 
@@ -9,6 +9,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "docker",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	});
 
 export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {

server/src/domain/checks/check.hardware.aggregations.ts

@@ -1,4 +1,4 @@
-import { CheckModel } from "@/domain/checks/check.model.js";
+import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
 import mongoose from "mongoose";
 import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
 
@@ -70,6 +70,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
 		"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
 		"metadata.type": "hardware",
 		createdAt: { $gte: dates.start, $lte: dates.end },
+		...EXCLUDE_DEGRADED_EGRESS_MATCH,
 	});
 };
 

server/src/domain/checks/check.model.ts

@@ -1,5 +1,6 @@
 import { Schema, model, Types } from "mongoose";
 import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
+import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js";
 import type {
 	Check,
 	CheckAudits,
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
 			type: containerSummarySchema,
 			default: undefined,
 		},
+		egressStatus: {
+			type: String,
+			enum: EgressStatuses,
+			default: undefined,
+		},
 	},
 	{
 		timestamps: true,
@@ -359,6 +365,10 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
 
 const CheckModel = model<CheckDocument>("Check", CheckSchema);
 
+// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
+// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query.
+const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
+
 export type { CheckDocument, CheckMetadataDocument };
-export { CheckModel, containerSummarySchema };
+export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH };
 export default CheckModel;

server/src/domain/checks/check.repository.mongo.ts

@@ -15,7 +15,7 @@ import type {
 	HardwareCheckStats,
 } from "@/domain/checks/check.type.js";
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
-import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
+import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js";
 import mongoose from "mongoose";
 import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
 import { ILogger } from "@/utils/logger.js";
@@ -179,6 +179,7 @@ class MongoChecksRepository implements IChecksRepository {
 			audits: mapAudits(doc.audits),
 			containers: doc.containers,
 			containerSummary: doc.containerSummary,
+			...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }),
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
 		};
@@ -348,6 +349,7 @@ class MongoChecksRepository implements IChecksRepository {
 		const baseMatch = {
 			"metadata.teamId": new mongoose.Types.ObjectId(teamId),
 			createdAt: { $gte: getDateForRange(dateRange) },
+			...EXCLUDE_DEGRADED_EGRESS_MATCH,
 		};
 
 		const [totalResult, downResult] = await Promise.all([
@@ -393,6 +395,7 @@ class MongoChecksRepository implements IChecksRepository {
 				$match: {
 					"metadata.monitorId": { $in: objectIds },
 					createdAt: { $gte: windowStart },
+					...EXCLUDE_DEGRADED_EGRESS_MATCH,
 				},
 			},
 			{
@@ -480,6 +483,8 @@ class MongoChecksRepository implements IChecksRepository {
 			{
 				$facet: {
 					uptimePercentage: [
+						// Response-time series (groupedUpChecks / groupedDownChecks) deliberately keep degraded checks so they still show as failures on the graph.
+						{ $match: EXCLUDE_DEGRADED_EGRESS_MATCH },
 						{
 							$group: {
 								_id: null,

server/src/domain/checks/check.type.ts

@@ -1,6 +1,7 @@
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
 import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
 import type { Response } from "got";
+import type { EgressStatus } from "@/domain/egress/egress.type.js";
 
 export const CHECK_TTL_SENTINEL = 366;
 
@@ -116,6 +117,9 @@ export interface Check {
 	seo?: number;
 	performance?: number;
 	audits?: CheckAudits;
+	// Set only on checks produced while the reliability probe ran (a failing check).
+	// "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target.
+	egressStatus?: EgressStatus;
 	createdAt: string;
 	updatedAt: string;
 }

server/src/worker/worker.check-evaluator.ts

@@ -4,17 +4,52 @@ import { Check } from "@/domain/checks/check.type.js";
 import { Monitor } from "@/domain/monitors/monitor.type.js";
 import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
 import { IStatusService } from "@/service/statusService.js";
+import { MonitorActionDecision } from "@/worker/worker.helper.js";
+import { ILogger } from "@/utils/logger.js";
 
 export interface ICheckEvaluator {
 	evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
 }
 
+const SERVICE_NAME = "CheckEvaluator";
+
 export class CheckEvaluator implements ICheckEvaluator {
 	constructor(
 		private statusService: IStatusService,
-		private monitorStatusPolicy: IMonitorStatusPolicy
+		private monitorStatusPolicy: IMonitorStatusPolicy,
+		private logger: ILogger
 	) {}
+
+	// A check taken while the instance had no outbound connectivity says nothing about the target.
+	// It is left out of the status window, running stats and monitor status entirely, so that no incident
+	// opens for it and no spurious "resolved"/"up" fires once egress returns.
+	private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
+		this.logger.debug({
+			message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
+			service: SERVICE_NAME,
+			method: "evaluate",
+		});
+		const decision: MonitorActionDecision = {
+			shouldCreateIncident: false,
+			shouldResolveIncident: false,
+			shouldSendNotification: false,
+			incidentReason: null,
+			notificationReason: null,
+		};
+		return {
+			monitor,
+			status,
+			check,
+			statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
+			decision,
+		};
+	};
+
 	evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
+		if (check.egressStatus === "degraded") {
+			return this.skipDegradedEgressCheck(status, check, monitor);
+		}
+
 		// ****************************
 		// Step 3:  Evaluate and return result to reactors
 		// ****************************

server/src/worker/worker.check-producer.ts

@@ -11,6 +11,7 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
 import { IProxyResolver } from "@/service/network/ProxyResolver.js";
 import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
+import { IEgressService } from "@/domain/egress/egress.service.js";
 
 export interface ICheckProducer {
 	produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -27,6 +28,7 @@ export class CheckProducer implements ICheckProducer {
 		private proxyResolver: IProxyResolver,
 		private bufferService: IBufferService,
 		private dockerLogsService: IDockerLogsService,
+		private egressService: IEgressService,
 		private logger: ILogger
 	) {}
 
@@ -74,6 +76,10 @@ export class CheckProducer implements ICheckProducer {
 			throw new Error("No network response");
 		}
 
+		// Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target.
+		// Null means the egress check is disabled (or failed internally) and the check is treated as usual.
+		const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null;
+
 		// ****************************
 		// Step 2: Record
 		// ****************************
@@ -89,6 +95,9 @@ export class CheckProducer implements ICheckProducer {
 			});
 			return null;
 		}
+		if (egressStatus !== null) {
+			check.egressStatus = egressStatus;
+		}
 		// Step 2b: Add to buffer
 		this.bufferService.addToBuffer(check);