Flag checks taken during degraded egress and exclude them from uptime (#3937)

CheckProducer asks EgressService to assess the instance's own connectivity
after a failing check and stamps the result on the check as egressStatus.
CheckEvaluator short-circuits degraded checks so no incident opens and no
spurious recovery fires later. Uptime and down-count aggregations exclude
degraded checks; the response-time series keeps them.

🔧

server/src/worker/worker.check-evaluator.ts

@@ -4,17 +4,52 @@ import { Check } from "@/domain/checks/check.type.js";
 import { Monitor } from "@/domain/monitors/monitor.type.js";
 import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
 import { IStatusService } from "@/service/statusService.js";
+import { MonitorActionDecision } from "@/worker/worker.helper.js";
+import { ILogger } from "@/utils/logger.js";
 
 export interface ICheckEvaluator {
 	evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
 }
 
+const SERVICE_NAME = "CheckEvaluator";
+
 export class CheckEvaluator implements ICheckEvaluator {
 	constructor(
 		private statusService: IStatusService,
-		private monitorStatusPolicy: IMonitorStatusPolicy
+		private monitorStatusPolicy: IMonitorStatusPolicy,
+		private logger: ILogger
 	) {}
+
+	// A check taken while the instance had no outbound connectivity says nothing about the target.
+	// It is left out of the status window, running stats and monitor status entirely, so that no incident
+	// opens for it and no spurious "resolved"/"up" fires once egress returns.
+	private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
+		this.logger.debug({
+			message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
+			service: SERVICE_NAME,
+			method: "evaluate",
+		});
+		const decision: MonitorActionDecision = {
+			shouldCreateIncident: false,
+			shouldResolveIncident: false,
+			shouldSendNotification: false,
+			incidentReason: null,
+			notificationReason: null,
+		};
+		return {
+			monitor,
+			status,
+			check,
+			statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
+			decision,
+		};
+	};
+
 	evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
+		if (check.egressStatus === "degraded") {
+			return this.skipDegradedEgressCheck(status, check, monitor);
+		}
+
 		// ****************************
 		// Step 3:  Evaluate and return result to reactors
 		// ****************************

server/src/worker/worker.check-producer.ts

@@ -11,6 +11,7 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
 import { IProxyResolver } from "@/service/network/ProxyResolver.js";
 import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
+import { IEgressService } from "@/domain/egress/egress.service.js";
 
 export interface ICheckProducer {
 	produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -27,6 +28,7 @@ export class CheckProducer implements ICheckProducer {
 		private proxyResolver: IProxyResolver,
 		private bufferService: IBufferService,
 		private dockerLogsService: IDockerLogsService,
+		private egressService: IEgressService,
 		private logger: ILogger
 	) {}
 
@@ -74,6 +76,10 @@ export class CheckProducer implements ICheckProducer {
 			throw new Error("No network response");
 		}
 
+		// Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target.
+		// Null means the egress check is disabled (or failed internally) and the check is treated as usual.
+		const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null;
+
 		// ****************************
 		// Step 2: Record
 		// ****************************
@@ -89,6 +95,9 @@ export class CheckProducer implements ICheckProducer {
 			});
 			return null;
 		}
+		if (egressStatus !== null) {
+			check.egressStatus = egressStatus;
+		}
 		// Step 2b: Add to buffer
 		this.bufferService.addToBuffer(check);