Address review of degraded-egress check persistence

CheckProducer consults egress only for a transport failure: the check
failed, no HTTP status came back (isHttpStatusCode from PR 1), and the
monitor type is one whose failure can be a loss of egress
(EgressAttributableTypes; hardware and docker are excluded). A 4xx/5xx
or a content mismatch is evaluated as usual while the instance is
degraded. A rejected assessment is logged and treated as unknown so the
check is still buffered.

EXCLUDE_DEGRADED_EGRESS_MATCH moves out of the model into
check.query.ts, with an expression form for $group accumulators.
Daily status buckets and docker stats buckets exclude degraded checks
from their up/total counts while keeping them in the response-time
average, matching the uptime facet. toEntity maps egressStatus like its
siblings.

Adds heartbeatJobDegradedEgress integration tests, extends the
repository exclusion tests, and makes the harness egress stub settable.

🔧
This commit is contained in:
Craig Lawson committed 2026-09-17 22:06:16 +01:00
1 parent 4f5dba560d
commit 4335e6b7b6
13 files changed
+334 -29

No files matched your search

@@ -1,5 +1,6 @@
import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
import CheckModel from "@/domain/checks/check.model.js";
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
type DateRange = { start: Date; end: Date };
@@ -35,8 +36,9 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
$group: {
_id: { $dateToString: { format: dateString, date: "$createdAt" } },
avgResponseTime: { $avg: "$responseTime" },
upCount: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
totalCount: { $sum: 1 },
// Counts exclude degraded-egress checks like aggregateData.totalChecks does; the response-time average keeps them.
upCount: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
totalCount: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
avgRunning: { $avg: "$containerSummary.running" },
avgTotal: { $avg: "$containerSummary.total" },
avgUnhealthy: { $avg: "$containerSummary.unhealthy" },
@@ -1,6 +1,7 @@
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
import { CheckModel } from "@/domain/checks/check.model.js";
import mongoose from "mongoose";
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
type DateRange = { start: Date; end: Date };
type HardwareUpChecks = { totalChecks: number };
+2 -6
View File
@@ -1,6 +1,6 @@
import { Schema, model, Types } from "mongoose";
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js";
import { EgressStatuses } from "@/domain/egress/egress.type.js";
import type {
Check,
CheckAudits,
@@ -365,10 +365,6 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
const CheckModel = model<CheckDocument>("Check", CheckSchema);
// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query.
const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
export type { CheckDocument, CheckMetadataDocument };
export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH };
export { CheckModel, containerSummarySchema };
export default CheckModel;
+8
View File
@@ -0,0 +1,8 @@
import type { EgressStatus } from "@/domain/egress/egress.type.js";
// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
// are left out of every uptime percentage and up/down count. Response-time series keep them.
// Spread this into the $match / filter of a query, or use the expression form inside a $group accumulator.
export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
export const IS_NOT_DEGRADED_EGRESS_EXPR = { $ne: ["$egressStatus", "degraded" satisfies EgressStatus] } as const;
@@ -15,7 +15,8 @@ import type {
HardwareCheckStats,
} from "@/domain/checks/check.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js";
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
import { ILogger } from "@/utils/logger.js";
@@ -179,7 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
audits: mapAudits(doc.audits),
containers: doc.containers,
containerSummary: doc.containerSummary,
...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }),
egressStatus: doc.egressStatus,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
};
@@ -395,7 +396,6 @@ class MongoChecksRepository implements IChecksRepository {
$match: {
"metadata.monitorId": { $in: objectIds },
createdAt: { $gte: windowStart },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{
@@ -404,11 +404,14 @@ class MongoChecksRepository implements IChecksRepository {
monitorId: "$metadata.monitorId",
day: { $dateTrunc: { date: "$createdAt", unit: "day", timezone } },
},
totalChecks: { $sum: 1 },
upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
// Counts exclude degraded-egress checks; the response-time average keeps them.
totalChecks: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
upChecks: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
avgResponseTime: { $avg: "$responseTime" },
},
},
// A day with nothing attributable to the target renders as an empty bar; the client divides by totalChecks for the tooltip.
{ $match: { totalChecks: { $gt: 0 } } },
{ $sort: { "_id.day": 1 } },
{
$project: {
+4 -3
View File
@@ -1,7 +1,7 @@
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { Response } from "got";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
export const CHECK_TTL_SENTINEL = 366;
@@ -117,8 +117,9 @@ export interface Check {
seo?: number;
performance?: number;
audits?: CheckAudits;
// Set only on checks produced while the reliability probe ran (a failing check).
// "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target.
// Set on failing checks while the egress check is enabled; absent otherwise.
// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
// "ok": a reliability target was reachable.
egressStatus?: EgressStatus;
createdAt: string;
updatedAt: string;
+1 -1
View File
@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
const PROBE_MONITOR_ID = "egress-probe";
@@ -50,6 +50,21 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
// Types whose transport failure can be the instance's own loss of egress. Hardware (Capture agent) and docker
// (local socket or host) failures are local, so they are never attributed to egress.
export const EgressAttributableTypes = [
"http",
"ping",
"pagespeed",
"port",
"game",
"grpc",
"websocket",
"dns",
] as const satisfies readonly MonitorType[];
export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
export const isEgressAttributable = (type: MonitorType): type is EgressAttributableType => EgressAttributableTypes.some((t) => t === type);
export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
export type MonitorStatus = (typeof MonitorStatuses)[number];
+25 -3
View File
@@ -11,7 +11,9 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { IEgressService } from "@/domain/egress/egress.service.js";
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { isEgressAttributable } from "@/domain/monitors/monitor.type.js";
export interface ICheckProducer {
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -44,6 +46,26 @@ export class CheckProducer implements ICheckProducer {
return dockerTlsKey ?? undefined;
}
// Only a failure to reach the target at all can be the instance's own fault. Any HTTP response (4xx, 5xx,
// or a 200 with a content mismatch) proves the target was reached, and hardware/docker failures are local.
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
return status.status === false && isEgressAttributable(monitor.type) && !isHttpStatusCode(status.code);
}
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
try {
return await this.egressService.assessAfterFailure();
} catch (error: unknown) {
this.logger.warn({
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "assessEgress",
});
return null;
}
}
produce = async (monitor: Monitor) => {
if (!monitor.id) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
@@ -76,9 +98,9 @@ export class CheckProducer implements ICheckProducer {
throw new Error("No network response");
}
// Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target.
// Step 1c: On a transport failure, ask whether the instance itself can reach anything before blaming the target.
// Null means the egress check is disabled (or failed internally) and the check is treated as usual.
const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null;
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
// ****************************
// Step 2: Record