Merge pull request #3963 from web-engineer/feat/egress-2-check-persistence

Egress self-check (2/4): flag checks taken during degraded egress (#3937)
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-09-20 16:31:06 -07:00
commit 253642dde9
42 files changed
+1319 -74

No files matched your search

@@ -1,6 +1,6 @@
import { Schema, model, type Types } from "mongoose";
import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
_id: Types.ObjectId;
@@ -45,7 +45,6 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
globalProxyId: { type: String, default: null },
egressCheckEnabled: { type: Boolean, default: false },
egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
},
{ timestamps: true }
@@ -3,7 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
class MongoSettingsRepository implements ISettingsRepository {
private toEntity = (doc: AppSettingsDocument): Settings => {
@@ -33,7 +33,6 @@ class MongoSettingsRepository implements ISettingsRepository {
globalProxyId: doc.globalProxyId ?? undefined,
egressCheckEnabled: doc.egressCheckEnabled ?? false,
egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
egressNotifications: doc.egressNotifications ?? [],
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
@@ -54,7 +54,6 @@ export interface Settings {
// Egress self-check (see domain/egress)
egressCheckEnabled: boolean;
egressCheckTargets: string[];
egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
createdAt: string;
updatedAt: string;
@@ -1,5 +1,6 @@
import CheckModel from "@/domain/checks/check.model.js";
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
type DateRange = { start: Date; end: Date };
@@ -9,6 +10,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
@@ -16,6 +18,7 @@ export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Pr
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
status: true,
});
return { totalChecks: count };
@@ -28,6 +31,7 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{
@@ -48,6 +52,7 @@ export const getDockerLatestCheck = async (monitorId: string) =>
CheckModel.findOne({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
...EXCLUDE_DEGRADED_EGRESS_MATCH,
})
.sort({ createdAt: -1 })
.select("containers containerSummary createdAt")
@@ -65,6 +70,7 @@ export const getDockerContainerStats = async (
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
"containers.name": containerName,
},
},
@@ -88,6 +94,7 @@ export const getDockerContainerLatestCheck = async (monitorId: string, container
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
"containers.name": containerName,
...EXCLUDE_DEGRADED_EGRESS_MATCH,
})
.sort({ createdAt: -1 })
.select("containers createdAt")
@@ -1,6 +1,7 @@
import { CheckModel } from "@/domain/checks/check.model.js";
import mongoose from "mongoose";
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
type DateRange = { start: Date; end: Date };
type HardwareUpChecks = { totalChecks: number };
@@ -70,6 +71,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
};
@@ -78,6 +80,7 @@ export const getHardwareUpChecks = async (monitorId: string, dates: DateRange):
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
status: true,
});
return { totalChecks: count };
@@ -95,6 +98,7 @@ export const getHardwareStats = async (monitorId: string, dates: DateRange, date
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{ $sort: { createdAt: 1 } },
+6
View File
@@ -1,5 +1,6 @@
import { Schema, model, Types } from "mongoose";
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
import { EgressStatuses } from "@/domain/egress/egress.type.js";
import type {
Check,
CheckAudits,
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
type: containerSummarySchema,
default: undefined,
},
egressStatus: {
type: String,
enum: EgressStatuses,
default: undefined,
},
},
{
timestamps: true,
+7
View File
@@ -0,0 +1,7 @@
import type { EgressStatus } from "@/domain/egress/egress.type.js";
// A check recorded while the instance's own egress was down says nothing about the target, so it is left out of
// every aggregate: uptime percentages, up/down counts and the response-time series alike. A degraded episode
// therefore reads as a gap in monitoring rather than as downtime or as a dip in response time.
// Spread this into the $match or filter of any query that reports statistics. Check listings keep degraded checks.
export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
@@ -16,6 +16,7 @@ import type {
} from "@/domain/checks/check.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
import { ILogger } from "@/utils/logger.js";
@@ -179,6 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
audits: mapAudits(doc.audits),
containers: doc.containers,
containerSummary: doc.containerSummary,
egressStatus: doc.egressStatus,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
};
@@ -348,6 +350,7 @@ class MongoChecksRepository implements IChecksRepository {
const baseMatch = {
"metadata.teamId": new mongoose.Types.ObjectId(teamId),
createdAt: { $gte: getDateForRange(dateRange) },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [totalResult, downResult] = await Promise.all([
@@ -393,6 +396,7 @@ class MongoChecksRepository implements IChecksRepository {
$match: {
"metadata.monitorId": { $in: objectIds },
createdAt: { $gte: windowStart },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{
@@ -473,6 +477,7 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [result] = await CheckModel.aggregate([
{ $match: matchStage },
@@ -638,6 +643,7 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [result] = await CheckModel.aggregate([
+5
View File
@@ -1,4 +1,5 @@
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { Response } from "got";
@@ -116,6 +117,10 @@ export interface Check {
seo?: number;
performance?: number;
audits?: CheckAudits;
// Set on failing checks while the egress check is enabled; absent otherwise.
// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
// "ok": a reliability target was reachable.
egressStatus?: EgressStatus;
createdAt: string;
updatedAt: string;
}
+6 -11
View File
@@ -7,7 +7,7 @@ import type { INetworkService } from "@/service/networkService.js";
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
import type { ILogger } from "@/utils/logger.js";
import {
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
EGRESS_RECOVERY_POLL_SECONDS,
DEFAULT_EGRESS_TARGETS,
type EgressAssessment,
type EgressProbeResult,
@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
const PROBE_MONITOR_ID = "egress-probe";
@@ -136,11 +136,6 @@ export class EgressService implements IEgressService {
return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
};
private toIntervalMs = (seconds: number | undefined): number => {
const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
};
// ****************************
// Event-triggered assessment
// ****************************
@@ -181,7 +176,7 @@ export class EgressService implements IEgressService {
const state = await this.egressStateRepository.findSingleton();
if (state.status === "degraded") {
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
await this.scheduleRecoveryCheck(false);
return "degraded";
}
@@ -195,7 +190,7 @@ export class EgressService implements IEgressService {
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
await this.scheduleRecoveryCheck(true);
const degraded = await this.egressStateRepository.markDegraded(results, now);
this.logger.warn({
message: degraded
@@ -224,8 +219,8 @@ export class EgressService implements IEgressService {
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
// so re-arming while degraded never pushes a pending run back.
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
private scheduleRecoveryCheck = async (reschedule: boolean) => {
const intervalMs = EGRESS_RECOVERY_POLL_SECONDS * 1000;
const seed: JobSeed = {
id: RECOVERY_JOB_ID,
type: RECOVERY_JOB_TYPE,
+5 -3
View File
@@ -9,11 +9,13 @@ export type EgressAssessment = EgressStatus | "disabled";
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
export const MAX_EGRESS_TARGETS = 10;
// While the instance is marked degraded, every failing check is flagged from stored state without probing, so
// this interval is the window in which a recovered instance still discards its checks. It is deliberately fixed
// and not configurable: the only effect of raising it is a longer blind window.
export const EGRESS_RECOVERY_POLL_SECONDS = 5;
// Instance-global singleton persisted in MongoDB so that every worker process and the API
// process share one view of the state (the queue worker may run separately from the API).
export interface EgressState {
@@ -50,6 +50,31 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
// however far away the agent is. `unknown` is excluded because no request is made for it.
export const EgressAttributableTypes = [
"http",
"ping",
"pagespeed",
"hardware",
"docker",
"port",
"game",
"grpc",
"websocket",
"dns",
] as const satisfies readonly MonitorType[];
export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
// A Docker daemon reached over a unix socket is local IPC and cannot fail through egress; one reached over
// TCP or TLS is as remote as any other target. The URL, not the type, settles it.
const isLocalDockerSocket = (monitor: Pick<Monitor, "type" | "url">): boolean =>
monitor.type === "docker" && (monitor.url?.trim().startsWith("unix://") ?? false);
export const isEgressAttributable = (monitor: Pick<Monitor, "type" | "url">): boolean =>
EgressAttributableTypes.some((t) => t === monitor.type) && !isLocalDockerSocket(monitor);
export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
export type MonitorStatus = (typeof MonitorStatuses)[number];