Merge pull request #3963 from web-engineer/feat/egress-2-check-persistence
Egress self-check (2/4): flag checks taken during degraded egress (#3937)
This commit is contained in:
42 files changed
+1319
-74
No files matched your search
@@ -1,6 +1,6 @@
|
||||
import { Schema, model, type Types } from "mongoose";
|
||||
import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
|
||||
_id: Types.ObjectId;
|
||||
@@ -45,7 +45,6 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
|
||||
globalProxyId: { type: String, default: null },
|
||||
egressCheckEnabled: { type: Boolean, default: false },
|
||||
egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
|
||||
egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
|
||||
egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
|
||||
},
|
||||
{ timestamps: true }
|
||||
|
||||
@@ -3,7 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
|
||||
import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
|
||||
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
class MongoSettingsRepository implements ISettingsRepository {
|
||||
private toEntity = (doc: AppSettingsDocument): Settings => {
|
||||
@@ -33,7 +33,6 @@ class MongoSettingsRepository implements ISettingsRepository {
|
||||
globalProxyId: doc.globalProxyId ?? undefined,
|
||||
egressCheckEnabled: doc.egressCheckEnabled ?? false,
|
||||
egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
|
||||
egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
egressNotifications: doc.egressNotifications ?? [],
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
|
||||
@@ -54,7 +54,6 @@ export interface Settings {
|
||||
// Egress self-check (see domain/egress)
|
||||
egressCheckEnabled: boolean;
|
||||
egressCheckTargets: string[];
|
||||
egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
|
||||
egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import CheckModel from "@/domain/checks/check.model.js";
|
||||
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
|
||||
import mongoose from "mongoose";
|
||||
|
||||
type DateRange = { start: Date; end: Date };
|
||||
@@ -9,6 +10,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
});
|
||||
|
||||
export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
|
||||
@@ -16,6 +18,7 @@ export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Pr
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
status: true,
|
||||
});
|
||||
return { totalChecks: count };
|
||||
@@ -28,6 +31,7 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -48,6 +52,7 @@ export const getDockerLatestCheck = async (monitorId: string) =>
|
||||
CheckModel.findOne({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
})
|
||||
.sort({ createdAt: -1 })
|
||||
.select("containers containerSummary createdAt")
|
||||
@@ -65,6 +70,7 @@ export const getDockerContainerStats = async (
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
"containers.name": containerName,
|
||||
},
|
||||
},
|
||||
@@ -88,6 +94,7 @@ export const getDockerContainerLatestCheck = async (monitorId: string, container
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
"containers.name": containerName,
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
})
|
||||
.sort({ createdAt: -1 })
|
||||
.select("containers createdAt")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { CheckModel } from "@/domain/checks/check.model.js";
|
||||
import mongoose from "mongoose";
|
||||
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
|
||||
|
||||
type DateRange = { start: Date; end: Date };
|
||||
type HardwareUpChecks = { totalChecks: number };
|
||||
@@ -70,6 +71,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "hardware",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -78,6 +80,7 @@ export const getHardwareUpChecks = async (monitorId: string, dates: DateRange):
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "hardware",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
status: true,
|
||||
});
|
||||
return { totalChecks: count };
|
||||
@@ -95,6 +98,7 @@ export const getHardwareStats = async (monitorId: string, dates: DateRange, date
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "hardware",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{ $sort: { createdAt: 1 } },
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Schema, model, Types } from "mongoose";
|
||||
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { EgressStatuses } from "@/domain/egress/egress.type.js";
|
||||
import type {
|
||||
Check,
|
||||
CheckAudits,
|
||||
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
|
||||
type: containerSummarySchema,
|
||||
default: undefined,
|
||||
},
|
||||
egressStatus: {
|
||||
type: String,
|
||||
enum: EgressStatuses,
|
||||
default: undefined,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
|
||||
// A check recorded while the instance's own egress was down says nothing about the target, so it is left out of
|
||||
// every aggregate: uptime percentages, up/down counts and the response-time series alike. A degraded episode
|
||||
// therefore reads as a gap in monitoring rather than as downtime or as a dip in response time.
|
||||
// Spread this into the $match or filter of any query that reports statistics. Check listings keep degraded checks.
|
||||
export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
|
||||
@@ -16,6 +16,7 @@ import type {
|
||||
} from "@/domain/checks/check.type.js";
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
|
||||
import mongoose from "mongoose";
|
||||
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
@@ -179,6 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
audits: mapAudits(doc.audits),
|
||||
containers: doc.containers,
|
||||
containerSummary: doc.containerSummary,
|
||||
egressStatus: doc.egressStatus,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
@@ -348,6 +350,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const baseMatch = {
|
||||
"metadata.teamId": new mongoose.Types.ObjectId(teamId),
|
||||
createdAt: { $gte: getDateForRange(dateRange) },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
};
|
||||
|
||||
const [totalResult, downResult] = await Promise.all([
|
||||
@@ -393,6 +396,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
$match: {
|
||||
"metadata.monitorId": { $in: objectIds },
|
||||
createdAt: { $gte: windowStart },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -473,6 +477,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const matchStage = {
|
||||
"metadata.monitorId": monitorObjectId,
|
||||
createdAt: { $gte: startDate, $lte: endDate },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
};
|
||||
const [result] = await CheckModel.aggregate([
|
||||
{ $match: matchStage },
|
||||
@@ -638,6 +643,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const matchStage = {
|
||||
"metadata.monitorId": monitorObjectId,
|
||||
createdAt: { $gte: startDate, $lte: endDate },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
};
|
||||
|
||||
const [result] = await CheckModel.aggregate([
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
|
||||
import type { Response } from "got";
|
||||
|
||||
@@ -116,6 +117,10 @@ export interface Check {
|
||||
seo?: number;
|
||||
performance?: number;
|
||||
audits?: CheckAudits;
|
||||
// Set on failing checks while the egress check is enabled; absent otherwise.
|
||||
// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
|
||||
// "ok": a reliability target was reachable.
|
||||
egressStatus?: EgressStatus;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import type { INetworkService } from "@/service/networkService.js";
|
||||
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import {
|
||||
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
EGRESS_RECOVERY_POLL_SECONDS,
|
||||
DEFAULT_EGRESS_TARGETS,
|
||||
type EgressAssessment,
|
||||
type EgressProbeResult,
|
||||
@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
|
||||
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
|
||||
|
||||
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
|
||||
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
|
||||
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
|
||||
const PROBE_MONITOR_ID = "egress-probe";
|
||||
@@ -136,11 +136,6 @@ export class EgressService implements IEgressService {
|
||||
return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
|
||||
};
|
||||
|
||||
private toIntervalMs = (seconds: number | undefined): number => {
|
||||
const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
|
||||
return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
|
||||
};
|
||||
|
||||
// ****************************
|
||||
// Event-triggered assessment
|
||||
// ****************************
|
||||
@@ -181,7 +176,7 @@ export class EgressService implements IEgressService {
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status === "degraded") {
|
||||
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
|
||||
await this.scheduleRecoveryCheck(false);
|
||||
return "degraded";
|
||||
}
|
||||
|
||||
@@ -195,7 +190,7 @@ export class EgressService implements IEgressService {
|
||||
|
||||
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
|
||||
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
|
||||
await this.scheduleRecoveryCheck(true);
|
||||
const degraded = await this.egressStateRepository.markDegraded(results, now);
|
||||
this.logger.warn({
|
||||
message: degraded
|
||||
@@ -224,8 +219,8 @@ export class EgressService implements IEgressService {
|
||||
|
||||
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
|
||||
// so re-arming while degraded never pushes a pending run back.
|
||||
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
|
||||
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
|
||||
private scheduleRecoveryCheck = async (reschedule: boolean) => {
|
||||
const intervalMs = EGRESS_RECOVERY_POLL_SECONDS * 1000;
|
||||
const seed: JobSeed = {
|
||||
id: RECOVERY_JOB_ID,
|
||||
type: RECOVERY_JOB_TYPE,
|
||||
|
||||
@@ -9,11 +9,13 @@ export type EgressAssessment = EgressStatus | "disabled";
|
||||
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
|
||||
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
|
||||
export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
|
||||
export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
|
||||
export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
|
||||
export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
|
||||
export const MAX_EGRESS_TARGETS = 10;
|
||||
|
||||
// While the instance is marked degraded, every failing check is flagged from stored state without probing, so
|
||||
// this interval is the window in which a recovered instance still discards its checks. It is deliberately fixed
|
||||
// and not configurable: the only effect of raising it is a longer blind window.
|
||||
export const EGRESS_RECOVERY_POLL_SECONDS = 5;
|
||||
|
||||
// Instance-global singleton persisted in MongoDB so that every worker process and the API
|
||||
// process share one view of the state (the queue worker may run separately from the API).
|
||||
export interface EgressState {
|
||||
|
||||
@@ -50,6 +50,31 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
|
||||
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
|
||||
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
|
||||
|
||||
// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
|
||||
// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
|
||||
// however far away the agent is. `unknown` is excluded because no request is made for it.
|
||||
export const EgressAttributableTypes = [
|
||||
"http",
|
||||
"ping",
|
||||
"pagespeed",
|
||||
"hardware",
|
||||
"docker",
|
||||
"port",
|
||||
"game",
|
||||
"grpc",
|
||||
"websocket",
|
||||
"dns",
|
||||
] as const satisfies readonly MonitorType[];
|
||||
export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
|
||||
|
||||
// A Docker daemon reached over a unix socket is local IPC and cannot fail through egress; one reached over
|
||||
// TCP or TLS is as remote as any other target. The URL, not the type, settles it.
|
||||
const isLocalDockerSocket = (monitor: Pick<Monitor, "type" | "url">): boolean =>
|
||||
monitor.type === "docker" && (monitor.url?.trim().startsWith("unix://") ?? false);
|
||||
|
||||
export const isEgressAttributable = (monitor: Pick<Monitor, "type" | "url">): boolean =>
|
||||
EgressAttributableTypes.some((t) => t === monitor.type) && !isLocalDockerSocket(monitor);
|
||||
|
||||
export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
|
||||
export type MonitorStatus = (typeof MonitorStatuses)[number];
|
||||
|
||||
|
||||
Reference in new issue
Block a user