Add egress state and probe service (#3937)
Adds the EgressService that probes a configurable set of reliability targets
(ping, TCP or HTTP) when asked, persists a single instance-wide egress state
document, and runs a recovery poll loop while egress is degraded. Adds the four
egress settings fields with validation. The worker constructs and initialises
the service but nothing calls assessAfterFailure yet, so behaviour is unchanged.
🔧
This commit is contained in:
1 parent
f41e0f0619
commit
ec01a5dfe4
17 files changed
+1080
-1
No files matched your search
@@ -1,6 +1,15 @@
|
||||
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
|
||||
import { z } from "zod";
|
||||
|
||||
// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL.
|
||||
const egressTargetValidation = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(253)
|
||||
.regex(/^(https?:\/\/\S+|[A-Za-z0-9.\-:[\]]+)$/, "Invalid egress target");
|
||||
|
||||
//****************************************
|
||||
// Settings Validations
|
||||
//****************************************
|
||||
@@ -42,6 +51,11 @@ export const updateAppSettingsBodyValidation = z
|
||||
.optional(),
|
||||
globalProxyEnabled: z.boolean().optional(),
|
||||
globalProxyId: z.string().nullable().optional(),
|
||||
|
||||
egressCheckEnabled: z.boolean().optional(),
|
||||
egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
|
||||
egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
|
||||
egressNotifications: z.array(z.string().regex(/^[a-f\d]{24}$/i, "Invalid notification id")).optional(),
|
||||
})
|
||||
.strip()
|
||||
.superRefine((body, ctx) => {
|
||||
|
||||
@@ -7,6 +7,7 @@ import { ITagsService, TagsService } from "@/domain/tags/tag.service.js";
|
||||
import { IUserService, UserService } from "@/domain/users/user.service.js";
|
||||
import { IJobScheduler } from "@/worker/worker.interface.js";
|
||||
import { ProxiesService, IProxiesService } from "@/domain/proxies/proxy.service.js";
|
||||
import { EgressStateService, IEgressStateService } from "@/domain/egress/egress-state.service.js";
|
||||
import { SharedServices } from "@/config/services.shared.js";
|
||||
|
||||
// Third-party
|
||||
@@ -24,6 +25,7 @@ export interface ApiServices extends SharedServices {
|
||||
tagsService: ITagsService;
|
||||
diagnosticService: IDiagnosticService;
|
||||
proxiesService: IProxiesService;
|
||||
egressStateService: IEgressStateService;
|
||||
}
|
||||
|
||||
export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => {
|
||||
@@ -48,6 +50,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
maintenanceWindowsRepository,
|
||||
jobsRepository,
|
||||
proxiesRepository,
|
||||
egressStateRepository,
|
||||
} = shared;
|
||||
|
||||
const userService = new UserService({
|
||||
@@ -100,6 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
const tagsService = new TagsService(tagsRepository, monitorsRepository);
|
||||
const diagnosticService = new DiagnosticService(db);
|
||||
const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService);
|
||||
const egressStateService = new EgressStateService(egressStateRepository);
|
||||
return {
|
||||
...shared,
|
||||
worker: jobScheduler,
|
||||
@@ -111,5 +115,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
tagsService,
|
||||
diagnosticService,
|
||||
proxiesService,
|
||||
egressStateService,
|
||||
};
|
||||
};
|
||||
@@ -56,6 +56,7 @@ import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
|
||||
import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.js";
|
||||
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
|
||||
// Mongo repository implementations
|
||||
import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js";
|
||||
@@ -76,6 +77,7 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js";
|
||||
import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
|
||||
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
|
||||
import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js";
|
||||
import MongoEgressStateRepository from "@/domain/egress/egress-state.repository.mongo.js";
|
||||
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
|
||||
|
||||
// Shared infrastructure + business services that both the API and the worker process construct.
|
||||
@@ -114,6 +116,7 @@ export interface SharedServices {
|
||||
teamsRepository: ITeamsRepository;
|
||||
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
proxiesRepository: IProxiesRepository;
|
||||
egressStateRepository: IEgressStateRepository;
|
||||
}
|
||||
|
||||
export const buildShared = async ({
|
||||
@@ -157,6 +160,7 @@ export const buildShared = async ({
|
||||
const teamsRepository = new MongoTeamsRepository();
|
||||
const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
|
||||
const proxiesRepository = new MongoProxiesRepository();
|
||||
const egressStateRepository = new MongoEgressStateRepository();
|
||||
|
||||
// Inject settings repository into settings service (now that DB is connected)
|
||||
(settingsService as SettingsService).setRepository(settingsRepository);
|
||||
@@ -246,6 +250,7 @@ export const buildShared = async ({
|
||||
teamsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
proxiesRepository,
|
||||
egressStateRepository,
|
||||
};
|
||||
return sharedServices;
|
||||
};
|
||||
@@ -28,6 +28,7 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
|
||||
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
|
||||
import { ProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import { IEgressService, EgressService } from "@/domain/egress/egress.service.js";
|
||||
// Network providers
|
||||
import { PingProvider } from "@/service/network/PingProvider.js";
|
||||
import { HttpProvider } from "@/service/network/HttpProvider.js";
|
||||
@@ -46,6 +47,7 @@ export interface WorkerServices {
|
||||
networkService: INetworkService;
|
||||
bufferService: IBufferService;
|
||||
statusService: IStatusService;
|
||||
egressService: IEgressService;
|
||||
}
|
||||
|
||||
export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise<WorkerServices> => {
|
||||
@@ -70,6 +72,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
teamsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
proxiesRepository,
|
||||
egressStateRepository,
|
||||
} = shared;
|
||||
|
||||
// ***********************
|
||||
@@ -103,6 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
|
||||
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
|
||||
const monitorStatusPolicy = new MonitorStatusPolicy();
|
||||
const egressService = new EgressService(settingsService, egressStateRepository, networkService, logger);
|
||||
|
||||
// ***********************
|
||||
// Reactors and dispatcher
|
||||
@@ -185,5 +189,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
});
|
||||
}
|
||||
|
||||
return { worker, networkService, bufferService, statusService };
|
||||
// Resume recovery polling if the instance was mid-episode when it last stopped
|
||||
await egressService.init();
|
||||
|
||||
return { worker, networkService, bufferService, statusService, egressService };
|
||||
};
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Schema, model, type Types } from "mongoose";
|
||||
import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
|
||||
_id: Types.ObjectId;
|
||||
@@ -42,6 +43,10 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
|
||||
globalThresholds: { type: thresholdsSchema },
|
||||
globalProxyEnabled: { type: Boolean, default: false },
|
||||
globalProxyId: { type: String, default: null },
|
||||
egressCheckEnabled: { type: Boolean, default: false },
|
||||
egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
|
||||
egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
|
||||
egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
@@ -3,6 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
|
||||
import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
|
||||
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
class MongoSettingsRepository implements ISettingsRepository {
|
||||
private toEntity = (doc: AppSettingsDocument): Settings => {
|
||||
@@ -30,6 +31,10 @@ class MongoSettingsRepository implements ISettingsRepository {
|
||||
globalThresholds: doc.globalThresholds ?? undefined,
|
||||
globalProxyEnabled: doc.globalProxyEnabled ?? false,
|
||||
globalProxyId: doc.globalProxyId ?? undefined,
|
||||
egressCheckEnabled: doc.egressCheckEnabled ?? false,
|
||||
egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
|
||||
egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
egressNotifications: doc.egressNotifications ?? [],
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
|
||||
@@ -51,6 +51,11 @@ export interface Settings {
|
||||
globalThresholds?: SettingsThresholds;
|
||||
globalProxyEnabled: boolean;
|
||||
globalProxyId?: string | null;
|
||||
// Egress self-check (see domain/egress)
|
||||
egressCheckEnabled: boolean;
|
||||
egressCheckTargets: string[];
|
||||
egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
|
||||
egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { Schema, model, type Types } from "mongoose";
|
||||
import { EgressStatuses, type EgressProbeResult, type EgressState } from "@/domain/egress/egress.type.js";
|
||||
|
||||
interface EgressStateDocument extends Omit<EgressState, "id" | "degradedSince" | "lastRecoveredAt" | "lastProbeAt" | "createdAt" | "updatedAt"> {
|
||||
_id: Types.ObjectId;
|
||||
singleton: boolean;
|
||||
degradedSince: Date | null;
|
||||
lastRecoveredAt: Date | null;
|
||||
lastProbeAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
const probeResultSchema = new Schema<EgressProbeResult>(
|
||||
{
|
||||
target: { type: String, required: true },
|
||||
reachable: { type: Boolean, required: true },
|
||||
responseTime: { type: Number, default: 0 },
|
||||
message: { type: String },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const EgressStateSchema = new Schema<EgressStateDocument>(
|
||||
{
|
||||
singleton: { type: Boolean, required: true, unique: true, default: true },
|
||||
status: { type: String, enum: EgressStatuses, required: true, default: "ok" },
|
||||
degradedSince: { type: Date, default: null },
|
||||
lastRecoveredAt: { type: Date, default: null },
|
||||
lastProbeAt: { type: Date, default: null },
|
||||
lastProbeResults: { type: [probeResultSchema], default: () => [] },
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
const EgressStateModel = model<EgressStateDocument>("EgressState", EgressStateSchema);
|
||||
|
||||
export type { EgressStateDocument };
|
||||
export { EgressStateModel };
|
||||
export default EgressStateModel;
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
|
||||
|
||||
export interface IEgressStateRepository {
|
||||
// read
|
||||
findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call
|
||||
// update
|
||||
recordProbe(results: EgressProbeResult[], now: Date): Promise<EgressState>;
|
||||
// Atomic transitions. Each returns the updated state when this caller performed the transition,
|
||||
// or null when the state was already on the target side (another worker got there first).
|
||||
markDegraded(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
|
||||
markRecovered(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
|
||||
import { EgressStateModel, type EgressStateDocument } from "@/domain/egress/egress-state.model.js";
|
||||
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
|
||||
|
||||
const SERVICE_NAME = "EgressStateRepository";
|
||||
|
||||
class MongoEgressStateRepository implements IEgressStateRepository {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private toEntity = (doc: EgressStateDocument): EgressState => {
|
||||
return {
|
||||
id: toStringId(doc._id),
|
||||
status: doc.status,
|
||||
degradedSince: doc.degradedSince ? toDateString(doc.degradedSince) : null,
|
||||
lastRecoveredAt: doc.lastRecoveredAt ? toDateString(doc.lastRecoveredAt) : null,
|
||||
lastProbeAt: doc.lastProbeAt ? toDateString(doc.lastProbeAt) : null,
|
||||
lastProbeResults: (doc.lastProbeResults ?? []).map((result) => ({
|
||||
target: result.target,
|
||||
reachable: result.reachable,
|
||||
responseTime: result.responseTime ?? 0,
|
||||
...(result.message !== undefined && { message: result.message }),
|
||||
})),
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
};
|
||||
|
||||
findSingleton = async () => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $setOnInsert: { singleton: true, status: "ok" } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
};
|
||||
|
||||
recordProbe = async (results: EgressProbeResult[], now: Date) => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $set: { lastProbeAt: now, lastProbeResults: results }, $setOnInsert: { singleton: true, status: "ok" } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
};
|
||||
|
||||
markDegraded = async (results: EgressProbeResult[], now: Date) => {
|
||||
// Only the caller that flips ok -> degraded sees a document back; everyone else sees null.
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true, status: "ok" },
|
||||
{ $set: { status: "degraded", degradedSince: now, lastProbeAt: now, lastProbeResults: results } },
|
||||
{ new: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return doc ? this.toEntity(doc) : null;
|
||||
};
|
||||
|
||||
markRecovered = async (results: EgressProbeResult[], now: Date) => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true, status: "degraded" },
|
||||
{ $set: { status: "ok", lastRecoveredAt: now, lastProbeAt: now, lastProbeResults: results } },
|
||||
{ new: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return doc ? this.toEntity(doc) : null;
|
||||
};
|
||||
}
|
||||
|
||||
export default MongoEgressStateRepository;
|
||||
@@ -0,0 +1,20 @@
|
||||
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import type { EgressState } from "@/domain/egress/egress.type.js";
|
||||
|
||||
export interface IEgressStateService {
|
||||
getState(): Promise<EgressState>;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "EgressStateService";
|
||||
|
||||
// Read-only view of the instance-global egress state for the API process. Transitions are
|
||||
// owned by the worker (egress.service.ts); this exists so the controller never touches the repository.
|
||||
export class EgressStateService implements IEgressStateService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor(private egressStateRepository: IEgressStateRepository) {}
|
||||
|
||||
getState = async (): Promise<EgressState> => {
|
||||
return this.egressStateRepository.findSingleton();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import type { INetworkService } from "@/service/networkService.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import {
|
||||
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
DEFAULT_EGRESS_TARGETS,
|
||||
type EgressProbeResult,
|
||||
type EgressStatus,
|
||||
} from "@/domain/egress/egress.type.js";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
|
||||
const SERVICE_NAME = "EgressService";
|
||||
const PROBE_TIMEOUT_MS = 5000;
|
||||
|
||||
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
|
||||
const PROBE_MONITOR_ID = "egress-probe";
|
||||
const PROBE_TEAM_ID = "system";
|
||||
|
||||
// "[host]:port" or "host:port" where host has no colons (so a bare IPv6 address is not mistaken for host:port).
|
||||
const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/;
|
||||
|
||||
export interface IEgressService {
|
||||
probeTargets(targets: string[]): Promise<EgressProbeResult[]>;
|
||||
assessAfterFailure(): Promise<EgressStatus | null>;
|
||||
startRecoveryPolling(): void;
|
||||
stop(): void;
|
||||
init(): Promise<void>;
|
||||
}
|
||||
|
||||
export class EgressService implements IEgressService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private polling = false;
|
||||
private pollTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(
|
||||
private settingsService: ISettingsService,
|
||||
private egressStateRepository: IEgressStateRepository,
|
||||
private networkService: INetworkService,
|
||||
private logger: ILogger
|
||||
) {}
|
||||
|
||||
// ****************************
|
||||
// Probing
|
||||
// ****************************
|
||||
|
||||
// Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor.
|
||||
private toProbeMonitor = (target: string): Monitor => {
|
||||
const base = {
|
||||
id: PROBE_MONITOR_ID,
|
||||
teamId: PROBE_TEAM_ID,
|
||||
name: `Egress probe: ${target}`,
|
||||
method: "GET",
|
||||
useAdvancedMatching: false,
|
||||
ignoreTlsErrors: false,
|
||||
customUpCodes: [],
|
||||
};
|
||||
|
||||
if (/^https?:\/\//i.test(target)) {
|
||||
return { ...base, type: "http", url: target } as unknown as Monitor;
|
||||
}
|
||||
|
||||
const hostPort = HOST_PORT_PATTERN.exec(target);
|
||||
if (hostPort) {
|
||||
const host = hostPort[1] ?? hostPort[2];
|
||||
return { ...base, type: "port", url: host, port: Number(hostPort[3]) } as unknown as Monitor;
|
||||
}
|
||||
|
||||
return { ...base, type: "ping", url: target } as unknown as Monitor;
|
||||
};
|
||||
|
||||
private probeTarget = async (target: string): Promise<EgressProbeResult> => {
|
||||
const monitor = this.toProbeMonitor(target);
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const { response, responseTime, error } = await timeRequest(() =>
|
||||
Promise.race([
|
||||
this.networkService.requestStatus(monitor),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS);
|
||||
}),
|
||||
]).finally(() => clearTimeout(timer))
|
||||
);
|
||||
|
||||
if (error || !response) {
|
||||
return {
|
||||
target,
|
||||
reachable: false,
|
||||
responseTime,
|
||||
message: error instanceof Error ? error.message : error ? String(error) : "No response",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
target,
|
||||
reachable: response.status === true,
|
||||
responseTime: response.responseTime ?? responseTime,
|
||||
message: response.message,
|
||||
};
|
||||
};
|
||||
|
||||
probeTargets = async (targets: string[]): Promise<EgressProbeResult[]> => {
|
||||
return await Promise.all(targets.map((target) => this.probeTarget(target)));
|
||||
};
|
||||
|
||||
private resolveTargets = (configured: string[] | undefined): string[] => {
|
||||
const targets = (configured ?? []).map((target) => target.trim()).filter((target) => target.length > 0);
|
||||
// An empty list would make "all unreachable" vacuously true, so fall back to the defaults.
|
||||
return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
|
||||
};
|
||||
|
||||
// ****************************
|
||||
// Event-triggered assessment
|
||||
// ****************************
|
||||
|
||||
assessAfterFailure = async (): Promise<EgressStatus | null> => {
|
||||
try {
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
if (!settings.egressCheckEnabled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status === "degraded") {
|
||||
// Recovery detection belongs to the poll loop. Make sure one is running in this process in case
|
||||
// the worker that entered the degraded state is no longer around.
|
||||
this.startRecoveryPolling();
|
||||
return "degraded";
|
||||
}
|
||||
|
||||
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
|
||||
const now = new Date();
|
||||
|
||||
if (results.some((result) => result.reachable)) {
|
||||
await this.egressStateRepository.recordProbe(results, now);
|
||||
return "ok";
|
||||
}
|
||||
|
||||
const degraded = await this.egressStateRepository.markDegraded(results, now);
|
||||
this.logger.warn({
|
||||
message: degraded
|
||||
? "Instance egress degraded: every reliability target is unreachable. Monitor failures will not be counted until it recovers"
|
||||
: "Instance egress degraded (transition already recorded by another worker)",
|
||||
service: SERVICE_NAME,
|
||||
method: "assessAfterFailure",
|
||||
details: { results },
|
||||
});
|
||||
this.startRecoveryPolling();
|
||||
return "degraded";
|
||||
} catch (error: unknown) {
|
||||
// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: "assessAfterFailure",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
// ****************************
|
||||
// Recovery poll loop
|
||||
// ****************************
|
||||
|
||||
startRecoveryPolling = () => {
|
||||
if (this.polling) {
|
||||
return;
|
||||
}
|
||||
this.polling = true;
|
||||
this.logger.info({ message: "Starting egress recovery polling", service: SERVICE_NAME, method: "startRecoveryPolling" });
|
||||
void this.scheduleNextTick();
|
||||
};
|
||||
|
||||
stop = () => {
|
||||
this.polling = false;
|
||||
if (this.pollTimer) {
|
||||
clearTimeout(this.pollTimer);
|
||||
this.pollTimer = null;
|
||||
}
|
||||
};
|
||||
|
||||
private readPollIntervalMs = async () => {
|
||||
try {
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
const seconds = settings.egressPollIntervalSeconds;
|
||||
return (Number.isFinite(seconds) && seconds > 0 ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: `Could not read egress poll interval, using default: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "readPollIntervalMs",
|
||||
});
|
||||
return DEFAULT_EGRESS_POLL_INTERVAL_SECONDS * 1000;
|
||||
}
|
||||
};
|
||||
|
||||
private scheduleNextTick = async () => {
|
||||
if (!this.polling) {
|
||||
return;
|
||||
}
|
||||
const intervalMs = await this.readPollIntervalMs();
|
||||
if (!this.polling) {
|
||||
return;
|
||||
}
|
||||
this.pollTimer = setTimeout(() => void this.tick(), intervalMs);
|
||||
// Never hold the process open for this loop; the state is persisted and init() resumes it on restart.
|
||||
this.pollTimer.unref();
|
||||
};
|
||||
|
||||
private tick = async () => {
|
||||
this.pollTimer = null;
|
||||
if (!this.polling) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status !== "degraded") {
|
||||
this.logger.info({ message: "Egress no longer degraded, stopping recovery polling", service: SERVICE_NAME, method: "tick" });
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
|
||||
const now = new Date();
|
||||
|
||||
if (!results.some((result) => result.reachable)) {
|
||||
await this.egressStateRepository.recordProbe(results, now);
|
||||
return;
|
||||
}
|
||||
|
||||
const recovered = await this.egressStateRepository.markRecovered(results, now);
|
||||
this.stop();
|
||||
if (!recovered) {
|
||||
// Another process performed the transition.
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.info({
|
||||
message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`,
|
||||
service: SERVICE_NAME,
|
||||
method: "tick",
|
||||
details: { results },
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: "tick",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
} finally {
|
||||
// No-op once stop() has been called; otherwise keep the loop alive through errors.
|
||||
await this.scheduleNextTick();
|
||||
}
|
||||
};
|
||||
|
||||
// ****************************
|
||||
// Lifecycle
|
||||
// ****************************
|
||||
|
||||
init = async () => {
|
||||
try {
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
if (!settings.egressCheckEnabled) {
|
||||
return;
|
||||
}
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status === "degraded") {
|
||||
this.logger.warn({
|
||||
message: `Instance egress was degraded at startup (since ${state.degradedSince ?? "unknown"}), resuming recovery polling`,
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
});
|
||||
this.startRecoveryPolling();
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Egress self-check: detects loss of the instance's own outbound connectivity so that
|
||||
// a burst of simultaneous monitor failures is not reported as a burst of real outages.
|
||||
|
||||
export const EgressStatuses = ["ok", "degraded"] as const;
|
||||
export type EgressStatus = (typeof EgressStatuses)[number];
|
||||
|
||||
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
|
||||
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
|
||||
export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
|
||||
export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
|
||||
export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
|
||||
export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
|
||||
export const MAX_EGRESS_TARGETS = 10;
|
||||
|
||||
// Instance-global singleton persisted in MongoDB so that every worker process and the API
|
||||
// process share one view of the state (the queue worker may run separately from the API).
|
||||
export interface EgressState {
|
||||
id: string;
|
||||
status: EgressStatus;
|
||||
degradedSince: string | null; // ISO timestamp of the current or most recent degraded entry
|
||||
lastRecoveredAt: string | null; // ISO timestamp of the most recent recovery
|
||||
lastProbeAt: string | null; // ISO timestamp of the most recent reliability probe
|
||||
lastProbeResults: EgressProbeResult[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface EgressProbeResult {
|
||||
target: string;
|
||||
reachable: boolean;
|
||||
responseTime: number; // ms
|
||||
message?: string;
|
||||
}
|
||||
Reference in new issue
Block a user