From ec01a5dfe4791992f7178238b0515a3fd62a77cf Mon Sep 17 00:00:00 2001 From: Craig Lawson Date: Mon, 14 Sep 2026 10:26:09 +0100 Subject: [PATCH] Add egress state and probe service (#3937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the EgressService that probes a configurable set of reliability targets (ping, TCP or HTTP) when asked, persists a single instance-wide egress state document, and runs a recovery poll loop while egress is degraded. Adds the four egress settings fields with validation. The worker constructs and initialises the service but nothing calls assessAfterFailure yet, so behaviour is unchanged. 🔧 --- server/openapi.json | 25 ++ .../src/api/validation/settingsValidation.ts | 14 + server/src/config/services.api.ts | 5 + server/src/config/services.shared.ts | 5 + server/src/config/services.worker.ts | 9 +- .../domain/app-settings/app-settings.model.ts | 5 + .../app-settings.repository.mongo.ts | 5 + .../domain/app-settings/app-settings.type.ts | 5 + .../src/domain/egress/egress-state.model.ts | 40 ++ .../egress-state.repository.interface.ts | 12 + .../egress/egress-state.repository.mongo.ts | 67 +++ .../src/domain/egress/egress-state.service.ts | 20 + server/src/domain/egress/egress.service.ts | 290 +++++++++++++ server/src/domain/egress/egress.type.ts | 33 ++ .../test/unit/services/egressService.test.ts | 404 ++++++++++++++++++ .../unit/services/egressStateService.test.ts | 70 +++ .../validation/settingsValidation.test.ts | 72 ++++ 17 files changed, 1080 insertions(+), 1 deletion(-) create mode 100644 server/src/domain/egress/egress-state.model.ts create mode 100644 server/src/domain/egress/egress-state.repository.interface.ts create mode 100644 server/src/domain/egress/egress-state.repository.mongo.ts create mode 100644 server/src/domain/egress/egress-state.service.ts create mode 100644 server/src/domain/egress/egress.service.ts create mode 100644 server/src/domain/egress/egress.type.ts create mode 100644 server/test/unit/services/egressService.test.ts create mode 100644 server/test/unit/services/egressStateService.test.ts create mode 100644 server/test/unit/validation/settingsValidation.test.ts diff --git a/server/openapi.json b/server/openapi.json index 4b4faf896..d0ae81f26 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -13294,6 +13294,31 @@ "globalProxyId": { "type": "string", "nullable": true + }, + "egressCheckEnabled": { + "type": "boolean" + }, + "egressCheckTargets": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^(https?:\\/\\/\\S+|[A-Za-z0-9.\\-:[\\]]+)$" + }, + "maxItems": 10 + }, + "egressPollIntervalSeconds": { + "type": "integer", + "minimum": 5, + "maximum": 600 + }, + "egressNotifications": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f\\d]{24}$/i" + } } } } diff --git a/server/src/api/validation/settingsValidation.ts b/server/src/api/validation/settingsValidation.ts index 50c75b043..33fe433fc 100644 --- a/server/src/api/validation/settingsValidation.ts +++ b/server/src/api/validation/settingsValidation.ts @@ -1,6 +1,15 @@ import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js"; +import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js"; import { z } from "zod"; +// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL. +const egressTargetValidation = z + .string() + .trim() + .min(1) + .max(253) + .regex(/^(https?:\/\/\S+|[A-Za-z0-9.\-:[\]]+)$/, "Invalid egress target"); + //**************************************** // Settings Validations //**************************************** @@ -42,6 +51,11 @@ export const updateAppSettingsBodyValidation = z .optional(), globalProxyEnabled: z.boolean().optional(), globalProxyId: z.string().nullable().optional(), + + egressCheckEnabled: z.boolean().optional(), + egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS + egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(), + egressNotifications: z.array(z.string().regex(/^[a-f\d]{24}$/i, "Invalid notification id")).optional(), }) .strip() .superRefine((body, ctx) => { diff --git a/server/src/config/services.api.ts b/server/src/config/services.api.ts index c5f58ac19..e929fbfe8 100644 --- a/server/src/config/services.api.ts +++ b/server/src/config/services.api.ts @@ -7,6 +7,7 @@ import { ITagsService, TagsService } from "@/domain/tags/tag.service.js"; import { IUserService, UserService } from "@/domain/users/user.service.js"; import { IJobScheduler } from "@/worker/worker.interface.js"; import { ProxiesService, IProxiesService } from "@/domain/proxies/proxy.service.js"; +import { EgressStateService, IEgressStateService } from "@/domain/egress/egress-state.service.js"; import { SharedServices } from "@/config/services.shared.js"; // Third-party @@ -24,6 +25,7 @@ export interface ApiServices extends SharedServices { tagsService: ITagsService; diagnosticService: IDiagnosticService; proxiesService: IProxiesService; + egressStateService: IEgressStateService; } export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => { @@ -48,6 +50,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A maintenanceWindowsRepository, jobsRepository, proxiesRepository, + egressStateRepository, } = shared; const userService = new UserService({ @@ -100,6 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A const tagsService = new TagsService(tagsRepository, monitorsRepository); const diagnosticService = new DiagnosticService(db); const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService); + const egressStateService = new EgressStateService(egressStateRepository); return { ...shared, worker: jobScheduler, @@ -111,5 +115,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A tagsService, diagnosticService, proxiesService, + egressStateService, }; }; diff --git a/server/src/config/services.shared.ts b/server/src/config/services.shared.ts index 3b4fdb147..a1a69566b 100644 --- a/server/src/config/services.shared.ts +++ b/server/src/config/services.shared.ts @@ -56,6 +56,7 @@ import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.js"; +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; // Mongo repository implementations import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js"; @@ -76,6 +77,7 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js"; import MongoTeamsRepository from "@/domain/teams/team.repository.model.js"; import MongoUsersRepository from "@/domain/users/user.repository.mongo.js"; import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js"; +import MongoEgressStateRepository from "@/domain/egress/egress-state.repository.mongo.js"; import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js"; // Shared infrastructure + business services that both the API and the worker process construct. @@ -114,6 +116,7 @@ export interface SharedServices { teamsRepository: ITeamsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; proxiesRepository: IProxiesRepository; + egressStateRepository: IEgressStateRepository; } export const buildShared = async ({ @@ -157,6 +160,7 @@ export const buildShared = async ({ const teamsRepository = new MongoTeamsRepository(); const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); const proxiesRepository = new MongoProxiesRepository(); + const egressStateRepository = new MongoEgressStateRepository(); // Inject settings repository into settings service (now that DB is connected) (settingsService as SettingsService).setRepository(settingsRepository); @@ -246,6 +250,7 @@ export const buildShared = async ({ teamsRepository, maintenanceWindowsRepository, proxiesRepository, + egressStateRepository, }; return sharedServices; }; diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index acd4a944a..83a875f05 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -28,6 +28,7 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { DBQueueWorker } from "@/worker/worker.db-queue.js"; import { ProxyResolver } from "@/service/network/ProxyResolver.js"; +import { IEgressService, EgressService } from "@/domain/egress/egress.service.js"; // Network providers import { PingProvider } from "@/service/network/PingProvider.js"; import { HttpProvider } from "@/service/network/HttpProvider.js"; @@ -46,6 +47,7 @@ export interface WorkerServices { networkService: INetworkService; bufferService: IBufferService; statusService: IStatusService; + egressService: IEgressService; } export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise => { @@ -70,6 +72,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig teamsRepository, maintenanceWindowsRepository, proxiesRepository, + egressStateRepository, } = shared; // *********************** @@ -103,6 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository); const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); const monitorStatusPolicy = new MonitorStatusPolicy(); + const egressService = new EgressService(settingsService, egressStateRepository, networkService, logger); // *********************** // Reactors and dispatcher @@ -185,5 +189,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig }); } - return { worker, networkService, bufferService, statusService }; + // Resume recovery polling if the instance was mid-episode when it last stopped + await egressService.init(); + + return { worker, networkService, bufferService, statusService, egressService }; }; diff --git a/server/src/domain/app-settings/app-settings.model.ts b/server/src/domain/app-settings/app-settings.model.ts index 3dd3c3abe..dfe138b8f 100644 --- a/server/src/domain/app-settings/app-settings.model.ts +++ b/server/src/domain/app-settings/app-settings.model.ts @@ -1,5 +1,6 @@ import { Schema, model, type Types } from "mongoose"; import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js"; +import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js"; interface AppSettingsDocument extends Omit { _id: Types.ObjectId; @@ -42,6 +43,10 @@ const AppSettingsSchema = new Schema( globalThresholds: { type: thresholdsSchema }, globalProxyEnabled: { type: Boolean, default: false }, globalProxyId: { type: String, default: null }, + egressCheckEnabled: { type: Boolean, default: false }, + egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] }, + egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS }, + egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId }, { timestamps: true } ); diff --git a/server/src/domain/app-settings/app-settings.repository.mongo.ts b/server/src/domain/app-settings/app-settings.repository.mongo.ts index d80289a7b..f2ef3351c 100644 --- a/server/src/domain/app-settings/app-settings.repository.mongo.ts +++ b/server/src/domain/app-settings/app-settings.repository.mongo.ts @@ -3,6 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js"; import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js"; import { toStringId, toDateString } from "@/utils/mongoMappers.js"; +import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js"; class MongoSettingsRepository implements ISettingsRepository { private toEntity = (doc: AppSettingsDocument): Settings => { @@ -30,6 +31,10 @@ class MongoSettingsRepository implements ISettingsRepository { globalThresholds: doc.globalThresholds ?? undefined, globalProxyEnabled: doc.globalProxyEnabled ?? false, globalProxyId: doc.globalProxyId ?? undefined, + egressCheckEnabled: doc.egressCheckEnabled ?? false, + egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS], + egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, + egressNotifications: doc.egressNotifications ?? [], createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; diff --git a/server/src/domain/app-settings/app-settings.type.ts b/server/src/domain/app-settings/app-settings.type.ts index e4d2f9559..9dc4eae45 100644 --- a/server/src/domain/app-settings/app-settings.type.ts +++ b/server/src/domain/app-settings/app-settings.type.ts @@ -51,6 +51,11 @@ export interface Settings { globalThresholds?: SettingsThresholds; globalProxyEnabled: boolean; globalProxyId?: string | null; + // Egress self-check (see domain/egress) + egressCheckEnabled: boolean; + egressCheckTargets: string[]; + egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule + egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert createdAt: string; updatedAt: string; } diff --git a/server/src/domain/egress/egress-state.model.ts b/server/src/domain/egress/egress-state.model.ts new file mode 100644 index 000000000..14f7cafe8 --- /dev/null +++ b/server/src/domain/egress/egress-state.model.ts @@ -0,0 +1,40 @@ +import { Schema, model, type Types } from "mongoose"; +import { EgressStatuses, type EgressProbeResult, type EgressState } from "@/domain/egress/egress.type.js"; + +interface EgressStateDocument extends Omit { + _id: Types.ObjectId; + singleton: boolean; + degradedSince: Date | null; + lastRecoveredAt: Date | null; + lastProbeAt: Date | null; + createdAt: Date; + updatedAt: Date; +} + +const probeResultSchema = new Schema( + { + target: { type: String, required: true }, + reachable: { type: Boolean, required: true }, + responseTime: { type: Number, default: 0 }, + message: { type: String }, + }, + { _id: false } +); + +const EgressStateSchema = new Schema( + { + singleton: { type: Boolean, required: true, unique: true, default: true }, + status: { type: String, enum: EgressStatuses, required: true, default: "ok" }, + degradedSince: { type: Date, default: null }, + lastRecoveredAt: { type: Date, default: null }, + lastProbeAt: { type: Date, default: null }, + lastProbeResults: { type: [probeResultSchema], default: () => [] }, + }, + { timestamps: true } +); + +const EgressStateModel = model("EgressState", EgressStateSchema); + +export type { EgressStateDocument }; +export { EgressStateModel }; +export default EgressStateModel; diff --git a/server/src/domain/egress/egress-state.repository.interface.ts b/server/src/domain/egress/egress-state.repository.interface.ts new file mode 100644 index 000000000..533aad656 --- /dev/null +++ b/server/src/domain/egress/egress-state.repository.interface.ts @@ -0,0 +1,12 @@ +import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js"; + +export interface IEgressStateRepository { + // read + findSingleton(): Promise; // creates the "ok" singleton on first call + // update + recordProbe(results: EgressProbeResult[], now: Date): Promise; + // Atomic transitions. Each returns the updated state when this caller performed the transition, + // or null when the state was already on the target side (another worker got there first). + markDegraded(results: EgressProbeResult[], now: Date): Promise; + markRecovered(results: EgressProbeResult[], now: Date): Promise; +} diff --git a/server/src/domain/egress/egress-state.repository.mongo.ts b/server/src/domain/egress/egress-state.repository.mongo.ts new file mode 100644 index 000000000..606551e5a --- /dev/null +++ b/server/src/domain/egress/egress-state.repository.mongo.ts @@ -0,0 +1,67 @@ +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js"; +import { EgressStateModel, type EgressStateDocument } from "@/domain/egress/egress-state.model.js"; +import { toStringId, toDateString } from "@/utils/mongoMappers.js"; + +const SERVICE_NAME = "EgressStateRepository"; + +class MongoEgressStateRepository implements IEgressStateRepository { + static SERVICE_NAME = SERVICE_NAME; + + private toEntity = (doc: EgressStateDocument): EgressState => { + return { + id: toStringId(doc._id), + status: doc.status, + degradedSince: doc.degradedSince ? toDateString(doc.degradedSince) : null, + lastRecoveredAt: doc.lastRecoveredAt ? toDateString(doc.lastRecoveredAt) : null, + lastProbeAt: doc.lastProbeAt ? toDateString(doc.lastProbeAt) : null, + lastProbeResults: (doc.lastProbeResults ?? []).map((result) => ({ + target: result.target, + reachable: result.reachable, + responseTime: result.responseTime ?? 0, + ...(result.message !== undefined && { message: result.message }), + })), + createdAt: toDateString(doc.createdAt), + updatedAt: toDateString(doc.updatedAt), + }; + }; + + findSingleton = async () => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $setOnInsert: { singleton: true, status: "ok" } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + }; + + recordProbe = async (results: EgressProbeResult[], now: Date) => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $set: { lastProbeAt: now, lastProbeResults: results }, $setOnInsert: { singleton: true, status: "ok" } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + }; + + markDegraded = async (results: EgressProbeResult[], now: Date) => { + // Only the caller that flips ok -> degraded sees a document back; everyone else sees null. + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true, status: "ok" }, + { $set: { status: "degraded", degradedSince: now, lastProbeAt: now, lastProbeResults: results } }, + { new: true } + ).lean(); + return doc ? this.toEntity(doc) : null; + }; + + markRecovered = async (results: EgressProbeResult[], now: Date) => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true, status: "degraded" }, + { $set: { status: "ok", lastRecoveredAt: now, lastProbeAt: now, lastProbeResults: results } }, + { new: true } + ).lean(); + return doc ? this.toEntity(doc) : null; + }; +} + +export default MongoEgressStateRepository; diff --git a/server/src/domain/egress/egress-state.service.ts b/server/src/domain/egress/egress-state.service.ts new file mode 100644 index 000000000..35800c7ac --- /dev/null +++ b/server/src/domain/egress/egress-state.service.ts @@ -0,0 +1,20 @@ +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { EgressState } from "@/domain/egress/egress.type.js"; + +export interface IEgressStateService { + getState(): Promise; +} + +const SERVICE_NAME = "EgressStateService"; + +// Read-only view of the instance-global egress state for the API process. Transitions are +// owned by the worker (egress.service.ts); this exists so the controller never touches the repository. +export class EgressStateService implements IEgressStateService { + static SERVICE_NAME = SERVICE_NAME; + + constructor(private egressStateRepository: IEgressStateRepository) {} + + getState = async (): Promise => { + return this.egressStateRepository.findSingleton(); + }; +} diff --git a/server/src/domain/egress/egress.service.ts b/server/src/domain/egress/egress.service.ts new file mode 100644 index 000000000..cb3b79ebb --- /dev/null +++ b/server/src/domain/egress/egress.service.ts @@ -0,0 +1,290 @@ +import type { Monitor } from "@/domain/monitors/monitor.type.js"; +import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; +import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { INetworkService } from "@/service/networkService.js"; +import type { ILogger } from "@/utils/logger.js"; +import { + DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, + DEFAULT_EGRESS_TARGETS, + type EgressProbeResult, + type EgressStatus, +} from "@/domain/egress/egress.type.js"; +import { timeRequest } from "@/service/network/utils.js"; + +const SERVICE_NAME = "EgressService"; +const PROBE_TIMEOUT_MS = 5000; + +// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs. +const PROBE_MONITOR_ID = "egress-probe"; +const PROBE_TEAM_ID = "system"; + +// "[host]:port" or "host:port" where host has no colons (so a bare IPv6 address is not mistaken for host:port). +const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/; + +export interface IEgressService { + probeTargets(targets: string[]): Promise; + assessAfterFailure(): Promise; + startRecoveryPolling(): void; + stop(): void; + init(): Promise; +} + +export class EgressService implements IEgressService { + static SERVICE_NAME = SERVICE_NAME; + + private polling = false; + private pollTimer: NodeJS.Timeout | null = null; + + constructor( + private settingsService: ISettingsService, + private egressStateRepository: IEgressStateRepository, + private networkService: INetworkService, + private logger: ILogger + ) {} + + // **************************** + // Probing + // **************************** + + // Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor. + private toProbeMonitor = (target: string): Monitor => { + const base = { + id: PROBE_MONITOR_ID, + teamId: PROBE_TEAM_ID, + name: `Egress probe: ${target}`, + method: "GET", + useAdvancedMatching: false, + ignoreTlsErrors: false, + customUpCodes: [], + }; + + if (/^https?:\/\//i.test(target)) { + return { ...base, type: "http", url: target } as unknown as Monitor; + } + + const hostPort = HOST_PORT_PATTERN.exec(target); + if (hostPort) { + const host = hostPort[1] ?? hostPort[2]; + return { ...base, type: "port", url: host, port: Number(hostPort[3]) } as unknown as Monitor; + } + + return { ...base, type: "ping", url: target } as unknown as Monitor; + }; + + private probeTarget = async (target: string): Promise => { + const monitor = this.toProbeMonitor(target); + let timer: NodeJS.Timeout | undefined; + + const { response, responseTime, error } = await timeRequest(() => + Promise.race([ + this.networkService.requestStatus(monitor), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS); + }), + ]).finally(() => clearTimeout(timer)) + ); + + if (error || !response) { + return { + target, + reachable: false, + responseTime, + message: error instanceof Error ? error.message : error ? String(error) : "No response", + }; + } + + return { + target, + reachable: response.status === true, + responseTime: response.responseTime ?? responseTime, + message: response.message, + }; + }; + + probeTargets = async (targets: string[]): Promise => { + return await Promise.all(targets.map((target) => this.probeTarget(target))); + }; + + private resolveTargets = (configured: string[] | undefined): string[] => { + const targets = (configured ?? []).map((target) => target.trim()).filter((target) => target.length > 0); + // An empty list would make "all unreachable" vacuously true, so fall back to the defaults. + return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS]; + }; + + // **************************** + // Event-triggered assessment + // **************************** + + assessAfterFailure = async (): Promise => { + try { + const settings = await this.settingsService.getDBSettings(); + if (!settings.egressCheckEnabled) { + return null; + } + + const state = await this.egressStateRepository.findSingleton(); + if (state.status === "degraded") { + // Recovery detection belongs to the poll loop. Make sure one is running in this process in case + // the worker that entered the degraded state is no longer around. + this.startRecoveryPolling(); + return "degraded"; + } + + const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); + const now = new Date(); + + if (results.some((result) => result.reachable)) { + await this.egressStateRepository.recordProbe(results, now); + return "ok"; + } + + const degraded = await this.egressStateRepository.markDegraded(results, now); + this.logger.warn({ + message: degraded + ? "Instance egress degraded: every reliability target is unreachable. Monitor failures will not be counted until it recovers" + : "Instance egress degraded (transition already recorded by another worker)", + service: SERVICE_NAME, + method: "assessAfterFailure", + details: { results }, + }); + this.startRecoveryPolling(); + return "degraded"; + } catch (error: unknown) { + // The egress check must never break check production; treat an internal failure as "unknown" and carry on. + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "assessAfterFailure", + stack: error instanceof Error ? error.stack : undefined, + }); + return null; + } + }; + + // **************************** + // Recovery poll loop + // **************************** + + startRecoveryPolling = () => { + if (this.polling) { + return; + } + this.polling = true; + this.logger.info({ message: "Starting egress recovery polling", service: SERVICE_NAME, method: "startRecoveryPolling" }); + void this.scheduleNextTick(); + }; + + stop = () => { + this.polling = false; + if (this.pollTimer) { + clearTimeout(this.pollTimer); + this.pollTimer = null; + } + }; + + private readPollIntervalMs = async () => { + try { + const settings = await this.settingsService.getDBSettings(); + const seconds = settings.egressPollIntervalSeconds; + return (Number.isFinite(seconds) && seconds > 0 ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000; + } catch (error: unknown) { + this.logger.warn({ + message: `Could not read egress poll interval, using default: ${error instanceof Error ? error.message : String(error)}`, + service: SERVICE_NAME, + method: "readPollIntervalMs", + }); + return DEFAULT_EGRESS_POLL_INTERVAL_SECONDS * 1000; + } + }; + + private scheduleNextTick = async () => { + if (!this.polling) { + return; + } + const intervalMs = await this.readPollIntervalMs(); + if (!this.polling) { + return; + } + this.pollTimer = setTimeout(() => void this.tick(), intervalMs); + // Never hold the process open for this loop; the state is persisted and init() resumes it on restart. + this.pollTimer.unref(); + }; + + private tick = async () => { + this.pollTimer = null; + if (!this.polling) { + return; + } + + try { + const state = await this.egressStateRepository.findSingleton(); + if (state.status !== "degraded") { + this.logger.info({ message: "Egress no longer degraded, stopping recovery polling", service: SERVICE_NAME, method: "tick" }); + this.stop(); + return; + } + + const settings = await this.settingsService.getDBSettings(); + const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); + const now = new Date(); + + if (!results.some((result) => result.reachable)) { + await this.egressStateRepository.recordProbe(results, now); + return; + } + + const recovered = await this.egressStateRepository.markRecovered(results, now); + this.stop(); + if (!recovered) { + // Another process performed the transition. + return; + } + + this.logger.info({ + message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`, + service: SERVICE_NAME, + method: "tick", + details: { results }, + }); + } catch (error: unknown) { + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "tick", + stack: error instanceof Error ? error.stack : undefined, + }); + } finally { + // No-op once stop() has been called; otherwise keep the loop alive through errors. + await this.scheduleNextTick(); + } + }; + + // **************************** + // Lifecycle + // **************************** + + init = async () => { + try { + const settings = await this.settingsService.getDBSettings(); + if (!settings.egressCheckEnabled) { + return; + } + const state = await this.egressStateRepository.findSingleton(); + if (state.status === "degraded") { + this.logger.warn({ + message: `Instance egress was degraded at startup (since ${state.degradedSince ?? "unknown"}), resuming recovery polling`, + service: SERVICE_NAME, + method: "init", + }); + this.startRecoveryPolling(); + } + } catch (error: unknown) { + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "init", + stack: error instanceof Error ? error.stack : undefined, + }); + } + }; +} diff --git a/server/src/domain/egress/egress.type.ts b/server/src/domain/egress/egress.type.ts new file mode 100644 index 000000000..386116ef7 --- /dev/null +++ b/server/src/domain/egress/egress.type.ts @@ -0,0 +1,33 @@ +// Egress self-check: detects loss of the instance's own outbound connectivity so that +// a burst of simultaneous monitor failures is not reported as a burst of real outages. + +export const EgressStatuses = ["ok", "degraded"] as const; +export type EgressStatus = (typeof EgressStatuses)[number]; + +// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect, +// http(s):// URLs are fetched. Any one target being reachable means egress is fine. +export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const; +export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30; +export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5; +export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600; +export const MAX_EGRESS_TARGETS = 10; + +// Instance-global singleton persisted in MongoDB so that every worker process and the API +// process share one view of the state (the queue worker may run separately from the API). +export interface EgressState { + id: string; + status: EgressStatus; + degradedSince: string | null; // ISO timestamp of the current or most recent degraded entry + lastRecoveredAt: string | null; // ISO timestamp of the most recent recovery + lastProbeAt: string | null; // ISO timestamp of the most recent reliability probe + lastProbeResults: EgressProbeResult[]; + createdAt: string; + updatedAt: string; +} + +export interface EgressProbeResult { + target: string; + reachable: boolean; + responseTime: number; // ms + message?: string; +} diff --git a/server/test/unit/services/egressService.test.ts b/server/test/unit/services/egressService.test.ts new file mode 100644 index 000000000..8c4fbf183 --- /dev/null +++ b/server/test/unit/services/egressService.test.ts @@ -0,0 +1,404 @@ +import { afterEach, describe, expect, it, jest } from "@jest/globals"; +import { EgressService } from "../../../src/domain/egress/egress.service.ts"; +import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const makeState = (overrides?: Partial): EgressState => ({ + id: "egress-1", + status: "ok", + degradedSince: null, + lastRecoveredAt: null, + lastProbeAt: null, + lastProbeResults: [], + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + ...overrides, +}); + +const makeSettings = (overrides?: Record) => ({ + egressCheckEnabled: true, + egressCheckTargets: ["1.1.1.1", "8.8.8.8"], + egressPollIntervalSeconds: 30, + egressNotifications: ["notif-1"], + ...overrides, +}); + +// requestStatus mock that answers per target url +const statusFor = (reachableTargets: string[]) => + jest.fn().mockImplementation(async (monitor: any) => ({ + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: reachableTargets.includes(monitor.url), + code: reachableTargets.includes(monitor.url) ? 200 : 5000, + message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed", + responseTime: 12, + })); + +const createService = (overrides?: Record) => { + const defaults = { + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) }, + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState()), + recordProbe: jest.fn().mockResolvedValue(makeState()), + markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), + markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })), + }, + networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) }, + logger: createMockLogger(), + ...overrides, + }; + const service = new EgressService( + defaults.settingsService as any, + defaults.egressStateRepository as any, + defaults.networkService as any, + defaults.logger as any + ); + return { service, defaults }; +}; + +// Lets the setTimeout chain in startRecoveryPolling settle (settings read, then timer armed) before advancing time. +const runOneTick = async (intervalMs: number) => { + await jest.advanceTimersByTimeAsync(0); + await jest.advanceTimersByTimeAsync(intervalMs); +}; + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("EgressService", () => { + afterEach(() => { + jest.useRealTimers(); + }); + + // ── probeTargets ────────────────────────────────────────────────────────── + + describe("probeTargets", () => { + it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => { + const { service, defaults } = createService(); + + await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443"]); + + const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any); + expect(monitors).toEqual([ + expect.objectContaining({ type: "ping", url: "1.1.1.1" }), + expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }), + expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }), + expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }), + ]); + }); + + it("reports reachable only when the provider says status is true", async () => { + const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } }); + + const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]); + + expect(results).toEqual([ + expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }), + expect.objectContaining({ target: "8.8.8.8", reachable: false }), + ]); + }); + + it("counts a thrown provider error as unreachable", async () => { + const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } }); + + const results = await service.probeTargets(["1.1.1.1"]); + + expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]); + }); + + it("counts a hung provider as unreachable once the probe timeout elapses", async () => { + jest.useFakeTimers(); + const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } }); + + const pending = service.probeTargets(["1.1.1.1"]); + await jest.advanceTimersByTimeAsync(5000); + const results = await pending; + + expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]); + }); + }); + + // ── assessAfterFailure ──────────────────────────────────────────────────── + + describe("assessAfterFailure", () => { + it("returns null and does not probe or record when the feature is disabled", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBeNull(); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + }); + + it("returns ok and records the probe when any target is reachable", async () => { + const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("ok"); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith( + [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })], + expect.any(Date) + ); + expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); + }); + + it("marks degraded, warns and returns degraded when every target is unreachable", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("degraded"); + expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith( + [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })], + expect.any(Date) + ); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") })); + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); + service.stop(); + }); + + it("returns degraded without probing when the persisted state is already degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("degraded"); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + service.stop(); + }); + + it("falls back to the default targets when the configured list is empty", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) }, + }); + + await service.assessAfterFailure(); + + const urls = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url); + expect(urls).toEqual(["1.1.1.1", "8.8.8.8"]); + }); + + it("returns null and logs when the assessment itself fails", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockRejectedValue(new Error("db down")) }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBeNull(); + expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" })); + }); + }); + + // ── recovery polling ────────────────────────────────────────────────────── + + describe("recovery polling", () => { + it("marks recovered once and stops polling when a target becomes reachable", async () => { + jest.useFakeTimers(); + const recovered = makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" }); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(recovered), + }, + networkService: { requestStatus: statusFor(["1.1.1.1"]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })); + + // Loop has stopped: a further interval produces no more work + await runOneTick(30_000); + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + }); + + it("does not log a recovery when another process already recorded it", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(null), + }, + networkService: { requestStatus: statusFor(["1.1.1.1"]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.logger.info).not.toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }) + ); + }); + + it("records the probe and keeps polling while every target stays unreachable", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); + expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); + service.stop(); + }); + + it("stops polling when the persisted state is no longer degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "ok" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + await runOneTick(30_000); + expect(defaults.egressStateRepository.findSingleton).toHaveBeenCalledTimes(1); + }); + + it("survives a failing tick and reschedules", async () => { + jest.useFakeTimers(); + const findSingleton = jest + .fn() + .mockRejectedValueOnce(new Error("db blip")) + .mockResolvedValue(makeState({ status: "degraded" })); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton, + recordProbe: jest.fn().mockResolvedValue(makeState()), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db blip", method: "tick" })); + + await runOneTick(30_000); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + + it("re-reads the poll interval from settings on each tick", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: 5 })) }, + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(5_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + + it("is idempotent: a second start does not arm a second loop", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + }); + + // ── init ────────────────────────────────────────────────────────────────── + + describe("init", () => { + it("resumes polling when the feature is enabled and the persisted state is degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(makeState()), + }, + }); + + await service.init(); + + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); + service.stop(); + }); + + it("does nothing when the persisted state is ok", async () => { + const { service, defaults } = createService(); + + await service.init(); + + expect(defaults.logger.info).not.toHaveBeenCalled(); + }); + + it("does nothing when the feature is disabled", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + }); + + await service.init(); + + expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/server/test/unit/services/egressStateService.test.ts b/server/test/unit/services/egressStateService.test.ts new file mode 100644 index 000000000..85c32f32d --- /dev/null +++ b/server/test/unit/services/egressStateService.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { EgressStateService } from "../../../src/domain/egress/egress-state.service.ts"; +import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const createEgressStateRepo = () => ({ + findSingleton: jest.fn(), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), +}); + +const createService = () => { + const egressStateRepository = createEgressStateRepo(); + const service = new EgressStateService(egressStateRepository as any); + return { service, egressStateRepository }; +}; + +const makeState = (overrides?: Partial): EgressState => ({ + id: "egress-1", + status: "ok", + degradedSince: null, + lastRecoveredAt: null, + lastProbeAt: null, + lastProbeResults: [], + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + ...overrides, +}); + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("EgressStateService", () => { + describe("getState", () => { + it("returns the singleton state from the repository", async () => { + const { service, egressStateRepository } = createService(); + const state = makeState({ + status: "degraded", + degradedSince: "2026-01-01T00:05:00Z", + lastProbeAt: "2026-01-01T00:05:00Z", + lastProbeResults: [{ target: "1.1.1.1", reachable: false, responseTime: 0, message: "timeout" }], + }); + (egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(state); + + const result = await service.getState(); + + expect(egressStateRepository.findSingleton).toHaveBeenCalledTimes(1); + expect(result).toEqual(state); + }); + + it("does not call any transition method", async () => { + const { service, egressStateRepository } = createService(); + (egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(makeState()); + + await service.getState(); + + expect(egressStateRepository.recordProbe).not.toHaveBeenCalled(); + expect(egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(egressStateRepository.markRecovered).not.toHaveBeenCalled(); + }); + + it("propagates repository errors", async () => { + const { service, egressStateRepository } = createService(); + (egressStateRepository.findSingleton as jest.Mock).mockRejectedValue(new Error("db down")); + + await expect(service.getState()).rejects.toThrow("db down"); + }); + }); +}); diff --git a/server/test/unit/validation/settingsValidation.test.ts b/server/test/unit/validation/settingsValidation.test.ts new file mode 100644 index 000000000..36e25c3b2 --- /dev/null +++ b/server/test/unit/validation/settingsValidation.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "@jest/globals"; +import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts"; +import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts"; + +const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8"; + +describe("settingsValidation", () => { + describe("updateAppSettingsBodyValidation egress fields", () => { + it("accepts a full set of valid egress settings", () => { + const parsed = updateAppSettingsBodyValidation.parse({ + egressCheckEnabled: true, + egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"], + egressPollIntervalSeconds: 30, + egressNotifications: [validNotificationId], + }); + + expect(parsed.egressCheckEnabled).toBe(true); + expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]); + expect(parsed.egressPollIntervalSeconds).toBe(30); + expect(parsed.egressNotifications).toEqual([validNotificationId]); + }); + + it("accepts a body with no egress fields", () => { + expect(() => updateAppSettingsBodyValidation.parse({ checkTTL: 30 })).not.toThrow(); + }); + + it("accepts an empty target list (the worker falls back to the default targets)", () => { + expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: [] }).egressCheckTargets).toEqual([]); + }); + + it("rejects more targets than the maximum", () => { + const targets = Array.from({ length: MAX_EGRESS_TARGETS + 1 }, (_, i) => `10.0.0.${i}`); + + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow(); + }); + + it("rejects targets that are not a host, host:port or http(s) URL", () => { + for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm"]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow(); + } + }); + + it("accepts the poll interval boundaries", () => { + expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe( + MIN_EGRESS_POLL_INTERVAL_SECONDS + ); + expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe( + MAX_EGRESS_POLL_INTERVAL_SECONDS + ); + }); + + it("rejects out-of-range and non-integer poll intervals", () => { + for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow(); + } + }); + + it("accepts an empty notification list", () => { + expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]); + }); + + it("rejects notification ids that are not 24-character hex", () => { + for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow(); + } + }); + + it("rejects a non-boolean enabled flag", () => { + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckEnabled: "yes" })).toThrow(); + }); + }); +});