diff --git a/server/openapi.json b/server/openapi.json index 4b4faf896..d0ae81f26 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -13294,6 +13294,31 @@ "globalProxyId": { "type": "string", "nullable": true + }, + "egressCheckEnabled": { + "type": "boolean" + }, + "egressCheckTargets": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^(https?:\\/\\/\\S+|[A-Za-z0-9.\\-:[\\]]+)$" + }, + "maxItems": 10 + }, + "egressPollIntervalSeconds": { + "type": "integer", + "minimum": 5, + "maximum": 600 + }, + "egressNotifications": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f\\d]{24}$/i" + } } } } diff --git a/server/src/api/validation/settingsValidation.ts b/server/src/api/validation/settingsValidation.ts index 50c75b043..33fe433fc 100644 --- a/server/src/api/validation/settingsValidation.ts +++ b/server/src/api/validation/settingsValidation.ts @@ -1,6 +1,15 @@ import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js"; +import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js"; import { z } from "zod"; +// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL. +const egressTargetValidation = z + .string() + .trim() + .min(1) + .max(253) + .regex(/^(https?:\/\/\S+|[A-Za-z0-9.\-:[\]]+)$/, "Invalid egress target"); + //**************************************** // Settings Validations //**************************************** @@ -42,6 +51,11 @@ export const updateAppSettingsBodyValidation = z .optional(), globalProxyEnabled: z.boolean().optional(), globalProxyId: z.string().nullable().optional(), + + egressCheckEnabled: z.boolean().optional(), + egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS + egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(), + egressNotifications: z.array(z.string().regex(/^[a-f\d]{24}$/i, "Invalid notification id")).optional(), }) .strip() .superRefine((body, ctx) => { diff --git a/server/src/config/services.api.ts b/server/src/config/services.api.ts index c5f58ac19..e929fbfe8 100644 --- a/server/src/config/services.api.ts +++ b/server/src/config/services.api.ts @@ -7,6 +7,7 @@ import { ITagsService, TagsService } from "@/domain/tags/tag.service.js"; import { IUserService, UserService } from "@/domain/users/user.service.js"; import { IJobScheduler } from "@/worker/worker.interface.js"; import { ProxiesService, IProxiesService } from "@/domain/proxies/proxy.service.js"; +import { EgressStateService, IEgressStateService } from "@/domain/egress/egress-state.service.js"; import { SharedServices } from "@/config/services.shared.js"; // Third-party @@ -24,6 +25,7 @@ export interface ApiServices extends SharedServices { tagsService: ITagsService; diagnosticService: IDiagnosticService; proxiesService: IProxiesService; + egressStateService: IEgressStateService; } export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => { @@ -48,6 +50,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A maintenanceWindowsRepository, jobsRepository, proxiesRepository, + egressStateRepository, } = shared; const userService = new UserService({ @@ -100,6 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A const tagsService = new TagsService(tagsRepository, monitorsRepository); const diagnosticService = new DiagnosticService(db); const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService); + const egressStateService = new EgressStateService(egressStateRepository); return { ...shared, worker: jobScheduler, @@ -111,5 +115,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A tagsService, diagnosticService, proxiesService, + egressStateService, }; }; diff --git a/server/src/config/services.shared.ts b/server/src/config/services.shared.ts index 3b4fdb147..a1a69566b 100644 --- a/server/src/config/services.shared.ts +++ b/server/src/config/services.shared.ts @@ -56,6 +56,7 @@ import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.js"; +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; // Mongo repository implementations import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js"; @@ -76,6 +77,7 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js"; import MongoTeamsRepository from "@/domain/teams/team.repository.model.js"; import MongoUsersRepository from "@/domain/users/user.repository.mongo.js"; import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js"; +import MongoEgressStateRepository from "@/domain/egress/egress-state.repository.mongo.js"; import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js"; // Shared infrastructure + business services that both the API and the worker process construct. @@ -114,6 +116,7 @@ export interface SharedServices { teamsRepository: ITeamsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; proxiesRepository: IProxiesRepository; + egressStateRepository: IEgressStateRepository; } export const buildShared = async ({ @@ -157,6 +160,7 @@ export const buildShared = async ({ const teamsRepository = new MongoTeamsRepository(); const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); const proxiesRepository = new MongoProxiesRepository(); + const egressStateRepository = new MongoEgressStateRepository(); // Inject settings repository into settings service (now that DB is connected) (settingsService as SettingsService).setRepository(settingsRepository); @@ -246,6 +250,7 @@ export const buildShared = async ({ teamsRepository, maintenanceWindowsRepository, proxiesRepository, + egressStateRepository, }; return sharedServices; }; diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index acd4a944a..83a875f05 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -28,6 +28,7 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { DBQueueWorker } from "@/worker/worker.db-queue.js"; import { ProxyResolver } from "@/service/network/ProxyResolver.js"; +import { IEgressService, EgressService } from "@/domain/egress/egress.service.js"; // Network providers import { PingProvider } from "@/service/network/PingProvider.js"; import { HttpProvider } from "@/service/network/HttpProvider.js"; @@ -46,6 +47,7 @@ export interface WorkerServices { networkService: INetworkService; bufferService: IBufferService; statusService: IStatusService; + egressService: IEgressService; } export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise => { @@ -70,6 +72,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig teamsRepository, maintenanceWindowsRepository, proxiesRepository, + egressStateRepository, } = shared; // *********************** @@ -103,6 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository); const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); const monitorStatusPolicy = new MonitorStatusPolicy(); + const egressService = new EgressService(settingsService, egressStateRepository, networkService, logger); // *********************** // Reactors and dispatcher @@ -185,5 +189,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig }); } - return { worker, networkService, bufferService, statusService }; + // Resume recovery polling if the instance was mid-episode when it last stopped + await egressService.init(); + + return { worker, networkService, bufferService, statusService, egressService }; }; diff --git a/server/src/domain/app-settings/app-settings.model.ts b/server/src/domain/app-settings/app-settings.model.ts index 3dd3c3abe..dfe138b8f 100644 --- a/server/src/domain/app-settings/app-settings.model.ts +++ b/server/src/domain/app-settings/app-settings.model.ts @@ -1,5 +1,6 @@ import { Schema, model, type Types } from "mongoose"; import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js"; +import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js"; interface AppSettingsDocument extends Omit { _id: Types.ObjectId; @@ -42,6 +43,10 @@ const AppSettingsSchema = new Schema( globalThresholds: { type: thresholdsSchema }, globalProxyEnabled: { type: Boolean, default: false }, globalProxyId: { type: String, default: null }, + egressCheckEnabled: { type: Boolean, default: false }, + egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] }, + egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS }, + egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId }, { timestamps: true } ); diff --git a/server/src/domain/app-settings/app-settings.repository.mongo.ts b/server/src/domain/app-settings/app-settings.repository.mongo.ts index d80289a7b..f2ef3351c 100644 --- a/server/src/domain/app-settings/app-settings.repository.mongo.ts +++ b/server/src/domain/app-settings/app-settings.repository.mongo.ts @@ -3,6 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js"; import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js"; import { toStringId, toDateString } from "@/utils/mongoMappers.js"; +import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js"; class MongoSettingsRepository implements ISettingsRepository { private toEntity = (doc: AppSettingsDocument): Settings => { @@ -30,6 +31,10 @@ class MongoSettingsRepository implements ISettingsRepository { globalThresholds: doc.globalThresholds ?? undefined, globalProxyEnabled: doc.globalProxyEnabled ?? false, globalProxyId: doc.globalProxyId ?? undefined, + egressCheckEnabled: doc.egressCheckEnabled ?? false, + egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS], + egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, + egressNotifications: doc.egressNotifications ?? [], createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; diff --git a/server/src/domain/app-settings/app-settings.type.ts b/server/src/domain/app-settings/app-settings.type.ts index e4d2f9559..9dc4eae45 100644 --- a/server/src/domain/app-settings/app-settings.type.ts +++ b/server/src/domain/app-settings/app-settings.type.ts @@ -51,6 +51,11 @@ export interface Settings { globalThresholds?: SettingsThresholds; globalProxyEnabled: boolean; globalProxyId?: string | null; + // Egress self-check (see domain/egress) + egressCheckEnabled: boolean; + egressCheckTargets: string[]; + egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule + egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert createdAt: string; updatedAt: string; } diff --git a/server/src/domain/egress/egress-state.model.ts b/server/src/domain/egress/egress-state.model.ts new file mode 100644 index 000000000..14f7cafe8 --- /dev/null +++ b/server/src/domain/egress/egress-state.model.ts @@ -0,0 +1,40 @@ +import { Schema, model, type Types } from "mongoose"; +import { EgressStatuses, type EgressProbeResult, type EgressState } from "@/domain/egress/egress.type.js"; + +interface EgressStateDocument extends Omit { + _id: Types.ObjectId; + singleton: boolean; + degradedSince: Date | null; + lastRecoveredAt: Date | null; + lastProbeAt: Date | null; + createdAt: Date; + updatedAt: Date; +} + +const probeResultSchema = new Schema( + { + target: { type: String, required: true }, + reachable: { type: Boolean, required: true }, + responseTime: { type: Number, default: 0 }, + message: { type: String }, + }, + { _id: false } +); + +const EgressStateSchema = new Schema( + { + singleton: { type: Boolean, required: true, unique: true, default: true }, + status: { type: String, enum: EgressStatuses, required: true, default: "ok" }, + degradedSince: { type: Date, default: null }, + lastRecoveredAt: { type: Date, default: null }, + lastProbeAt: { type: Date, default: null }, + lastProbeResults: { type: [probeResultSchema], default: () => [] }, + }, + { timestamps: true } +); + +const EgressStateModel = model("EgressState", EgressStateSchema); + +export type { EgressStateDocument }; +export { EgressStateModel }; +export default EgressStateModel; diff --git a/server/src/domain/egress/egress-state.repository.interface.ts b/server/src/domain/egress/egress-state.repository.interface.ts new file mode 100644 index 000000000..533aad656 --- /dev/null +++ b/server/src/domain/egress/egress-state.repository.interface.ts @@ -0,0 +1,12 @@ +import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js"; + +export interface IEgressStateRepository { + // read + findSingleton(): Promise; // creates the "ok" singleton on first call + // update + recordProbe(results: EgressProbeResult[], now: Date): Promise; + // Atomic transitions. Each returns the updated state when this caller performed the transition, + // or null when the state was already on the target side (another worker got there first). + markDegraded(results: EgressProbeResult[], now: Date): Promise; + markRecovered(results: EgressProbeResult[], now: Date): Promise; +} diff --git a/server/src/domain/egress/egress-state.repository.mongo.ts b/server/src/domain/egress/egress-state.repository.mongo.ts new file mode 100644 index 000000000..606551e5a --- /dev/null +++ b/server/src/domain/egress/egress-state.repository.mongo.ts @@ -0,0 +1,67 @@ +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js"; +import { EgressStateModel, type EgressStateDocument } from "@/domain/egress/egress-state.model.js"; +import { toStringId, toDateString } from "@/utils/mongoMappers.js"; + +const SERVICE_NAME = "EgressStateRepository"; + +class MongoEgressStateRepository implements IEgressStateRepository { + static SERVICE_NAME = SERVICE_NAME; + + private toEntity = (doc: EgressStateDocument): EgressState => { + return { + id: toStringId(doc._id), + status: doc.status, + degradedSince: doc.degradedSince ? toDateString(doc.degradedSince) : null, + lastRecoveredAt: doc.lastRecoveredAt ? toDateString(doc.lastRecoveredAt) : null, + lastProbeAt: doc.lastProbeAt ? toDateString(doc.lastProbeAt) : null, + lastProbeResults: (doc.lastProbeResults ?? []).map((result) => ({ + target: result.target, + reachable: result.reachable, + responseTime: result.responseTime ?? 0, + ...(result.message !== undefined && { message: result.message }), + })), + createdAt: toDateString(doc.createdAt), + updatedAt: toDateString(doc.updatedAt), + }; + }; + + findSingleton = async () => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $setOnInsert: { singleton: true, status: "ok" } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + }; + + recordProbe = async (results: EgressProbeResult[], now: Date) => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $set: { lastProbeAt: now, lastProbeResults: results }, $setOnInsert: { singleton: true, status: "ok" } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + }; + + markDegraded = async (results: EgressProbeResult[], now: Date) => { + // Only the caller that flips ok -> degraded sees a document back; everyone else sees null. + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true, status: "ok" }, + { $set: { status: "degraded", degradedSince: now, lastProbeAt: now, lastProbeResults: results } }, + { new: true } + ).lean(); + return doc ? this.toEntity(doc) : null; + }; + + markRecovered = async (results: EgressProbeResult[], now: Date) => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true, status: "degraded" }, + { $set: { status: "ok", lastRecoveredAt: now, lastProbeAt: now, lastProbeResults: results } }, + { new: true } + ).lean(); + return doc ? this.toEntity(doc) : null; + }; +} + +export default MongoEgressStateRepository; diff --git a/server/src/domain/egress/egress-state.service.ts b/server/src/domain/egress/egress-state.service.ts new file mode 100644 index 000000000..35800c7ac --- /dev/null +++ b/server/src/domain/egress/egress-state.service.ts @@ -0,0 +1,20 @@ +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { EgressState } from "@/domain/egress/egress.type.js"; + +export interface IEgressStateService { + getState(): Promise; +} + +const SERVICE_NAME = "EgressStateService"; + +// Read-only view of the instance-global egress state for the API process. Transitions are +// owned by the worker (egress.service.ts); this exists so the controller never touches the repository. +export class EgressStateService implements IEgressStateService { + static SERVICE_NAME = SERVICE_NAME; + + constructor(private egressStateRepository: IEgressStateRepository) {} + + getState = async (): Promise => { + return this.egressStateRepository.findSingleton(); + }; +} diff --git a/server/src/domain/egress/egress.service.ts b/server/src/domain/egress/egress.service.ts new file mode 100644 index 000000000..cb3b79ebb --- /dev/null +++ b/server/src/domain/egress/egress.service.ts @@ -0,0 +1,290 @@ +import type { Monitor } from "@/domain/monitors/monitor.type.js"; +import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; +import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { INetworkService } from "@/service/networkService.js"; +import type { ILogger } from "@/utils/logger.js"; +import { + DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, + DEFAULT_EGRESS_TARGETS, + type EgressProbeResult, + type EgressStatus, +} from "@/domain/egress/egress.type.js"; +import { timeRequest } from "@/service/network/utils.js"; + +const SERVICE_NAME = "EgressService"; +const PROBE_TIMEOUT_MS = 5000; + +// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs. +const PROBE_MONITOR_ID = "egress-probe"; +const PROBE_TEAM_ID = "system"; + +// "[host]:port" or "host:port" where host has no colons (so a bare IPv6 address is not mistaken for host:port). +const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/; + +export interface IEgressService { + probeTargets(targets: string[]): Promise; + assessAfterFailure(): Promise; + startRecoveryPolling(): void; + stop(): void; + init(): Promise; +} + +export class EgressService implements IEgressService { + static SERVICE_NAME = SERVICE_NAME; + + private polling = false; + private pollTimer: NodeJS.Timeout | null = null; + + constructor( + private settingsService: ISettingsService, + private egressStateRepository: IEgressStateRepository, + private networkService: INetworkService, + private logger: ILogger + ) {} + + // **************************** + // Probing + // **************************** + + // Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor. + private toProbeMonitor = (target: string): Monitor => { + const base = { + id: PROBE_MONITOR_ID, + teamId: PROBE_TEAM_ID, + name: `Egress probe: ${target}`, + method: "GET", + useAdvancedMatching: false, + ignoreTlsErrors: false, + customUpCodes: [], + }; + + if (/^https?:\/\//i.test(target)) { + return { ...base, type: "http", url: target } as unknown as Monitor; + } + + const hostPort = HOST_PORT_PATTERN.exec(target); + if (hostPort) { + const host = hostPort[1] ?? hostPort[2]; + return { ...base, type: "port", url: host, port: Number(hostPort[3]) } as unknown as Monitor; + } + + return { ...base, type: "ping", url: target } as unknown as Monitor; + }; + + private probeTarget = async (target: string): Promise => { + const monitor = this.toProbeMonitor(target); + let timer: NodeJS.Timeout | undefined; + + const { response, responseTime, error } = await timeRequest(() => + Promise.race([ + this.networkService.requestStatus(monitor), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS); + }), + ]).finally(() => clearTimeout(timer)) + ); + + if (error || !response) { + return { + target, + reachable: false, + responseTime, + message: error instanceof Error ? error.message : error ? String(error) : "No response", + }; + } + + return { + target, + reachable: response.status === true, + responseTime: response.responseTime ?? responseTime, + message: response.message, + }; + }; + + probeTargets = async (targets: string[]): Promise => { + return await Promise.all(targets.map((target) => this.probeTarget(target))); + }; + + private resolveTargets = (configured: string[] | undefined): string[] => { + const targets = (configured ?? []).map((target) => target.trim()).filter((target) => target.length > 0); + // An empty list would make "all unreachable" vacuously true, so fall back to the defaults. + return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS]; + }; + + // **************************** + // Event-triggered assessment + // **************************** + + assessAfterFailure = async (): Promise => { + try { + const settings = await this.settingsService.getDBSettings(); + if (!settings.egressCheckEnabled) { + return null; + } + + const state = await this.egressStateRepository.findSingleton(); + if (state.status === "degraded") { + // Recovery detection belongs to the poll loop. Make sure one is running in this process in case + // the worker that entered the degraded state is no longer around. + this.startRecoveryPolling(); + return "degraded"; + } + + const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); + const now = new Date(); + + if (results.some((result) => result.reachable)) { + await this.egressStateRepository.recordProbe(results, now); + return "ok"; + } + + const degraded = await this.egressStateRepository.markDegraded(results, now); + this.logger.warn({ + message: degraded + ? "Instance egress degraded: every reliability target is unreachable. Monitor failures will not be counted until it recovers" + : "Instance egress degraded (transition already recorded by another worker)", + service: SERVICE_NAME, + method: "assessAfterFailure", + details: { results }, + }); + this.startRecoveryPolling(); + return "degraded"; + } catch (error: unknown) { + // The egress check must never break check production; treat an internal failure as "unknown" and carry on. + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "assessAfterFailure", + stack: error instanceof Error ? error.stack : undefined, + }); + return null; + } + }; + + // **************************** + // Recovery poll loop + // **************************** + + startRecoveryPolling = () => { + if (this.polling) { + return; + } + this.polling = true; + this.logger.info({ message: "Starting egress recovery polling", service: SERVICE_NAME, method: "startRecoveryPolling" }); + void this.scheduleNextTick(); + }; + + stop = () => { + this.polling = false; + if (this.pollTimer) { + clearTimeout(this.pollTimer); + this.pollTimer = null; + } + }; + + private readPollIntervalMs = async () => { + try { + const settings = await this.settingsService.getDBSettings(); + const seconds = settings.egressPollIntervalSeconds; + return (Number.isFinite(seconds) && seconds > 0 ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000; + } catch (error: unknown) { + this.logger.warn({ + message: `Could not read egress poll interval, using default: ${error instanceof Error ? error.message : String(error)}`, + service: SERVICE_NAME, + method: "readPollIntervalMs", + }); + return DEFAULT_EGRESS_POLL_INTERVAL_SECONDS * 1000; + } + }; + + private scheduleNextTick = async () => { + if (!this.polling) { + return; + } + const intervalMs = await this.readPollIntervalMs(); + if (!this.polling) { + return; + } + this.pollTimer = setTimeout(() => void this.tick(), intervalMs); + // Never hold the process open for this loop; the state is persisted and init() resumes it on restart. + this.pollTimer.unref(); + }; + + private tick = async () => { + this.pollTimer = null; + if (!this.polling) { + return; + } + + try { + const state = await this.egressStateRepository.findSingleton(); + if (state.status !== "degraded") { + this.logger.info({ message: "Egress no longer degraded, stopping recovery polling", service: SERVICE_NAME, method: "tick" }); + this.stop(); + return; + } + + const settings = await this.settingsService.getDBSettings(); + const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); + const now = new Date(); + + if (!results.some((result) => result.reachable)) { + await this.egressStateRepository.recordProbe(results, now); + return; + } + + const recovered = await this.egressStateRepository.markRecovered(results, now); + this.stop(); + if (!recovered) { + // Another process performed the transition. + return; + } + + this.logger.info({ + message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`, + service: SERVICE_NAME, + method: "tick", + details: { results }, + }); + } catch (error: unknown) { + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "tick", + stack: error instanceof Error ? error.stack : undefined, + }); + } finally { + // No-op once stop() has been called; otherwise keep the loop alive through errors. + await this.scheduleNextTick(); + } + }; + + // **************************** + // Lifecycle + // **************************** + + init = async () => { + try { + const settings = await this.settingsService.getDBSettings(); + if (!settings.egressCheckEnabled) { + return; + } + const state = await this.egressStateRepository.findSingleton(); + if (state.status === "degraded") { + this.logger.warn({ + message: `Instance egress was degraded at startup (since ${state.degradedSince ?? "unknown"}), resuming recovery polling`, + service: SERVICE_NAME, + method: "init", + }); + this.startRecoveryPolling(); + } + } catch (error: unknown) { + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "init", + stack: error instanceof Error ? error.stack : undefined, + }); + } + }; +} diff --git a/server/src/domain/egress/egress.type.ts b/server/src/domain/egress/egress.type.ts new file mode 100644 index 000000000..386116ef7 --- /dev/null +++ b/server/src/domain/egress/egress.type.ts @@ -0,0 +1,33 @@ +// Egress self-check: detects loss of the instance's own outbound connectivity so that +// a burst of simultaneous monitor failures is not reported as a burst of real outages. + +export const EgressStatuses = ["ok", "degraded"] as const; +export type EgressStatus = (typeof EgressStatuses)[number]; + +// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect, +// http(s):// URLs are fetched. Any one target being reachable means egress is fine. +export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const; +export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30; +export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5; +export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600; +export const MAX_EGRESS_TARGETS = 10; + +// Instance-global singleton persisted in MongoDB so that every worker process and the API +// process share one view of the state (the queue worker may run separately from the API). +export interface EgressState { + id: string; + status: EgressStatus; + degradedSince: string | null; // ISO timestamp of the current or most recent degraded entry + lastRecoveredAt: string | null; // ISO timestamp of the most recent recovery + lastProbeAt: string | null; // ISO timestamp of the most recent reliability probe + lastProbeResults: EgressProbeResult[]; + createdAt: string; + updatedAt: string; +} + +export interface EgressProbeResult { + target: string; + reachable: boolean; + responseTime: number; // ms + message?: string; +} diff --git a/server/test/unit/services/egressService.test.ts b/server/test/unit/services/egressService.test.ts new file mode 100644 index 000000000..8c4fbf183 --- /dev/null +++ b/server/test/unit/services/egressService.test.ts @@ -0,0 +1,404 @@ +import { afterEach, describe, expect, it, jest } from "@jest/globals"; +import { EgressService } from "../../../src/domain/egress/egress.service.ts"; +import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const makeState = (overrides?: Partial): EgressState => ({ + id: "egress-1", + status: "ok", + degradedSince: null, + lastRecoveredAt: null, + lastProbeAt: null, + lastProbeResults: [], + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + ...overrides, +}); + +const makeSettings = (overrides?: Record) => ({ + egressCheckEnabled: true, + egressCheckTargets: ["1.1.1.1", "8.8.8.8"], + egressPollIntervalSeconds: 30, + egressNotifications: ["notif-1"], + ...overrides, +}); + +// requestStatus mock that answers per target url +const statusFor = (reachableTargets: string[]) => + jest.fn().mockImplementation(async (monitor: any) => ({ + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: reachableTargets.includes(monitor.url), + code: reachableTargets.includes(monitor.url) ? 200 : 5000, + message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed", + responseTime: 12, + })); + +const createService = (overrides?: Record) => { + const defaults = { + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) }, + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState()), + recordProbe: jest.fn().mockResolvedValue(makeState()), + markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), + markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })), + }, + networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) }, + logger: createMockLogger(), + ...overrides, + }; + const service = new EgressService( + defaults.settingsService as any, + defaults.egressStateRepository as any, + defaults.networkService as any, + defaults.logger as any + ); + return { service, defaults }; +}; + +// Lets the setTimeout chain in startRecoveryPolling settle (settings read, then timer armed) before advancing time. +const runOneTick = async (intervalMs: number) => { + await jest.advanceTimersByTimeAsync(0); + await jest.advanceTimersByTimeAsync(intervalMs); +}; + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("EgressService", () => { + afterEach(() => { + jest.useRealTimers(); + }); + + // ── probeTargets ────────────────────────────────────────────────────────── + + describe("probeTargets", () => { + it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => { + const { service, defaults } = createService(); + + await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443"]); + + const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any); + expect(monitors).toEqual([ + expect.objectContaining({ type: "ping", url: "1.1.1.1" }), + expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }), + expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }), + expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }), + ]); + }); + + it("reports reachable only when the provider says status is true", async () => { + const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } }); + + const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]); + + expect(results).toEqual([ + expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }), + expect.objectContaining({ target: "8.8.8.8", reachable: false }), + ]); + }); + + it("counts a thrown provider error as unreachable", async () => { + const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } }); + + const results = await service.probeTargets(["1.1.1.1"]); + + expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]); + }); + + it("counts a hung provider as unreachable once the probe timeout elapses", async () => { + jest.useFakeTimers(); + const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } }); + + const pending = service.probeTargets(["1.1.1.1"]); + await jest.advanceTimersByTimeAsync(5000); + const results = await pending; + + expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]); + }); + }); + + // ── assessAfterFailure ──────────────────────────────────────────────────── + + describe("assessAfterFailure", () => { + it("returns null and does not probe or record when the feature is disabled", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBeNull(); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + }); + + it("returns ok and records the probe when any target is reachable", async () => { + const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("ok"); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith( + [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })], + expect.any(Date) + ); + expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); + }); + + it("marks degraded, warns and returns degraded when every target is unreachable", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("degraded"); + expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith( + [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })], + expect.any(Date) + ); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") })); + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); + service.stop(); + }); + + it("returns degraded without probing when the persisted state is already degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("degraded"); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + service.stop(); + }); + + it("falls back to the default targets when the configured list is empty", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) }, + }); + + await service.assessAfterFailure(); + + const urls = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url); + expect(urls).toEqual(["1.1.1.1", "8.8.8.8"]); + }); + + it("returns null and logs when the assessment itself fails", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockRejectedValue(new Error("db down")) }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBeNull(); + expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" })); + }); + }); + + // ── recovery polling ────────────────────────────────────────────────────── + + describe("recovery polling", () => { + it("marks recovered once and stops polling when a target becomes reachable", async () => { + jest.useFakeTimers(); + const recovered = makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" }); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(recovered), + }, + networkService: { requestStatus: statusFor(["1.1.1.1"]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })); + + // Loop has stopped: a further interval produces no more work + await runOneTick(30_000); + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + }); + + it("does not log a recovery when another process already recorded it", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(null), + }, + networkService: { requestStatus: statusFor(["1.1.1.1"]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.logger.info).not.toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }) + ); + }); + + it("records the probe and keeps polling while every target stays unreachable", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); + expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); + service.stop(); + }); + + it("stops polling when the persisted state is no longer degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "ok" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + await runOneTick(30_000); + expect(defaults.egressStateRepository.findSingleton).toHaveBeenCalledTimes(1); + }); + + it("survives a failing tick and reschedules", async () => { + jest.useFakeTimers(); + const findSingleton = jest + .fn() + .mockRejectedValueOnce(new Error("db blip")) + .mockResolvedValue(makeState({ status: "degraded" })); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton, + recordProbe: jest.fn().mockResolvedValue(makeState()), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db blip", method: "tick" })); + + await runOneTick(30_000); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + + it("re-reads the poll interval from settings on each tick", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: 5 })) }, + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(5_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + + it("is idempotent: a second start does not arm a second loop", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + }); + + // ── init ────────────────────────────────────────────────────────────────── + + describe("init", () => { + it("resumes polling when the feature is enabled and the persisted state is degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(makeState()), + }, + }); + + await service.init(); + + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); + service.stop(); + }); + + it("does nothing when the persisted state is ok", async () => { + const { service, defaults } = createService(); + + await service.init(); + + expect(defaults.logger.info).not.toHaveBeenCalled(); + }); + + it("does nothing when the feature is disabled", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + }); + + await service.init(); + + expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/server/test/unit/services/egressStateService.test.ts b/server/test/unit/services/egressStateService.test.ts new file mode 100644 index 000000000..85c32f32d --- /dev/null +++ b/server/test/unit/services/egressStateService.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { EgressStateService } from "../../../src/domain/egress/egress-state.service.ts"; +import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const createEgressStateRepo = () => ({ + findSingleton: jest.fn(), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), +}); + +const createService = () => { + const egressStateRepository = createEgressStateRepo(); + const service = new EgressStateService(egressStateRepository as any); + return { service, egressStateRepository }; +}; + +const makeState = (overrides?: Partial): EgressState => ({ + id: "egress-1", + status: "ok", + degradedSince: null, + lastRecoveredAt: null, + lastProbeAt: null, + lastProbeResults: [], + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + ...overrides, +}); + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("EgressStateService", () => { + describe("getState", () => { + it("returns the singleton state from the repository", async () => { + const { service, egressStateRepository } = createService(); + const state = makeState({ + status: "degraded", + degradedSince: "2026-01-01T00:05:00Z", + lastProbeAt: "2026-01-01T00:05:00Z", + lastProbeResults: [{ target: "1.1.1.1", reachable: false, responseTime: 0, message: "timeout" }], + }); + (egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(state); + + const result = await service.getState(); + + expect(egressStateRepository.findSingleton).toHaveBeenCalledTimes(1); + expect(result).toEqual(state); + }); + + it("does not call any transition method", async () => { + const { service, egressStateRepository } = createService(); + (egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(makeState()); + + await service.getState(); + + expect(egressStateRepository.recordProbe).not.toHaveBeenCalled(); + expect(egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(egressStateRepository.markRecovered).not.toHaveBeenCalled(); + }); + + it("propagates repository errors", async () => { + const { service, egressStateRepository } = createService(); + (egressStateRepository.findSingleton as jest.Mock).mockRejectedValue(new Error("db down")); + + await expect(service.getState()).rejects.toThrow("db down"); + }); + }); +}); diff --git a/server/test/unit/validation/settingsValidation.test.ts b/server/test/unit/validation/settingsValidation.test.ts new file mode 100644 index 000000000..36e25c3b2 --- /dev/null +++ b/server/test/unit/validation/settingsValidation.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "@jest/globals"; +import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts"; +import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts"; + +const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8"; + +describe("settingsValidation", () => { + describe("updateAppSettingsBodyValidation egress fields", () => { + it("accepts a full set of valid egress settings", () => { + const parsed = updateAppSettingsBodyValidation.parse({ + egressCheckEnabled: true, + egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"], + egressPollIntervalSeconds: 30, + egressNotifications: [validNotificationId], + }); + + expect(parsed.egressCheckEnabled).toBe(true); + expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]); + expect(parsed.egressPollIntervalSeconds).toBe(30); + expect(parsed.egressNotifications).toEqual([validNotificationId]); + }); + + it("accepts a body with no egress fields", () => { + expect(() => updateAppSettingsBodyValidation.parse({ checkTTL: 30 })).not.toThrow(); + }); + + it("accepts an empty target list (the worker falls back to the default targets)", () => { + expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: [] }).egressCheckTargets).toEqual([]); + }); + + it("rejects more targets than the maximum", () => { + const targets = Array.from({ length: MAX_EGRESS_TARGETS + 1 }, (_, i) => `10.0.0.${i}`); + + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow(); + }); + + it("rejects targets that are not a host, host:port or http(s) URL", () => { + for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm"]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow(); + } + }); + + it("accepts the poll interval boundaries", () => { + expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe( + MIN_EGRESS_POLL_INTERVAL_SECONDS + ); + expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe( + MAX_EGRESS_POLL_INTERVAL_SECONDS + ); + }); + + it("rejects out-of-range and non-integer poll intervals", () => { + for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow(); + } + }); + + it("accepts an empty notification list", () => { + expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]); + }); + + it("rejects notification ids that are not 24-character hex", () => { + for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow(); + } + }); + + it("rejects a non-boolean enabled flag", () => { + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckEnabled: "yes" })).toThrow(); + }); + }); +});