Merge pull request #3959 from web-engineer/feat/egress-1-check-service

Egress self-check (1/4): egress state and probe service (#3937)

server/openapi.json

@@ -13294,6 +13294,30 @@
 									"globalProxyId": {
 										"type": "string",
 										"nullable": true
+									},
+									"egressCheckEnabled": {
+										"type": "boolean"
+									},
+									"egressCheckTargets": {
+										"type": "array",
+										"items": {
+											"type": "string",
+											"minLength": 1,
+											"maxLength": 253
+										},
+										"maxItems": 10
+									},
+									"egressPollIntervalSeconds": {
+										"type": "integer",
+										"minimum": 5,
+										"maximum": 600
+									},
+									"egressNotifications": {
+										"type": "array",
+										"items": {
+											"type": "string",
+											"pattern": "^[0-9a-fA-F]{24}$"
+										}
 									}
 								}
 							}

server/src/api/controllers/settingsController.ts

@@ -6,6 +6,7 @@ import { AppError } from "@/utils/AppError.js";
 import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
 import { IEmailService } from "@/service/emailService.js";
 import { IProxiesService } from "@/domain/proxies/proxy.service.js";
+import { IEgressStateService } from "@/domain/egress/egress-state.service.js";
 import { Settings } from "@/domain/app-settings/app-settings.type.js";
 
 export interface ISettingsController {
@@ -18,10 +19,17 @@ class SettingsController implements ISettingsController {
 	private settingsService: ISettingsService;
 	private emailService: IEmailService;
 	private proxiesService: IProxiesService;
-	constructor(settingsService: ISettingsService, emailService: IEmailService, proxiesService: IProxiesService) {
+	private egressStateService: IEgressStateService;
+	constructor(
+		settingsService: ISettingsService,
+		emailService: IEmailService,
+		proxiesService: IProxiesService,
+		egressStateService: IEgressStateService
+	) {
 		this.settingsService = settingsService;
 		this.emailService = emailService;
 		this.proxiesService = proxiesService;
+		this.egressStateService = egressStateService;
 	}
 
 	buildAppSettings = async (dbSettings: Settings) => {
@@ -70,7 +78,14 @@ class SettingsController implements ISettingsController {
 			}
 		}
 
+		const previousSettings = await this.settingsService.getDBSettings();
 		const updatedSettings = await this.settingsService.updateDbSettings(validatedBody);
+
+		// Switching the egress check on or off starts from a clean state: no degraded episode, no pending recovery job
+		if (validatedBody.egressCheckEnabled !== undefined && validatedBody.egressCheckEnabled !== previousSettings.egressCheckEnabled) {
+			await this.egressStateService.reset();
+		}
+
 		const returnSettings = await this.buildAppSettings(updatedSettings);
 		return res.status(200).json({
 			success: true,

server/src/api/validation/settingsValidation.ts

@@ -1,6 +1,17 @@
 import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
+import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
+import { parseEgressTarget } from "@/utils/egressTarget.js";
 import { z } from "zod";
 
+// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL.
+// Parsed with the same function the probe uses, so nothing the validator accepts can fail to probe.
+const egressTargetValidation = z
+	.string()
+	.trim()
+	.min(1)
+	.max(253)
+	.refine((target) => parseEgressTarget(target) !== null, "Enter a hostname or IP address, host:port, or http(s) URL");
+
 //****************************************
 // Settings Validations
 //****************************************
@@ -42,6 +53,11 @@ export const updateAppSettingsBodyValidation = z
 			.optional(),
 		globalProxyEnabled: z.boolean().optional(),
 		globalProxyId: z.string().nullable().optional(),
+
+		egressCheckEnabled: z.boolean().optional(),
+		egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
+		egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
+		egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(),
 	})
 	.strip()
 	.superRefine((body, ctx) => {

server/src/config/controllers.ts

@@ -36,7 +36,12 @@ export const initializeControllers = (apiServices: ApiServices): InitializedCont
 	return {
 		authController: new AuthController(apiServices.userService),
 		monitorController: new MonitorController(apiServices.monitorService, apiServices.notificationsService),
-		settingsController: new SettingsController(apiServices.settingsService, apiServices.emailService, apiServices.proxiesService),
+		settingsController: new SettingsController(
+			apiServices.settingsService,
+			apiServices.emailService,
+			apiServices.proxiesService,
+			apiServices.egressStateService
+		),
 		checkController: new CheckController(apiServices.checkService),
 		geoCheckController: new GeoCheckController(apiServices.geoChecksService),
 		inviteController: new InviteController(apiServices.inviteService),

server/src/config/services.api.ts

@@ -7,6 +7,7 @@ import { ITagsService, TagsService } from "@/domain/tags/tag.service.js";
 import { IUserService, UserService } from "@/domain/users/user.service.js";
 import { IJobScheduler } from "@/worker/worker.interface.js";
 import { ProxiesService, IProxiesService } from "@/domain/proxies/proxy.service.js";
+import { EgressStateService, IEgressStateService } from "@/domain/egress/egress-state.service.js";
 import { SharedServices } from "@/config/services.shared.js";
 
 // Third-party
@@ -24,6 +25,7 @@ export interface ApiServices extends SharedServices {
 	tagsService: ITagsService;
 	diagnosticService: IDiagnosticService;
 	proxiesService: IProxiesService;
+	egressStateService: IEgressStateService;
 }
 
 export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => {
@@ -48,6 +50,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
 		maintenanceWindowsRepository,
 		jobsRepository,
 		proxiesRepository,
+		egressStateRepository,
 	} = shared;
 
 	const userService = new UserService({
@@ -100,6 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
 	const tagsService = new TagsService(tagsRepository, monitorsRepository);
 	const diagnosticService = new DiagnosticService(db);
 	const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService);
+	const egressStateService = new EgressStateService(egressStateRepository, jobsRepository);
 	return {
 		...shared,
 		worker: jobScheduler,
@@ -111,5 +115,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
 		tagsService,
 		diagnosticService,
 		proxiesService,
+		egressStateService,
 	};
 };

server/src/config/services.shared.ts

@@ -56,6 +56,7 @@ import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js";
 import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
 import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
 import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.js";
+import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
 
 // Mongo repository implementations
 import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js";
@@ -76,6 +77,7 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js";
 import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
 import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
 import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js";
+import MongoEgressStateRepository from "@/domain/egress/egress-state.repository.mongo.js";
 import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
 
 // Shared infrastructure + business services that both the API and the worker process construct.
@@ -114,6 +116,7 @@ export interface SharedServices {
 	teamsRepository: ITeamsRepository;
 	maintenanceWindowsRepository: IMaintenanceWindowsRepository;
 	proxiesRepository: IProxiesRepository;
+	egressStateRepository: IEgressStateRepository;
 }
 
 export const buildShared = async ({
@@ -157,6 +160,7 @@ export const buildShared = async ({
 	const teamsRepository = new MongoTeamsRepository();
 	const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
 	const proxiesRepository = new MongoProxiesRepository();
+	const egressStateRepository = new MongoEgressStateRepository();
 
 	// Inject settings repository into settings service (now that DB is connected)
 	(settingsService as SettingsService).setRepository(settingsRepository);
@@ -246,6 +250,7 @@ export const buildShared = async ({
 		teamsRepository,
 		maintenanceWindowsRepository,
 		proxiesRepository,
+		egressStateRepository,
 	};
 	return sharedServices;
 };

server/src/config/services.worker.ts

@@ -28,6 +28,7 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
 import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
 import { DBQueueWorker } from "@/worker/worker.db-queue.js";
 import { ProxyResolver } from "@/service/network/ProxyResolver.js";
+import { IEgressService, EgressService } from "@/domain/egress/egress.service.js";
 // Network providers
 import { PingProvider } from "@/service/network/PingProvider.js";
 import { HttpProvider } from "@/service/network/HttpProvider.js";
@@ -46,6 +47,7 @@ export interface WorkerServices {
 	networkService: INetworkService;
 	bufferService: IBufferService;
 	statusService: IStatusService;
+	egressService: IEgressService;
 }
 
 export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise<WorkerServices> => {
@@ -70,12 +72,13 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
 		teamsRepository,
 		maintenanceWindowsRepository,
 		proxiesRepository,
+		egressStateRepository,
 	} = shared;
 
 	// ***********************
 	// Network providers
 	// ***********************
-	const pingProvider = new PingProvider(ping);
+	const pingProvider = new PingProvider(ping, net);
 	const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
 	const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
 	const hardwareProvider = new HardwareProvider(httpProvider);
@@ -103,6 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
 	const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
 	const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
 	const monitorStatusPolicy = new MonitorStatusPolicy();
+	const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger);
 
 	// ***********************
 	// Reactors and dispatcher
@@ -145,7 +149,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
 		checksRepository,
 		incidentsRepository,
 		geoChecksRepository,
-		dockerLogsRepository
+		dockerLogsRepository,
+		egressService
 	);
 
 	const worker = await DBQueueWorker.create({
@@ -185,5 +190,5 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
 		});
 	}
 
-	return { worker, networkService, bufferService, statusService };
+	return { worker, networkService, bufferService, statusService, egressService };
 };

server/src/domain/app-settings/app-settings-repository.interface.ts

@@ -6,6 +6,7 @@ export interface ISettingsRepository {
 	findSingleton(): Promise<Settings | null>;
 	// update
 	update(settings: SettingsUpdate): Promise<Settings>;
+	removeEgressNotification(notificationId: string): Promise<void>; // $pull, so it cannot race a concurrent settings update
 	// delete
 	deleteLegacy: () => Promise<boolean>;
 }

server/src/domain/app-settings/app-settings.model.ts

@@ -1,5 +1,6 @@
 import { Schema, model, type Types } from "mongoose";
 import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
+import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
 
 interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
 	_id: Types.ObjectId;
@@ -42,6 +43,10 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
 		globalThresholds: { type: thresholdsSchema },
 		globalProxyEnabled: { type: Boolean, default: false },
 		globalProxyId: { type: String, default: null },
+		egressCheckEnabled: { type: Boolean, default: false },
+		egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
+		egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
+		egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
 	},
 	{ timestamps: true }
 );

server/src/domain/app-settings/app-settings.repository.mongo.ts

@@ -3,6 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
 import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
 import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
 import { toStringId, toDateString } from "@/utils/mongoMappers.js";
+import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
 
 class MongoSettingsRepository implements ISettingsRepository {
 	private toEntity = (doc: AppSettingsDocument): Settings => {
@@ -30,6 +31,10 @@ class MongoSettingsRepository implements ISettingsRepository {
 			globalThresholds: doc.globalThresholds ?? undefined,
 			globalProxyEnabled: doc.globalProxyEnabled ?? false,
 			globalProxyId: doc.globalProxyId ?? undefined,
+			egressCheckEnabled: doc.egressCheckEnabled ?? false,
+			egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
+			egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
+			egressNotifications: doc.egressNotifications ?? [],
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
 		};
@@ -79,6 +84,10 @@ class MongoSettingsRepository implements ISettingsRepository {
 		return this.toEntity(updatedSettings);
 	};
 
+	removeEgressNotification = async (notificationId: string) => {
+		await AppSettingsModel.updateMany({ egressNotifications: notificationId }, { $pull: { egressNotifications: notificationId } });
+	};
+
 	deleteLegacy = async () => {
 		const res = await AppSettingsModel.deleteMany({ version: { $exists: false } });
 		return res.deletedCount > 0;

server/src/domain/app-settings/app-settings.service.ts

@@ -24,6 +24,7 @@ export interface ISettingsService {
 	areStatusPageThemesEnabled(): boolean;
 	getDBSettings(): Promise<Settings>;
 	updateDbSettings(newSettings: SettingsUpdate): Promise<Settings>;
+	removeEgressNotification(notificationId: string): Promise<void>;
 }
 
 export class SettingsService implements ISettingsService {
@@ -78,6 +79,10 @@ export class SettingsService implements ISettingsService {
 		return await this.getRepository().update(newSettings);
 	};
 
+	removeEgressNotification = async (notificationId: string) => {
+		await this.getRepository().removeEgressNotification(notificationId);
+	};
+
 	getDBSettings = async () => {
 		const repo = this.getRepository();
 		// Remove any old settings

server/src/domain/app-settings/app-settings.type.ts

@@ -51,6 +51,11 @@ export interface Settings {
 	globalThresholds?: SettingsThresholds;
 	globalProxyEnabled: boolean;
 	globalProxyId?: string | null;
+	// Egress self-check (see domain/egress)
+	egressCheckEnabled: boolean;
+	egressCheckTargets: string[];
+	egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
+	egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
 	createdAt: string;
 	updatedAt: string;
 }

server/src/domain/egress/egress-state.model.ts

@@ -0,0 +1,40 @@
+import { Schema, model, type Types } from "mongoose";
+import { EgressStatuses, type EgressProbeResult, type EgressState } from "@/domain/egress/egress.type.js";
+
+interface EgressStateDocument extends Omit<EgressState, "id" | "degradedSince" | "lastRecoveredAt" | "lastProbeAt" | "createdAt" | "updatedAt"> {
+	_id: Types.ObjectId;
+	singleton: boolean;
+	degradedSince: Date | null;
+	lastRecoveredAt: Date | null;
+	lastProbeAt: Date | null;
+	createdAt: Date;
+	updatedAt: Date;
+}
+
+const probeResultSchema = new Schema<EgressProbeResult>(
+	{
+		target: { type: String, required: true },
+		reachable: { type: Boolean, required: true },
+		responseTime: { type: Number, default: 0 },
+		message: { type: String },
+	},
+	{ _id: false }
+);
+
+const EgressStateSchema = new Schema<EgressStateDocument>(
+	{
+		singleton: { type: Boolean, required: true, unique: true, default: true },
+		status: { type: String, enum: EgressStatuses, required: true, default: "ok" },
+		degradedSince: { type: Date, default: null },
+		lastRecoveredAt: { type: Date, default: null },
+		lastProbeAt: { type: Date, default: null },
+		lastProbeResults: { type: [probeResultSchema], default: () => [] },
+	},
+	{ timestamps: true }
+);
+
+const EgressStateModel = model<EgressStateDocument>("EgressState", EgressStateSchema);
+
+export type { EgressStateDocument };
+export { EgressStateModel };
+export default EgressStateModel;

server/src/domain/egress/egress-state.repository.interface.ts

@@ -0,0 +1,13 @@
+import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
+
+export interface IEgressStateRepository {
+	// read
+	findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call, a plain read after that
+	// update
+	recordProbe(results: EgressProbeResult[], now: Date): Promise<EgressState>;
+	reset(): Promise<EgressState>; // back to "ok" with no episode; used when the feature is toggled
+	// Atomic transitions. Each returns the updated state when this caller performed the transition,
+	// or null when the state was already on the target side (another worker got there first).
+	markDegraded(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
+	markRecovered(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
+}

server/src/domain/egress/egress-state.repository.mongo.ts

@@ -0,0 +1,88 @@
+import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
+import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
+import { EgressStateModel, type EgressStateDocument } from "@/domain/egress/egress-state.model.js";
+import { toStringId, toDateString } from "@/utils/mongoMappers.js";
+
+const SERVICE_NAME = "EgressStateRepository";
+
+class MongoEgressStateRepository implements IEgressStateRepository {
+	static SERVICE_NAME = SERVICE_NAME;
+
+	private toEntity = (doc: EgressStateDocument): EgressState => {
+		return {
+			id: toStringId(doc._id),
+			status: doc.status,
+			degradedSince: doc.degradedSince ? toDateString(doc.degradedSince) : null,
+			lastRecoveredAt: doc.lastRecoveredAt ? toDateString(doc.lastRecoveredAt) : null,
+			lastProbeAt: doc.lastProbeAt ? toDateString(doc.lastProbeAt) : null,
+			lastProbeResults: (doc.lastProbeResults ?? []).map((result) => ({
+				target: result.target,
+				reachable: result.reachable,
+				responseTime: result.responseTime ?? 0,
+				...(result.message !== undefined && { message: result.message }),
+			})),
+			createdAt: toDateString(doc.createdAt),
+			updatedAt: toDateString(doc.updatedAt),
+		};
+	};
+
+	findSingleton = async () => {
+		// Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write.
+		const existing = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
+		if (existing) return this.toEntity(existing);
+		try {
+			const doc = await EgressStateModel.findOneAndUpdate(
+				{ singleton: true },
+				{ $setOnInsert: { singleton: true, status: "ok" } },
+				{ upsert: true, new: true, setDefaultsOnInsert: true }
+			).lean<EgressStateDocument>();
+			return this.toEntity(doc);
+		} catch (error: unknown) {
+			// Two processes can race the very first insert; the unique index rejects the loser, who reads the winner's row.
+			if ((error as { code?: number }).code === 11000) {
+				const doc = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
+				if (doc) return this.toEntity(doc);
+			}
+			throw error;
+		}
+	};
+
+	reset = async () => {
+		const doc = await EgressStateModel.findOneAndUpdate(
+			{ singleton: true },
+			{ $set: { status: "ok", degradedSince: null, lastProbeAt: null, lastProbeResults: [] }, $setOnInsert: { singleton: true } },
+			{ upsert: true, new: true, setDefaultsOnInsert: true }
+		).lean<EgressStateDocument>();
+		return this.toEntity(doc);
+	};
+
+	recordProbe = async (results: EgressProbeResult[], now: Date) => {
+		const doc = await EgressStateModel.findOneAndUpdate(
+			{ singleton: true },
+			{ $set: { lastProbeAt: now, lastProbeResults: results }, $setOnInsert: { singleton: true, status: "ok" } },
+			{ upsert: true, new: true, setDefaultsOnInsert: true }
+		).lean<EgressStateDocument>();
+		return this.toEntity(doc);
+	};
+
+	markDegraded = async (results: EgressProbeResult[], now: Date) => {
+		// Only the caller that flips ok -> degraded sees a document back; everyone else sees null.
+		const doc = await EgressStateModel.findOneAndUpdate(
+			{ singleton: true, status: "ok" },
+			{ $set: { status: "degraded", degradedSince: now, lastProbeAt: now, lastProbeResults: results } },
+			{ new: true }
+		).lean<EgressStateDocument>();
+		return doc ? this.toEntity(doc) : null;
+	};
+
+	markRecovered = async (results: EgressProbeResult[], now: Date) => {
+		const doc = await EgressStateModel.findOneAndUpdate(
+			{ singleton: true, status: "degraded" },
+			{ $set: { status: "ok", lastRecoveredAt: now, lastProbeAt: now, lastProbeResults: results } },
+			{ new: true }
+		).lean<EgressStateDocument>();
+		return doc ? this.toEntity(doc) : null;
+	};
+}
+
+export default MongoEgressStateRepository;

server/src/domain/egress/egress-state.service.ts

@@ -0,0 +1,31 @@
+import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
+import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
+import type { EgressState } from "@/domain/egress/egress.type.js";
+
+export interface IEgressStateService {
+	getState(): Promise<EgressState>;
+	reset(): Promise<EgressState>;
+}
+
+const SERVICE_NAME = "EgressStateService";
+
+// API-process view of the instance-global egress state. Probe-driven transitions are owned by the
+// worker (egress.service.ts); this exists so the controller never touches the repositories.
+export class EgressStateService implements IEgressStateService {
+	static SERVICE_NAME = SERVICE_NAME;
+
+	constructor(
+		private egressStateRepository: IEgressStateRepository,
+		private jobsRepository: IJobsRepository
+	) {}
+
+	getState = async (): Promise<EgressState> => {
+		return this.egressStateRepository.findSingleton();
+	};
+
+	// Called when the feature is switched on or off: drop any pending recovery job and start from "ok".
+	reset = async (): Promise<EgressState> => {
+		await this.jobsRepository.deleteGlobalJob("egress");
+		return this.egressStateRepository.reset();
+	};
+}

server/src/domain/egress/egress.service.ts

@@ -0,0 +1,283 @@
+import type { Monitor } from "@/domain/monitors/monitor.type.js";
+import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
+import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
+import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
+import { jobId, type Job, type JobSeed } from "@/domain/jobs/job.type.js";
+import type { INetworkService } from "@/service/networkService.js";
+import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
+import type { ILogger } from "@/utils/logger.js";
+import {
+	DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
+	DEFAULT_EGRESS_TARGETS,
+	type EgressAssessment,
+	type EgressProbeResult,
+	type EgressStatus,
+} from "@/domain/egress/egress.type.js";
+import { timeRequest } from "@/service/network/utils.js";
+import { parseEgressTarget, type EgressTarget } from "@/utils/egressTarget.js";
+
+const SERVICE_NAME = "EgressService";
+const PROBE_TIMEOUT_MS = 5000;
+// How long a settled assessment answers for later failures. An assessment still in flight is shared
+// for as long as it takes, so this only bounds how stale a reused answer can be.
+const ASSESSMENT_REUSE_MS = 5000;
+// Global job row (refId null) that re-probes while degraded. Inserted when egress degrades, removed on recovery.
+const RECOVERY_JOB_TYPE = "egress" as const;
+const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
+
+// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
+const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
+
+// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
+const PROBE_MONITOR_ID = "egress-probe";
+const PROBE_TEAM_ID = "system";
+
+export interface IEgressService {
+	probeTargets(targets: string[]): Promise<EgressProbeResult[]>;
+	// Called by the check producer when a check fails. Null when the feature is off or the assessment itself failed.
+	assessAfterFailure(): Promise<EgressStatus | null>;
+	// Handler for the "egress" job: re-probes while degraded and removes the job once egress is back.
+	checkRecovery(job: Job): Promise<void>;
+}
+
+type CachedAssessment = {
+	value: Promise<EgressAssessment | null>;
+	expiresAt: number | null; // null while the assessment is still in flight
+};
+
+export class EgressService implements IEgressService {
+	static SERVICE_NAME = SERVICE_NAME;
+
+	private assessment: CachedAssessment | null = null;
+
+	constructor(
+		private settingsService: ISettingsService,
+		private egressStateRepository: IEgressStateRepository,
+		private jobsRepository: IJobsRepository,
+		private networkService: INetworkService,
+		private proxyResolver: IProxyResolver,
+		private logger: ILogger
+	) {}
+
+	// ****************************
+	// Probing
+	// ****************************
+
+	// Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor.
+	// proxyMode "inherit" lets the resolver apply the global proxy to http probes, as it does for monitors.
+	private toProbeMonitor = (target: EgressTarget, raw: string): Monitor => {
+		const base = {
+			id: PROBE_MONITOR_ID,
+			teamId: PROBE_TEAM_ID,
+			name: `Egress probe: ${raw}`,
+			method: "GET",
+			proxyMode: "inherit",
+			useAdvancedMatching: false,
+			ignoreTlsErrors: false,
+			customUpCodes: [],
+		};
+
+		switch (target.kind) {
+			case "http":
+				return { ...base, type: "http", url: target.url } as unknown as Monitor;
+			case "port":
+				return { ...base, type: "port", url: target.host, port: target.port } as unknown as Monitor;
+			case "ping":
+				return { ...base, type: "ping", url: target.host } as unknown as Monitor;
+		}
+	};
+
+	private probeTarget = async (raw: string): Promise<EgressProbeResult> => {
+		const target = parseEgressTarget(raw);
+		if (!target) {
+			// Validation rejects these; a stored value can still be stale, and it must not count as reachable.
+			return { target: raw, reachable: false, responseTime: 0, message: "Invalid egress target" };
+		}
+
+		const monitor = this.toProbeMonitor(target, raw);
+		let timer: NodeJS.Timeout | undefined;
+
+		const { response, responseTime, error } = await timeRequest(async () => {
+			const proxyUrl = await this.proxyResolver.resolve(monitor);
+			return Promise.race([
+				this.networkService.requestStatus(monitor, { proxyUrl }),
+				new Promise<never>((_, reject) => {
+					timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS);
+				}),
+			]).finally(() => clearTimeout(timer));
+		});
+
+		if (error || !response) {
+			return {
+				target: raw,
+				reachable: false,
+				responseTime,
+				message: error instanceof Error ? error.message : error ? String(error) : "No response",
+			};
+		}
+
+		return {
+			target: raw,
+			// Any HTTP response, including 4xx/5xx, proves the instance can reach the target; only a transport
+			// failure counts against egress. Ping and port providers have no such distinction.
+			reachable: response.status === true || (target.kind === "http" && isHttpStatusCode(response.code)),
+			responseTime: response.responseTime ?? responseTime,
+			message: response.message,
+		};
+	};
+
+	probeTargets = async (targets: string[]): Promise<EgressProbeResult[]> => {
+		return await Promise.all(targets.map((target) => this.probeTarget(target)));
+	};
+
+	private resolveTargets = (configured: string[] | undefined): string[] => {
+		const targets = (configured ?? []).map((target) => target.trim()).filter((target) => target.length > 0);
+		// An empty list would make "all unreachable" vacuously true, so fall back to the defaults.
+		return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
+	};
+
+	private toIntervalMs = (seconds: number | undefined): number => {
+		const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
+		return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
+	};
+
+	// ****************************
+	// Event-triggered assessment
+	// ****************************
+
+	assessAfterFailure = async (): Promise<EgressStatus | null> => {
+		const assessment = await this.sharedAssessment();
+		return assessment === "disabled" ? null : assessment;
+	};
+
+	// When egress is down every monitor fails at once. Concurrent failures share the in-flight assessment
+	// however long it takes, and a settled one answers for ASSESSMENT_REUSE_MS. Same shape as ProxyResolver's cache.
+	private sharedAssessment = (): Promise<EgressAssessment | null> => {
+		const cached = this.assessment;
+		if (cached && (cached.expiresAt === null || cached.expiresAt > Date.now())) {
+			return cached.value;
+		}
+
+		const entry: CachedAssessment = { value: this.assess(), expiresAt: null };
+		this.assessment = entry;
+		void entry.value.then((result) => {
+			if (this.assessment !== entry) return;
+			if (result === null) {
+				this.assessment = null; // an internal failure is not cached, so the next failing check retries
+			} else {
+				entry.expiresAt = Date.now() + ASSESSMENT_REUSE_MS;
+			}
+		});
+		return entry.value;
+	};
+
+	private assess = async (): Promise<EgressAssessment | null> => {
+		try {
+			const settings = await this.settingsService.getDBSettings();
+			if (!settings.egressCheckEnabled) {
+				return "disabled";
+			}
+
+			const state = await this.egressStateRepository.findSingleton();
+			if (state.status === "degraded") {
+				// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
+				await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
+				return "degraded";
+			}
+
+			const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
+			const now = new Date();
+
+			if (results.some((result) => result.reachable)) {
+				await this.egressStateRepository.recordProbe(results, now);
+				return "ok";
+			}
+
+			// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
+			// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
+			await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
+			const degraded = await this.egressStateRepository.markDegraded(results, now);
+			this.logger.warn({
+				message: degraded
+					? "Instance egress degraded: every reliability target is unreachable. Monitor failures will not be counted until it recovers"
+					: "Instance egress degraded (transition already recorded by another worker)",
+				service: SERVICE_NAME,
+				method: "assessAfterFailure",
+				details: { results },
+			});
+			return "degraded";
+		} catch (error: unknown) {
+			// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
+			this.logger.error({
+				message: error instanceof Error ? error.message : String(error),
+				service: SERVICE_NAME,
+				method: "assessAfterFailure",
+				stack: error instanceof Error ? error.stack : undefined,
+			});
+			return null;
+		}
+	};
+
+	// ****************************
+	// Recovery job
+	// ****************************
+
+	// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
+	// so re-arming while degraded never pushes a pending run back.
+	private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
+		const intervalMs = this.toIntervalMs(pollIntervalSeconds);
+		const seed: JobSeed = {
+			id: RECOVERY_JOB_ID,
+			type: RECOVERY_JOB_TYPE,
+			refId: null,
+			isActive: true,
+			nextScheduledAt: Date.now() + intervalMs,
+			intervalMs,
+		};
+		if (reschedule) {
+			await this.jobsRepository.upsertCleanupJob(seed); // global-row upsert that also sets the schedule
+		} else {
+			await this.jobsRepository.upsertJob(seed);
+		}
+	};
+
+	// Removes the row only if nobody has rescheduled it since this run claimed it. Another worker may have seen
+	// egress fail again in the meantime and started a new episode; its schedule must survive.
+	private releaseRecoveryJob = async (job: Job) => {
+		await this.jobsRepository.deleteGlobalJobIfUnchanged(RECOVERY_JOB_TYPE, job.nextScheduledAt);
+	};
+
+	// Runs on the queue at the configured interval while degraded. Errors propagate so the queue records
+	// the failure and retries with its usual backoff.
+	checkRecovery = async (job: Job): Promise<void> => {
+		const settings = await this.settingsService.getDBSettings();
+		const state = await this.egressStateRepository.findSingleton();
+		if (!settings.egressCheckEnabled || state.status !== "degraded") {
+			// Feature switched off, recovery recorded by another worker, or state reset: nothing left to poll for.
+			await this.releaseRecoveryJob(job);
+			return;
+		}
+
+		const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
+		const now = new Date();
+
+		if (!results.some((result) => result.reachable)) {
+			await this.egressStateRepository.recordProbe(results, now);
+			return;
+		}
+
+		const recovered = await this.egressStateRepository.markRecovered(results, now);
+		await this.releaseRecoveryJob(job);
+		if (!recovered) {
+			// Another process performed the transition.
+			return;
+		}
+
+		this.logger.info({
+			message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`,
+			service: SERVICE_NAME,
+			method: "checkRecovery",
+			details: { results },
+		});
+	};
+}

server/src/domain/egress/egress.type.ts

@@ -0,0 +1,35 @@
+// Egress self-check: detects loss of the instance's own outbound connectivity so that
+// a burst of simultaneous monitor failures is not reported as a burst of real outages.
+
+export const EgressStatuses = ["ok", "degraded"] as const;
+export type EgressStatus = (typeof EgressStatuses)[number];
+// Outcome of one assessment: a status, or "disabled" when the feature is switched off (distinct from an internal failure).
+export type EgressAssessment = EgressStatus | "disabled";
+
+// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
+// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
+export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
+export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
+export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
+export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
+export const MAX_EGRESS_TARGETS = 10;
+
+// Instance-global singleton persisted in MongoDB so that every worker process and the API
+// process share one view of the state (the queue worker may run separately from the API).
+export interface EgressState {
+	id: string;
+	status: EgressStatus;
+	degradedSince: string | null; // ISO timestamp of the current or most recent degraded entry
+	lastRecoveredAt: string | null; // ISO timestamp of the most recent recovery
+	lastProbeAt: string | null; // ISO timestamp of the most recent reliability probe
+	lastProbeResults: EgressProbeResult[];
+	createdAt: string;
+	updatedAt: string;
+}
+
+export interface EgressProbeResult {
+	target: string;
+	reachable: boolean;
+	responseTime: number; // ms
+	message?: string;
+}

server/src/domain/jobs/job.repository.interface.ts

@@ -61,6 +61,11 @@ export interface IJobsRepository {
 	// Delete a single job of type
 	deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
 
+	// Global rows (refId null) are addressed by their canonical _id, see jobId
+	deleteGlobalJob(type: JobType): Promise<boolean>;
+	// Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged)
+	deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise<boolean>;
+
 	// Delete jobs that reference monitors that no longer exist
 	deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
 

server/src/domain/jobs/job.repository.mongo.ts

@@ -278,6 +278,16 @@ class MongoJobsRepository implements IJobsRepository {
 		return res.deletedCount > 0;
 	};
 
+	deleteGlobalJob = async (type: JobType) => {
+		const res = await JobModel.deleteOne({ _id: jobId(type, null) });
+		return res.deletedCount > 0;
+	};
+
+	deleteGlobalJobIfUnchanged = async (type: JobType, nextScheduledAt: number) => {
+		const res = await JobModel.deleteOne({ _id: jobId(type, null), nextScheduledAt });
+		return res.deletedCount > 0;
+	};
+
 	deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
 		// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
 		const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });

server/src/domain/jobs/job.type.ts

@@ -1,10 +1,10 @@
-export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
+export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"] as const;
 export type JobType = (typeof JOB_TYPES)[number];
 export const LOCK_MS = 60_000;
 export const BACKOFF_MS = 5_000;
 export const PARKED = Number.MAX_SAFE_INTEGER;
 
-// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
+// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
 export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
 
 export type Job = {

server/src/domain/notifications/notification.service.ts

@@ -179,6 +179,9 @@ export class NotificationsService implements INotificationsService {
 	deleteById = async (id: string, teamId: string): Promise<Notification> => {
 		await this.monitorsRepository.removeNotificationFromMonitors(id);
 		const deleted = await this.notificationsRepository.deleteById(id, teamId);
+		// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles.
+		// After the team-scoped delete, so a request for another team's id (404 above) cannot touch instance settings.
+		await this.settingsService.removeEgressNotification(id);
 		return deleted;
 	};
 }

server/src/service/network/PingProvider.ts

@@ -4,25 +4,33 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js";
 import { MonitorStatusResponse } from "@/types/network.js";
 import { AppError } from "@/utils/AppError.js";
 import ping from "ping";
+import * as net from "net";
 import { timeRequest } from "@/service/network/utils.js";
 const SERVICE_NAME = "PingProvider";
 
 type Ping = typeof ping;
+type NetType = typeof net;
 
 export class PingProvider implements IStatusProvider<PingStatusPayload> {
 	readonly type = "ping";
 
-	constructor(private ping: Ping) {}
+	constructor(
+		private ping: Ping,
+		private net: NetType
+	) {}
 
 	supports(type: MonitorType): boolean {
 		return type === "ping";
 	}
 
 	private sanitizeHost(url: string): string {
-		return url
-			.replace(/^https?:\/\//, "")
-			.replace(/\/.*$/, "")
-			.replace(/:.*/, "");
+		const host = url.replace(/^https?:\/\//, "").replace(/\/.*$/, "");
+		// "[2001:db8::1]" or "[2001:db8::1]:53": the address is inside the brackets
+		const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host);
+		if (bracketed) return bracketed[1]!;
+		// A bare IPv6 address contains colons that are not a port separator
+		if (this.net.isIPv6(host)) return host;
+		return host.replace(/:.*/, "");
 	}
 
 	async handle(monitor: Monitor): Promise<MonitorStatusResponse<PingStatusPayload>> {

server/src/utils/egressTarget.ts

@@ -0,0 +1,48 @@
+import { z } from "zod";
+import { dnsHostnameRegex } from "@/api/validation/shared.js";
+
+// A reliability target for the egress self-check, in one of three shapes. Parsed here so the
+// settings validator and the probe agree on exactly what is accepted.
+export type EgressTarget =
+	| { kind: "http"; url: string } // http(s):// URL, fetched
+	| { kind: "port"; host: string; port: number } // host:port or [ipv6]:port, TCP connect
+	| { kind: "ping"; host: string }; // bare hostname or IP address, ICMP
+
+// "[ipv6]:port" or "host:port" where host has no colons, so a bare IPv6 address is never read as host:port.
+const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s[\]]+)):(\d{1,5})$/;
+const BRACKETED_PATTERN = /^\[([^\]]+)\]$/;
+
+const isIpv6 = (value: string): boolean => z.ipv6().safeParse(value).success;
+const isHostOrIpv4 = (value: string): boolean => value === "localhost" || z.ipv4().safeParse(value).success || dnsHostnameRegex.test(value);
+
+const parseHttp = (target: string): EgressTarget | null => {
+	try {
+		const { protocol, hostname } = new URL(target);
+		return (protocol === "http:" || protocol === "https:") && hostname.length > 0 ? { kind: "http", url: target } : null;
+	} catch {
+		return null;
+	}
+};
+
+export const parseEgressTarget = (raw: string): EgressTarget | null => {
+	const target = raw.trim();
+	if (target.length === 0) return null;
+
+	if (/^https?:\/\//i.test(target)) return parseHttp(target);
+
+	const hostPort = HOST_PORT_PATTERN.exec(target);
+	if (hostPort) {
+		const port = Number(hostPort[3]);
+		if (port < 1 || port > 65535) return null;
+		const bracketedHost = hostPort[1];
+		if (bracketedHost !== undefined) return isIpv6(bracketedHost) ? { kind: "port", host: bracketedHost, port } : null;
+		const host = hostPort[2]!;
+		return isHostOrIpv4(host) ? { kind: "port", host, port } : null;
+	}
+
+	const bracketed = BRACKETED_PATTERN.exec(target);
+	if (bracketed) return isIpv6(bracketed[1]!) ? { kind: "ping", host: bracketed[1]! } : null;
+
+	if (isIpv6(target) || isHostOrIpv4(target)) return { kind: "ping", host: target };
+	return null;
+};

server/src/worker/worker.db-queue.ts

@@ -28,6 +28,7 @@ const CONCURRENCY: Record<JobType, number> = {
 	evaluate: 20,
 	"cleanup-orphaned": 1,
 	"cleanup-retention": 1,
+	egress: 1,
 };
 
 export interface DBQueueWorkerDependencies {
@@ -61,6 +62,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
 		evaluate: 0,
 		"cleanup-orphaned": 0,
 		"cleanup-retention": 0,
+		egress: 0,
 	};
 
 	private checksRepository: IChecksRepository;
@@ -175,6 +177,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
 				case "cleanup-retention":
 					await this.helper.getCleanupRetentionJob()(); // Get job and execute
 					break;
+				case "egress":
+					await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
+					break;
 			}
 
 			if (job.intervalMs === null) {

server/src/worker/worker.helper.ts

@@ -11,10 +11,13 @@ import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
 import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
 import { ILogger } from "@/utils/logger.js";
 import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
+import { IEgressService } from "@/domain/egress/egress.service.js";
+import type { Job } from "@/domain/jobs/job.type.js";
 
 export interface IWorkerHelper {
 	getCleanupOrphanedJob(): () => Promise<void>;
 	getCleanupRetentionJob(): () => Promise<void>;
+	getEgressRecoveryJob(): (job: Job) => Promise<void>;
 }
 
 export interface MonitorActionDecision {
@@ -45,6 +48,7 @@ export class WorkerHelper implements IWorkerHelper {
 	private incidentsRepository: IIncidentsRepository;
 	private geoChecksRepository: IGeoChecksRepository;
 	private dockerLogsRepository: IDockerLogsRepository;
+	private egressService: IEgressService;
 
 	constructor(
 		logger: ILogger,
@@ -57,7 +61,8 @@ export class WorkerHelper implements IWorkerHelper {
 		checksRepository: IChecksRepository,
 		incidentsRepository: IIncidentsRepository,
 		geoChecksRepository: IGeoChecksRepository,
-		dockerLogsRepository: IDockerLogsRepository
+		dockerLogsRepository: IDockerLogsRepository,
+		egressService: IEgressService
 	) {
 		this.logger = logger;
 		this.checkService = checkService;
@@ -70,8 +75,17 @@ export class WorkerHelper implements IWorkerHelper {
 		this.incidentsRepository = incidentsRepository;
 		this.geoChecksRepository = geoChecksRepository;
 		this.dockerLogsRepository = dockerLogsRepository;
+		this.egressService = egressService;
 	}
 
+	// Runs while instance egress is degraded; the service removes the row once egress is back.
+	// Errors propagate so the queue records the failure and retries with its usual backoff.
+	getEgressRecoveryJob = () => {
+		return async (job: Job) => {
+			await this.egressService.checkRecovery(job);
+		};
+	};
+
 	getCleanupOrphanedJob = () => {
 		return async () => {
 			try {

server/src/worker/worker.job-scheduler.ts

@@ -41,6 +41,7 @@ export class JobScheduler implements IJobScheduler {
 		evaluate: POLL_MS,
 		"cleanup-orphaned": POLL_MS,
 		"cleanup-retention": POLL_MS,
+		egress: POLL_MS,
 	};
 	// Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd.
 	protected toCheckJob = (monitor: MonitorScheduleFields, now: number, immediate = false): JobSeed => ({

server/test/integration/egressStateRepository.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
+import mongoose from "mongoose";
+import { MongoMemoryServer } from "mongodb-memory-server";
+import MongoEgressStateRepository from "../../src/domain/egress/egress-state.repository.mongo.ts";
+import { EgressStateModel } from "../../src/domain/egress/egress-state.model.ts";
+
+// ── Real-Mongo harness ─────────────────────────────────────────────────────────
+// The transitions here are the guarantees several worker processes rely on: exactly one
+// caller flips ok -> degraded (and degraded -> ok), and a plain read never writes. Both are
+// MongoDB filter/update semantics, so they need a live engine.
+
+let mongod: MongoMemoryServer;
+
+beforeAll(async () => {
+	mongod = await MongoMemoryServer.create();
+	await mongoose.connect(mongod.getUri());
+	await EgressStateModel.init(); // unique index on singleton
+}, 120_000);
+
+afterAll(async () => {
+	await mongoose.disconnect();
+	await mongod.stop();
+});
+
+beforeEach(async () => {
+	await EgressStateModel.deleteMany({});
+});
+
+const probe = (target: string, reachable: boolean) => ({ target, reachable, responseTime: 5, message: reachable ? "Success" : "timeout" });
+const NOW = new Date("2026-01-01T10:00:00.000Z");
+const LATER = new Date("2026-01-01T10:05:00.000Z");
+
+describe("MongoEgressStateRepository", () => {
+	const repo = new MongoEgressStateRepository();
+
+	describe("findSingleton", () => {
+		it("creates the ok singleton on first call and returns the same document afterwards", async () => {
+			const first = await repo.findSingleton();
+			const second = await repo.findSingleton();
+
+			expect(first.status).toBe("ok");
+			expect(second.id).toBe(first.id);
+			expect(await EgressStateModel.countDocuments()).toBe(1);
+		});
+
+		it("does not bump updatedAt on a read", async () => {
+			const created = await repo.findSingleton();
+			await new Promise((resolve) => setTimeout(resolve, 20));
+
+			const read = await repo.findSingleton();
+
+			expect(read.updatedAt).toBe(created.updatedAt);
+		});
+
+		it("survives two processes racing the first insert", async () => {
+			const [a, b] = await Promise.all([repo.findSingleton(), repo.findSingleton()]);
+
+			expect(a.id).toBe(b.id);
+			expect(await EgressStateModel.countDocuments()).toBe(1);
+		});
+	});
+
+	describe("markDegraded / markRecovered", () => {
+		it("markDegraded does nothing when the singleton has never been created (callers read it first)", async () => {
+			expect(await repo.markDegraded([probe("1.1.1.1", false)], NOW)).toBeNull();
+			expect(await EgressStateModel.countDocuments()).toBe(0);
+		});
+
+		it("lets exactly one caller perform each transition", async () => {
+			await repo.findSingleton();
+
+			const [d1, d2] = await Promise.all([repo.markDegraded([probe("1.1.1.1", false)], NOW), repo.markDegraded([probe("1.1.1.1", false)], NOW)]);
+			expect([d1, d2].filter((result) => result !== null)).toHaveLength(1);
+			expect((await repo.findSingleton()).status).toBe("degraded");
+
+			const [r1, r2] = await Promise.all([repo.markRecovered([probe("1.1.1.1", true)], LATER), repo.markRecovered([probe("1.1.1.1", true)], LATER)]);
+			expect([r1, r2].filter((result) => result !== null)).toHaveLength(1);
+			const state = await repo.findSingleton();
+			expect(state.status).toBe("ok");
+			expect(state.degradedSince).toBe(NOW.toISOString());
+			expect(state.lastRecoveredAt).toBe(LATER.toISOString());
+		});
+
+		it("markRecovered is a no-op while ok and markDegraded is a no-op while degraded", async () => {
+			await repo.findSingleton();
+			expect(await repo.markRecovered([probe("1.1.1.1", true)], NOW)).toBeNull();
+
+			await repo.markDegraded([probe("1.1.1.1", false)], NOW);
+			expect(await repo.markDegraded([probe("1.1.1.1", false)], LATER)).toBeNull();
+			expect((await repo.findSingleton()).degradedSince).toBe(NOW.toISOString());
+		});
+	});
+
+	describe("recordProbe", () => {
+		it("stores the probe without changing the status", async () => {
+			await repo.findSingleton();
+			await repo.markDegraded([probe("1.1.1.1", false)], NOW);
+
+			const state = await repo.recordProbe([probe("1.1.1.1", false), probe("8.8.8.8", false)], LATER);
+
+			expect(state.status).toBe("degraded");
+			expect(state.lastProbeAt).toBe(LATER.toISOString());
+			expect(state.lastProbeResults.map((result) => result.target)).toEqual(["1.1.1.1", "8.8.8.8"]);
+		});
+	});
+
+	describe("reset", () => {
+		it("returns a degraded singleton to ok with no episode and keeps the recovery history", async () => {
+			await repo.findSingleton();
+			await repo.markDegraded([probe("1.1.1.1", false)], NOW);
+			await repo.markRecovered([probe("1.1.1.1", true)], LATER);
+			await repo.markDegraded([probe("1.1.1.1", false)], LATER);
+
+			const state = await repo.reset();
+
+			expect(state.status).toBe("ok");
+			expect(state.degradedSince).toBeNull();
+			expect(state.lastProbeAt).toBeNull();
+			expect(state.lastProbeResults).toEqual([]);
+			expect(state.lastRecoveredAt).toBe(LATER.toISOString());
+		});
+
+		it("creates the singleton when none exists", async () => {
+			const state = await repo.reset();
+
+			expect(state.status).toBe("ok");
+			expect(await EgressStateModel.countDocuments()).toBe(1);
+		});
+	});
+});

server/test/integration/jobsRepository.test.ts

@@ -533,4 +533,47 @@ describe("MongoJobsRepository", () => {
 			expect(await repo.countDueBacklog(NOW)).toBe(0);
 		});
 	});
+
+	// ── global rows: egress recovery job ──────────────────────────────────────
+
+	describe("global rows", () => {
+		const repo = new MongoJobsRepository(WORKER_ID);
+		const egressRow = (nextScheduledAt: number) =>
+			seedJob({ _id: "egress", type: "egress" as JobType, refId: null, nextScheduledAt, intervalMs: 30_000 });
+
+		it("deleteGlobalJob removes the row by its canonical _id and leaves other global rows alone", async () => {
+			await egressRow(NOW);
+			await seedJob({ _id: "cleanup-orphaned", type: "cleanup-orphaned" as JobType, refId: null });
+
+			expect(await repo.deleteGlobalJob("egress")).toBe(true);
+
+			expect(await readRow("egress")).toBeNull();
+			expect(await readRow("cleanup-orphaned")).not.toBeNull();
+			expect(await repo.deleteGlobalJob("egress")).toBe(false);
+		});
+
+		it("deleteGlobalJobIfUnchanged removes the row only while nextScheduledAt matches the claimed value", async () => {
+			await egressRow(NOW);
+
+			// Another worker started a new episode and rescheduled the row after this run claimed it
+			await JobModel.updateOne({ _id: "egress" }, { $set: { nextScheduledAt: NOW + 30_000 } });
+			expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW)).toBe(false);
+			expect(await readRow("egress")).not.toBeNull();
+
+			// Nobody touched it: the row goes
+			expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW + 30_000)).toBe(true);
+			expect(await readRow("egress")).toBeNull();
+		});
+
+		it("recordSuccess after a refused conditional delete is a no-op once the lock has moved on", async () => {
+			await egressRow(NOW);
+			const [claimed] = await repo.claimDueBatch("egress", 1, NOW);
+			expect(claimed?.id).toBe("egress");
+
+			// Row deleted by the handler (recovered), then recordSuccess runs from the queue: nothing to update
+			expect(await repo.deleteGlobalJobIfUnchanged("egress", claimed!.nextScheduledAt)).toBe(true);
+			expect(await repo.recordSuccess("egress", claimed!.nextScheduledAt, 30_000, NOW + 1_000)).toBe(false);
+			expect(await readRow("egress")).toBeNull();
+		});
+	});
 });

server/test/unit/providers/network/pingProvider.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
 import { PingProvider } from "../../../../src/service/network/PingProvider.ts";
 import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts";
 import type { Monitor } from "../../../../src/domain/monitors/monitor.type.ts";
+import * as net from "net";
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
 
@@ -27,7 +28,7 @@ const createMockPing = (result?: Partial<{ alive: boolean; time: number | string
 // ── Contract ─────────────────────────────────────────────────────────────────
 
 testStatusProviderContract("PingProvider", {
-	create: () => new PingProvider(createMockPing() as any),
+	create: () => new PingProvider(createMockPing() as any, net),
 	supportedType: "ping",
 	unsupportedType: "http",
 	makeMonitor: () => makeMonitor(),
@@ -37,7 +38,7 @@ testStatusProviderContract("PingProvider", {
 
 describe("PingProvider", () => {
 	it("returns success when host is alive", async () => {
-		const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any);
+		const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any, net);
 
 		const result = await provider.handle(makeMonitor());
 
@@ -55,7 +56,7 @@ describe("PingProvider", () => {
 	});
 
 	it("returns failure when host is not alive", async () => {
-		const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any);
+		const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any, net);
 
 		const result = await provider.handle(makeMonitor());
 
@@ -69,7 +70,7 @@ describe("PingProvider", () => {
 	});
 
 	it("parses string time values", async () => {
-		const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any);
+		const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any, net);
 
 		const result = await provider.handle(makeMonitor());
 
@@ -77,7 +78,7 @@ describe("PingProvider", () => {
 	});
 
 	it("defaults responseTime to 0 for non-numeric time", async () => {
-		const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any);
+		const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any, net);
 
 		const result = await provider.handle(makeMonitor());
 
@@ -85,7 +86,7 @@ describe("PingProvider", () => {
 	});
 
 	it("defaults status to false when alive is undefined", async () => {
-		const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any);
+		const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any, net);
 
 		const result = await provider.handle(makeMonitor());
 
@@ -94,7 +95,7 @@ describe("PingProvider", () => {
 
 	it("sanitizes URL by stripping protocol and path", async () => {
 		const mockPing = createMockPing();
-		const provider = new PingProvider(mockPing as any);
+		const provider = new PingProvider(mockPing as any, net);
 
 		await provider.handle(makeMonitor({ url: "https://example.com/path?query=1" }));
 
@@ -103,36 +104,54 @@ describe("PingProvider", () => {
 
 	it("sanitizes URL by stripping port", async () => {
 		const mockPing = createMockPing();
-		const provider = new PingProvider(mockPing as any);
+		const provider = new PingProvider(mockPing as any, net);
 
 		await provider.handle(makeMonitor({ url: "example.com:8080" }));
 
 		expect(mockPing.promise.probe).toHaveBeenCalledWith("example.com");
 	});
 
+	it("keeps a bare IPv6 address intact", async () => {
+		const mockPing = createMockPing();
+		const provider = new PingProvider(mockPing as any, net);
+
+		await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" }));
+
+		expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
+	});
+
+	it("unwraps a bracketed IPv6 address and drops its port", async () => {
+		const mockPing = createMockPing();
+		const provider = new PingProvider(mockPing as any, net);
+
+		await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" }));
+
+		expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
+	});
+
 	it("throws AppError when url is missing", async () => {
-		const provider = new PingProvider(createMockPing() as any);
+		const provider = new PingProvider(createMockPing() as any, net);
 
 		await expect(provider.handle(makeMonitor({ url: "" }))).rejects.toThrow("URL is required for ping monitor");
 	});
 
 	it("throws AppError when ping.probe rejects", async () => {
 		const mockPing = { promise: { probe: jest.fn().mockRejectedValue(new Error("Host unreachable")) } };
-		const provider = new PingProvider(mockPing as any);
+		const provider = new PingProvider(mockPing as any, net);
 
 		await expect(provider.handle(makeMonitor())).rejects.toThrow("Host unreachable");
 	});
 
 	it("throws AppError when probe returns no response", async () => {
 		const mockPing = { promise: { probe: jest.fn().mockResolvedValue(null) } };
-		const provider = new PingProvider(mockPing as any);
+		const provider = new PingProvider(mockPing as any, net);
 
 		await expect(provider.handle(makeMonitor())).rejects.toThrow("No response from ping");
 	});
 
 	it("throws AppError with stringified message for non-Error throws", async () => {
 		const mockPing = { promise: { probe: jest.fn().mockRejectedValue(42) } };
-		const provider = new PingProvider(mockPing as any);
+		const provider = new PingProvider(mockPing as any, net);
 
 		await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
 	});

server/test/unit/services/dbQueueWorker.test.ts

@@ -9,7 +9,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
 // ── Notes ──────────────────────────────────────────────────────────────────────
 //
 // DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
-// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
+// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and
 // renews their locks, and registers itself in the worker heartbeat. The repository
 // claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
 // test only the orchestration around it, with the repository fully mocked.
@@ -113,6 +113,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
 	const helper = {
 		getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
 		getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
+		getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
 	};
 	const queueWorkersRepository = {
 		upsert: jest.fn<any>().mockResolvedValue(undefined),
@@ -226,7 +227,7 @@ describe("DBQueueWorker", () => {
 			const { mocks } = await start({ queueMode: "worker" });
 
 			const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
-			expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
+			expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"]));
 			expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
 			expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
 			expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
@@ -440,6 +441,20 @@ describe("DBQueueWorker", () => {
 			expect(retentionFn).toHaveBeenCalled();
 		});
 
+		it("egress job runs the helper's egress recovery function with the claimed job", async () => {
+			const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
+			const egressFn = jest.fn<any>().mockResolvedValue(undefined);
+			const helper = {
+				getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
+				getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
+				getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
+			};
+			const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
+
+			expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
+			expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
+		});
+
 		it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
 			const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
 			const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });

server/test/unit/services/egressService.test.ts

@@ -0,0 +1,472 @@
+import { afterEach, describe, expect, it, jest } from "@jest/globals";
+import { EgressService } from "../../../src/domain/egress/egress.service.ts";
+import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
+import type { Job } from "../../../src/domain/jobs/job.type.ts";
+import { createMockLogger } from "../../helpers/createMockLogger.ts";
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+const makeState = (overrides?: Partial<EgressState>): EgressState => ({
+	id: "egress-1",
+	status: "ok",
+	degradedSince: null,
+	lastRecoveredAt: null,
+	lastProbeAt: null,
+	lastProbeResults: [],
+	createdAt: "2026-01-01T00:00:00.000Z",
+	updatedAt: "2026-01-01T00:00:00.000Z",
+	...overrides,
+});
+
+const makeSettings = (overrides?: Record<string, unknown>) => ({
+	egressCheckEnabled: true,
+	egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
+	egressPollIntervalSeconds: 30,
+	egressNotifications: ["notif-1"],
+	...overrides,
+});
+
+// The claimed "egress" job row as the queue hands it to the handler
+const makeJob = (overrides?: Partial<Job>): Job => ({
+	id: "egress",
+	type: "egress",
+	refId: null,
+	isActive: true,
+	nextScheduledAt: 1_000,
+	intervalMs: 30_000,
+	lockedBy: "worker-1",
+	lockedUntil: 61_000,
+	runCount: 0,
+	failCount: 0,
+	lastFinishedAt: null,
+	lastFailReason: null,
+	...overrides,
+});
+
+// requestStatus mock that answers per target url
+const statusFor = (reachableTargets: string[]) =>
+	jest.fn().mockImplementation(async (monitor: any) => ({
+		monitorId: monitor.id,
+		teamId: monitor.teamId,
+		type: monitor.type,
+		status: reachableTargets.includes(monitor.url),
+		code: reachableTargets.includes(monitor.url) ? 200 : 5000,
+		message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed",
+		responseTime: 12,
+	}));
+
+const degradedRepository = (overrides?: Record<string, any>) => ({
+	findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
+	recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
+	markDegraded: jest.fn().mockResolvedValue(null),
+	markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
+	reset: jest.fn(),
+	...overrides,
+});
+
+const createService = (overrides?: Record<string, any>) => {
+	const defaults = {
+		settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) },
+		egressStateRepository: {
+			findSingleton: jest.fn().mockResolvedValue(makeState()),
+			recordProbe: jest.fn().mockResolvedValue(makeState()),
+			markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
+			markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })),
+			reset: jest.fn(),
+		},
+		jobsRepository: {
+			upsertJob: jest.fn().mockResolvedValue(true),
+			upsertCleanupJob: jest.fn().mockResolvedValue(true),
+			deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true),
+		},
+		networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) },
+		proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
+		logger: createMockLogger(),
+		...overrides,
+	};
+	const service = new EgressService(
+		defaults.settingsService as any,
+		defaults.egressStateRepository as any,
+		defaults.jobsRepository as any,
+		defaults.networkService as any,
+		defaults.proxyResolver as any,
+		defaults.logger as any
+	);
+	return { service, defaults };
+};
+
+const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) =>
+	(defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
+
+// ── Tests ────────────────────────────────────────────────────────────────────
+
+describe("EgressService", () => {
+	afterEach(() => {
+		jest.useRealTimers();
+	});
+
+	// ── probeTargets ──────────────────────────────────────────────────────────
+
+	describe("probeTargets", () => {
+		it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => {
+			const { service, defaults } = createService();
+
+			await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]);
+
+			const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any);
+			expect(monitors).toEqual([
+				expect.objectContaining({ type: "ping", url: "1.1.1.1" }),
+				expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }),
+				expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }),
+				expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }),
+				expect.objectContaining({ type: "ping", url: "2606:4700::1111" }),
+			]);
+		});
+
+		it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => {
+			const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128");
+			const { service, defaults } = createService({ proxyResolver: { resolve } });
+
+			await service.probeTargets(["https://example.com/health"]);
+
+			expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" }));
+			expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), {
+				proxyUrl: "http://proxy.internal:3128",
+			});
+		});
+
+		it("reports reachable only when the provider says status is true", async () => {
+			const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } });
+
+			const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]);
+
+			expect(results).toEqual([
+				expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }),
+				expect.objectContaining({ target: "8.8.8.8", reachable: false }),
+			]);
+		});
+
+		it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => {
+			const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({
+				monitorId: monitor.id,
+				teamId: monitor.teamId,
+				type: monitor.type,
+				status: false,
+				code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR
+				message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED",
+				responseTime: 3,
+			}));
+			const { service } = createService({ networkService: { requestStatus } });
+
+			const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]);
+
+			expect(results).toEqual([
+				expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }),
+				expect.objectContaining({ target: "https://example.com/refused", reachable: false }),
+				expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts
+			]);
+		});
+
+		it("counts an unparseable stored target as unreachable without probing", async () => {
+			const { service, defaults } = createService();
+
+			const results = await service.probeTargets(["[1.1.1.1"]);
+
+			expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]);
+			expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
+		});
+
+		it("counts a thrown provider error as unreachable", async () => {
+			const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } });
+
+			const results = await service.probeTargets(["1.1.1.1"]);
+
+			expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]);
+		});
+
+		it("counts a hung provider as unreachable once the probe timeout elapses", async () => {
+			jest.useFakeTimers();
+			const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
+
+			const pending = service.probeTargets(["1.1.1.1"]);
+			await jest.advanceTimersByTimeAsync(5000);
+			const results = await pending;
+
+			expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]);
+		});
+	});
+
+	// ── assessAfterFailure ────────────────────────────────────────────────────
+
+	describe("assessAfterFailure", () => {
+		it("returns null and does not probe or record when the feature is disabled", async () => {
+			const { service, defaults } = createService({
+				settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
+			});
+
+			const result = await service.assessAfterFailure();
+
+			expect(result).toBeNull();
+			expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
+			expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled();
+			expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
+		});
+
+		it("returns ok and records the probe when any target is reachable", async () => {
+			const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
+
+			const result = await service.assessAfterFailure();
+
+			expect(result).toBe("ok");
+			expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith(
+				[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })],
+				expect.any(Date)
+			);
+			expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
+			expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
+			expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
+		});
+
+		it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => {
+			const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } });
+
+			const result = await service.assessAfterFailure();
+
+			expect(result).toBe("degraded");
+			expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith(
+				[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
+				expect.any(Date)
+			);
+			expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
+			expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
+			expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
+				id: "egress",
+				type: "egress",
+				refId: null,
+				isActive: true,
+				nextScheduledAt: expect.any(Number),
+				intervalMs: 30_000,
+			});
+			expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
+			// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
+			const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!;
+			const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!;
+			expect(scheduleOrder).toBeLessThan(transitionOrder);
+		});
+
+		it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => {
+			const { service, defaults } = createService({
+				networkService: { requestStatus: statusFor([]) },
+				egressStateRepository: degradedRepository({
+					findSingleton: jest.fn().mockResolvedValue(makeState()),
+					markDegraded: jest.fn().mockResolvedValue(null),
+				}),
+			});
+
+			const result = await service.assessAfterFailure();
+
+			expect(result).toBe("degraded");
+			expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
+			expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
+		});
+
+		it("does not mark degraded when the recovery job cannot be scheduled", async () => {
+			const { service, defaults } = createService({
+				networkService: { requestStatus: statusFor([]) },
+				jobsRepository: {
+					upsertJob: jest.fn(),
+					upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")),
+					deleteGlobalJobIfUnchanged: jest.fn(),
+				},
+			});
+
+			expect(await service.assessAfterFailure()).toBeNull();
+			expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
+		});
+
+		it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
+			const { service, defaults } = createService({ egressStateRepository: degradedRepository() });
+
+			const result = await service.assessAfterFailure();
+
+			expect(result).toBe("degraded");
+			expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
+			expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
+			expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
+			expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
+			expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
+		});
+
+		it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
+			const { service, defaults } = createService({
+				settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
+				networkService: { requestStatus: statusFor([]) },
+			});
+
+			await service.assessAfterFailure();
+
+			expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
+		});
+
+		it("falls back to the default targets when the configured list is empty", async () => {
+			const { service, defaults } = createService({
+				settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
+			});
+
+			await service.assessAfterFailure();
+
+			expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]);
+		});
+
+		it("returns null and logs when the assessment itself fails", async () => {
+			const { service, defaults } = createService({
+				settingsService: { getDBSettings: jest.fn().mockRejectedValue(new Error("db down")) },
+			});
+
+			const result = await service.assessAfterFailure();
+
+			expect(result).toBeNull();
+			expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" }));
+		});
+	});
+
+	// ── shared assessment ─────────────────────────────────────────────────────
+
+	describe("shared assessment", () => {
+		it("shares one probe between concurrent failures", async () => {
+			const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
+
+			const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
+
+			expect(results).toEqual(["ok", "ok", "ok"]);
+			expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
+			expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
+			expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
+		});
+
+		it("keeps sharing an in-flight assessment that outlives the reuse window", async () => {
+			jest.useFakeTimers();
+			const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
+
+			const first = service.assessAfterFailure();
+			await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call
+			const second = service.assessAfterFailure();
+			await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle
+
+			expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]);
+			expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds
+		});
+
+		it("reuses a settled assessment within the reuse window and probes again after it", async () => {
+			jest.useFakeTimers();
+			const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
+
+			await service.assessAfterFailure();
+			await jest.advanceTimersByTimeAsync(1_000);
+			await service.assessAfterFailure();
+			expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
+
+			await jest.advanceTimersByTimeAsync(5_000);
+			await service.assessAfterFailure();
+			expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
+		});
+
+		it("reuses a disabled result instead of re-reading settings for every failure", async () => {
+			const getDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false }));
+			const { service } = createService({ settingsService: { getDBSettings } });
+
+			expect(await service.assessAfterFailure()).toBeNull();
+			expect(await service.assessAfterFailure()).toBeNull();
+
+			expect(getDBSettings).toHaveBeenCalledTimes(1);
+		});
+
+		it("does not cache an internal failure", async () => {
+			const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
+			const { service, defaults } = createService({ settingsService: { getDBSettings } });
+
+			expect(await service.assessAfterFailure()).toBeNull();
+			expect(await service.assessAfterFailure()).toBe("ok");
+			expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
+		});
+	});
+
+	// ── checkRecovery (the "egress" job) ──────────────────────────────────────
+
+	describe("checkRecovery", () => {
+		it("marks recovered, releases the job and logs when a target becomes reachable", async () => {
+			const { service, defaults } = createService({
+				egressStateRepository: degradedRepository(),
+				networkService: { requestStatus: statusFor(["1.1.1.1"]) },
+			});
+			const job = makeJob({ nextScheduledAt: 123_456 });
+
+			await service.checkRecovery(job);
+
+			expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith(
+				[expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
+				expect.any(Date)
+			);
+			// Conditional on the schedule this run claimed, so a concurrent new episode keeps its job
+			expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456);
+			expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }));
+		});
+
+		it("releases the job but does not log a recovery when another process already recorded it", async () => {
+			const { service, defaults } = createService({
+				egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }),
+				networkService: { requestStatus: statusFor(["1.1.1.1"]) },
+			});
+
+			await service.checkRecovery(makeJob());
+
+			expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
+			expect(defaults.logger.info).not.toHaveBeenCalledWith(
+				expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })
+			);
+		});
+
+		it("records the probe and keeps the job while every target stays unreachable", async () => {
+			const { service, defaults } = createService({
+				egressStateRepository: degradedRepository(),
+				networkService: { requestStatus: statusFor([]) },
+			});
+
+			await service.checkRecovery(makeJob());
+
+			expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
+			expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
+			expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled();
+		});
+
+		it("releases the job without probing when the persisted state is no longer degraded", async () => {
+			const { service, defaults } = createService();
+
+			await service.checkRecovery(makeJob());
+
+			expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
+			expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
+			expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
+		});
+
+		it("releases the job without probing when the feature has been disabled mid-episode", async () => {
+			const { service, defaults } = createService({
+				settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
+				egressStateRepository: degradedRepository(),
+			});
+
+			await service.checkRecovery(makeJob());
+
+			expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
+			expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
+		});
+
+		it("lets errors propagate so the queue records the failure and retries", async () => {
+			const { service } = createService({
+				egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }),
+			});
+
+			await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down");
+		});
+	});
+});

server/test/unit/services/egressStateService.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, it, jest } from "@jest/globals";
+import { EgressStateService } from "../../../src/domain/egress/egress-state.service.ts";
+import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+const createEgressStateRepo = () => ({
+	findSingleton: jest.fn(),
+	recordProbe: jest.fn(),
+	markDegraded: jest.fn(),
+	markRecovered: jest.fn(),
+	reset: jest.fn(),
+});
+
+const createJobsRepo = () => ({
+	deleteGlobalJob: jest.fn().mockResolvedValue(true),
+});
+
+const createService = () => {
+	const egressStateRepository = createEgressStateRepo();
+	const jobsRepository = createJobsRepo();
+	const service = new EgressStateService(egressStateRepository as any, jobsRepository as any);
+	return { service, egressStateRepository, jobsRepository };
+};
+
+const makeState = (overrides?: Partial<EgressState>): EgressState => ({
+	id: "egress-1",
+	status: "ok",
+	degradedSince: null,
+	lastRecoveredAt: null,
+	lastProbeAt: null,
+	lastProbeResults: [],
+	createdAt: "2026-01-01T00:00:00Z",
+	updatedAt: "2026-01-01T00:00:00Z",
+	...overrides,
+});
+
+// ── Tests ────────────────────────────────────────────────────────────────────
+
+describe("EgressStateService", () => {
+	describe("getState", () => {
+		it("returns the singleton state from the repository", async () => {
+			const { service, egressStateRepository } = createService();
+			const state = makeState({
+				status: "degraded",
+				degradedSince: "2026-01-01T00:05:00Z",
+				lastProbeAt: "2026-01-01T00:05:00Z",
+				lastProbeResults: [{ target: "1.1.1.1", reachable: false, responseTime: 0, message: "timeout" }],
+			});
+			(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(state);
+
+			const result = await service.getState();
+
+			expect(egressStateRepository.findSingleton).toHaveBeenCalledTimes(1);
+			expect(result).toEqual(state);
+		});
+
+		it("does not call any transition method", async () => {
+			const { service, egressStateRepository } = createService();
+			(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(makeState());
+
+			await service.getState();
+
+			expect(egressStateRepository.recordProbe).not.toHaveBeenCalled();
+			expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
+			expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
+		});
+
+		it("propagates repository errors", async () => {
+			const { service, egressStateRepository } = createService();
+			(egressStateRepository.findSingleton as jest.Mock).mockRejectedValue(new Error("db down"));
+
+			await expect(service.getState()).rejects.toThrow("db down");
+		});
+	});
+
+	describe("reset", () => {
+		it("removes any pending recovery job and returns the state to ok", async () => {
+			const { service, egressStateRepository, jobsRepository } = createService();
+			const reset = makeState();
+			(egressStateRepository.reset as jest.Mock).mockResolvedValue(reset);
+
+			const result = await service.reset();
+
+			expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress");
+			expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
+			expect(result).toEqual(reset);
+		});
+
+		it("does not touch the probe-driven transitions", async () => {
+			const { service, egressStateRepository } = createService();
+			(egressStateRepository.reset as jest.Mock).mockResolvedValue(makeState());
+
+			await service.reset();
+
+			expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
+			expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
+		});
+	});
+});

server/test/unit/services/notificationsService.test.ts

@@ -28,6 +28,7 @@ const createMonitorsRepo = () => ({
 
 const createSettingsService = (clientHost = "https://app.example.com") => ({
 	getSettings: jest.fn().mockReturnValue({ clientHost }),
+	removeEgressNotification: jest.fn().mockResolvedValue(undefined),
 });
 
 const createMessageBuilder = () => ({
@@ -371,5 +372,26 @@ describe("NotificationsService", () => {
 			expect(result).toBe(deleted);
 			expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
 		});
+
+		it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => {
+			const { service, notificationsRepository, settingsService } = createService();
+			(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
+
+			await service.deleteById("notif-1", "team-1");
+
+			expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1");
+			const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!;
+			const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!;
+			expect(deleteOrder).toBeLessThan(cascadeOrder);
+		});
+
+		it("does not touch app settings when the delete is refused", async () => {
+			const { service, notificationsRepository, settingsService } = createService();
+			(notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found"));
+
+			await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found");
+
+			expect(settingsService.removeEgressNotification).not.toHaveBeenCalled();
+		});
 	});
 });

server/test/unit/services/settingsService.test.ts

@@ -43,6 +43,7 @@ const createSettingsRepo = () =>
 		create: jest.fn().mockResolvedValue(makeSettings()),
 		findSingleton: jest.fn().mockResolvedValue(makeSettings()),
 		update: jest.fn().mockResolvedValue(makeSettings()),
+		removeEgressNotification: jest.fn().mockResolvedValue(undefined),
 		deleteLegacy: jest.fn().mockResolvedValue(true),
 	}) as unknown as jest.Mocked<ISettingsRepository>;
 
@@ -192,4 +193,20 @@ describe("SettingsService", () => {
 			await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized");
 		});
 	});
+
+	describe("removeEgressNotification", () => {
+		it("delegates the pull to the repository", async () => {
+			const { service, settingsRepository } = createService();
+
+			await service.removeEgressNotification("notif-1");
+
+			expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1");
+		});
+
+		it("throws when repository is not set", async () => {
+			const service = new SettingsService(makeEnv());
+
+			await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized");
+		});
+	});
 });

server/test/unit/services/workerHelper.test.ts

@@ -41,6 +41,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
 	const checkService = {
 		deleteOlderThan: jest.fn().mockResolvedValue(0),
 	};
+	const egressService = {
+		checkRecovery: jest.fn().mockResolvedValue(undefined),
+	};
 
 	const defaults = {
 		logger: createMockLogger(),
@@ -54,6 +57,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
 		incidentsRepository,
 		geoChecksRepository,
 		dockerLogsRepository,
+		egressService,
 		...overrides,
 	};
 
@@ -68,7 +72,8 @@ const createHelper = (overrides?: Record<string, unknown>) => {
 		defaults.checksRepository as any,
 		defaults.incidentsRepository as any,
 		defaults.geoChecksRepository as any,
-		defaults.dockerLogsRepository as any
+		defaults.dockerLogsRepository as any,
+		defaults.egressService as any
 	);
 	return { helper, defaults };
 };
@@ -184,4 +189,22 @@ describe("WorkerHelper", () => {
 			expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
 		});
 	});
+
+	describe("getEgressRecoveryJob", () => {
+		it("returns a function that hands the claimed job to the egress service", async () => {
+			const { helper, defaults } = createHelper();
+			const job = { id: "egress", type: "egress", refId: null, nextScheduledAt: 1000 } as any;
+
+			await helper.getEgressRecoveryJob()(job);
+
+			expect(defaults.egressService.checkRecovery).toHaveBeenCalledWith(job);
+		});
+
+		it("lets the egress service's error propagate so the queue records the failure", async () => {
+			const { helper, defaults } = createHelper();
+			(defaults.egressService.checkRecovery as jest.Mock).mockRejectedValue(new Error("db down"));
+
+			await expect(helper.getEgressRecoveryJob()({ id: "egress" } as any)).rejects.toThrow("db down");
+		});
+	});
 });

server/test/unit/utils/egressTarget.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "@jest/globals";
+import { parseEgressTarget } from "../../../src/utils/egressTarget.ts";
+
+describe("parseEgressTarget", () => {
+	it("parses http(s) URLs", () => {
+		expect(parseEgressTarget("https://example.com/health")).toEqual({ kind: "http", url: "https://example.com/health" });
+		expect(parseEgressTarget("HTTP://example.com")).toEqual({ kind: "http", url: "HTTP://example.com" });
+		expect(parseEgressTarget("http://[2606:4700::1111]:8080/")).toEqual({ kind: "http", url: "http://[2606:4700::1111]:8080/" });
+	});
+
+	it("parses host:port with a hostname, IPv4 address, localhost or bracketed IPv6 address", () => {
+		expect(parseEgressTarget("dns.google:53")).toEqual({ kind: "port", host: "dns.google", port: 53 });
+		expect(parseEgressTarget("8.8.8.8:53")).toEqual({ kind: "port", host: "8.8.8.8", port: 53 });
+		expect(parseEgressTarget("localhost:8080")).toEqual({ kind: "port", host: "localhost", port: 8080 });
+		expect(parseEgressTarget("[2606:4700::1111]:443")).toEqual({ kind: "port", host: "2606:4700::1111", port: 443 });
+	});
+
+	it("parses bare hosts and addresses as ping targets", () => {
+		expect(parseEgressTarget("1.1.1.1")).toEqual({ kind: "ping", host: "1.1.1.1" });
+		expect(parseEgressTarget("example.com")).toEqual({ kind: "ping", host: "example.com" });
+		expect(parseEgressTarget("2606:4700:4700::1111")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
+		expect(parseEgressTarget("[2606:4700:4700::1111]")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
+		expect(parseEgressTarget("  1.1.1.1  ")).toEqual({ kind: "ping", host: "1.1.1.1" });
+	});
+
+	it("rejects anything the probe could not reach", () => {
+		for (const target of [
+			"",
+			"ftp://example.com",
+			"http://",
+			"bad host",
+			"1.1.1.1;rm",
+			"[1.1.1.1",
+			"1.1.1.1]",
+			"[::1",
+			"::1]",
+			"[[::1]]",
+			"[not-an-address]:53",
+			"[1.1.1.1]:53",
+			"999.1.1.1",
+			"1.1.1.1:0",
+			"1.1.1.1:70000",
+			"1.1.1.1:",
+			"host-without-tld",
+		]) {
+			expect(parseEgressTarget(target)).toBeNull();
+		}
+	});
+});

server/test/unit/validation/settingsValidation.test.ts

@@ -0,0 +1,99 @@
+import { describe, expect, it } from "@jest/globals";
+import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts";
+import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts";
+
+const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8";
+
+describe("settingsValidation", () => {
+	describe("updateAppSettingsBodyValidation egress fields", () => {
+		it("accepts a full set of valid egress settings", () => {
+			const parsed = updateAppSettingsBodyValidation.parse({
+				egressCheckEnabled: true,
+				egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"],
+				egressPollIntervalSeconds: 30,
+				egressNotifications: [validNotificationId],
+			});
+
+			expect(parsed.egressCheckEnabled).toBe(true);
+			expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]);
+			expect(parsed.egressPollIntervalSeconds).toBe(30);
+			expect(parsed.egressNotifications).toEqual([validNotificationId]);
+		});
+
+		it("accepts a body with no egress fields", () => {
+			expect(() => updateAppSettingsBodyValidation.parse({ checkTTL: 30 })).not.toThrow();
+		});
+
+		it("accepts an empty target list (the worker falls back to the default targets)", () => {
+			expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: [] }).egressCheckTargets).toEqual([]);
+		});
+
+		it("rejects more targets than the maximum", () => {
+			const targets = Array.from({ length: MAX_EGRESS_TARGETS + 1 }, (_, i) => `10.0.0.${i}`);
+
+			expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow();
+		});
+
+		it("accepts IPv6 targets, bare or bracketed with a port", () => {
+			const targets = ["2606:4700:4700::1111", "[2606:4700:4700::1111]", "[2606:4700:4700::1111]:53", "localhost", "localhost:8080"];
+			expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets }).egressCheckTargets).toEqual(targets);
+		});
+
+		it("rejects targets that are not a host, host:port or http(s) URL", () => {
+			for (const target of ["", "   ", "ftp://example.com", "bad host", "1.1.1.1;rm", "http://"]) {
+				expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
+			}
+		});
+
+		it("rejects malformed addresses that the probe could never reach", () => {
+			for (const target of [
+				"[1.1.1.1",
+				"1.1.1.1]",
+				"[::1",
+				"::1]",
+				"[[::1]]",
+				"[not-an-address]:53",
+				"999.1.1.1",
+				"1.1.1.1:0",
+				"1.1.1.1:70000",
+				"[::1]:",
+			]) {
+				expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
+			}
+		});
+
+		it("accepts the poll interval boundaries", () => {
+			expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
+				MIN_EGRESS_POLL_INTERVAL_SECONDS
+			);
+			expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
+				MAX_EGRESS_POLL_INTERVAL_SECONDS
+			);
+		});
+
+		it("rejects out-of-range and non-integer poll intervals", () => {
+			for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) {
+				expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow();
+			}
+		});
+
+		it("accepts an empty notification list", () => {
+			expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
+		});
+
+		it("accepts upper-case hex notification ids", () => {
+			const id = "64B7F0C2A1D2E3F4A5B6C7D8";
+			expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [id] }).egressNotifications).toEqual([id]);
+		});
+
+		it("rejects notification ids that are not 24-character hex", () => {
+			for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) {
+				expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow();
+			}
+		});
+
+		it("rejects a non-boolean enabled flag", () => {
+			expect(() => updateAppSettingsBodyValidation.parse({ egressCheckEnabled: "yes" })).toThrow();
+		});
+	});
+});