Address review on the egress recovery job (#3937)

- The egress job runs through WorkerHelper.getEgressRecoveryJob(), like
  the cleanup jobs, instead of a direct service call from the queue
- Toggling egressCheckEnabled resets the egress state and removes any
  pending recovery job
- Probes resolve and pass the proxy, so http targets honour the global
  proxy as monitors do
- PingProvider keeps IPv6 addresses intact instead of truncating at the
  first colon
- Egress targets are parsed by one function shared by the validator and
  the probe; malformed addresses such as unbalanced brackets are rejected
- Notification id pattern drops the /i flag, which produced an invalid
  OpenAPI pattern
- EgressState findSingleton reads before it writes, so a read no longer
  bumps updatedAt
- "disabled" is a distinct assessment result and is cached like any other
- Concurrent failures share an in-flight assessment for as long as it
  runs, then reuse the settled result for 5 s
- The recovery job row is only removed if nobody rescheduled it since
  this run claimed it, so a concurrent new episode keeps its job
- Deleting a notification removes it from egressNotifications

🔧
This commit is contained in:
Craig Lawson committed 2026-09-15 08:42:09 +01:00
1 parent 46eb0b4063
commit 08f104e283
26 files changed
+600 -157

No files matched your search

@@ -2,9 +2,10 @@ import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type
export interface IEgressStateRepository {
// read
findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call
findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call, a plain read after that
// update
recordProbe(results: EgressProbeResult[], now: Date): Promise<EgressState>;
reset(): Promise<EgressState>; // back to "ok" with no episode; used when the feature is toggled
// Atomic transitions. Each returns the updated state when this caller performed the transition,
// or null when the state was already on the target side (another worker got there first).
markDegraded(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
@@ -27,6 +27,9 @@ class MongoEgressStateRepository implements IEgressStateRepository {
};
findSingleton = async () => {
// Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write.
const existing = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
if (existing) return this.toEntity(existing);
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
{ $setOnInsert: { singleton: true, status: "ok" } },
@@ -35,6 +38,15 @@ class MongoEgressStateRepository implements IEgressStateRepository {
return this.toEntity(doc);
};
reset = async () => {
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
{ $set: { status: "ok", degradedSince: null, lastProbeAt: null, lastProbeResults: [] }, $setOnInsert: { singleton: true } },
{ upsert: true, new: true, setDefaultsOnInsert: true }
).lean<EgressStateDocument>();
return this.toEntity(doc);
};
recordProbe = async (results: EgressProbeResult[], now: Date) => {
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
@@ -1,20 +1,31 @@
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import type { EgressState } from "@/domain/egress/egress.type.js";
export interface IEgressStateService {
getState(): Promise<EgressState>;
reset(): Promise<EgressState>;
}
const SERVICE_NAME = "EgressStateService";
// Read-only view of the instance-global egress state for the API process. Transitions are
// owned by the worker (egress.service.ts); this exists so the controller never touches the repository.
// API-process view of the instance-global egress state. Probe-driven transitions are owned by the
// worker (egress.service.ts); this exists so the controller never touches the repositories.
export class EgressStateService implements IEgressStateService {
static SERVICE_NAME = SERVICE_NAME;
constructor(private egressStateRepository: IEgressStateRepository) {}
constructor(
private egressStateRepository: IEgressStateRepository,
private jobsRepository: IJobsRepository
) {}
getState = async (): Promise<EgressState> => {
return this.egressStateRepository.findSingleton();
};
// Called when the feature is switched on or off: drop any pending recovery job and start from "ok".
reset = async (): Promise<EgressState> => {
await this.jobsRepository.deleteByIdAndType(null, "egress");
return this.egressStateRepository.reset();
};
}
+93 -61
View File
@@ -2,50 +2,57 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { jobId, type JobSeed } from "@/domain/jobs/job.type.js";
import { jobId, type Job, type JobSeed } from "@/domain/jobs/job.type.js";
import type { INetworkService } from "@/service/networkService.js";
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
import type { ILogger } from "@/utils/logger.js";
import {
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
DEFAULT_EGRESS_TARGETS,
type EgressAssessment,
type EgressProbeResult,
type EgressStatus,
} from "@/domain/egress/egress.type.js";
import { timeRequest } from "@/service/network/utils.js";
import { parseEgressTarget, type EgressTarget } from "@/utils/egressTarget.js";
const SERVICE_NAME = "EgressService";
const PROBE_TIMEOUT_MS = 5000;
// When egress is down every monitor fails at once, so concurrent failures share one assessment
// rather than each probing. Same in-flight cache shape as ProxyResolver.
const ASSESSMENT_TTL_MS = PROBE_TIMEOUT_MS;
// How long a settled assessment answers for later failures. An assessment still in flight is shared
// for as long as it takes, so this only bounds how stale a reused answer can be.
const ASSESSMENT_REUSE_MS = 5000;
// Global job row (refId null) that re-probes while degraded. Inserted when egress degrades, removed on recovery.
const RECOVERY_JOB_ID = jobId("egress", null);
const RECOVERY_JOB_TYPE = "egress" as const;
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
const PROBE_MONITOR_ID = "egress-probe";
const PROBE_TEAM_ID = "system";
// "[host]:port" or "host:port" where host has no colons (so a bare IPv6 address is not mistaken for host:port).
const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/;
export interface IEgressService {
probeTargets(targets: string[]): Promise<EgressProbeResult[]>;
// Called by the check producer when a check fails. Null when the feature is off or the assessment itself failed.
assessAfterFailure(): Promise<EgressStatus | null>;
// Handler for the "egress" job: re-probes while degraded and removes the job once egress is back.
checkRecovery(): Promise<void>;
checkRecovery(job: Job): Promise<void>;
}
type CachedAssessment = {
value: Promise<EgressAssessment | null>;
expiresAt: number | null; // null while the assessment is still in flight
};
export class EgressService implements IEgressService {
static SERVICE_NAME = SERVICE_NAME;
private assessment: { value: Promise<EgressStatus | null>; expiresAt: number } | null = null;
private assessment: CachedAssessment | null = null;
constructor(
private settingsService: ISettingsService,
private egressStateRepository: IEgressStateRepository,
private jobsRepository: IJobsRepository,
private networkService: INetworkService,
private proxyResolver: IProxyResolver,
private logger: ILogger
) {}
@@ -54,46 +61,52 @@ export class EgressService implements IEgressService {
// ****************************
// Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor.
private toProbeMonitor = (target: string): Monitor => {
// proxyMode "inherit" lets the resolver apply the global proxy to http probes, as it does for monitors.
private toProbeMonitor = (target: EgressTarget, raw: string): Monitor => {
const base = {
id: PROBE_MONITOR_ID,
teamId: PROBE_TEAM_ID,
name: `Egress probe: ${target}`,
name: `Egress probe: ${raw}`,
method: "GET",
proxyMode: "inherit",
useAdvancedMatching: false,
ignoreTlsErrors: false,
customUpCodes: [],
};
if (/^https?:\/\//i.test(target)) {
return { ...base, type: "http", url: target } as unknown as Monitor;
switch (target.kind) {
case "http":
return { ...base, type: "http", url: target.url } as unknown as Monitor;
case "port":
return { ...base, type: "port", url: target.host, port: target.port } as unknown as Monitor;
case "ping":
return { ...base, type: "ping", url: target.host } as unknown as Monitor;
}
const hostPort = HOST_PORT_PATTERN.exec(target);
if (hostPort) {
const host = hostPort[1] ?? hostPort[2];
return { ...base, type: "port", url: host, port: Number(hostPort[3]) } as unknown as Monitor;
}
return { ...base, type: "ping", url: target } as unknown as Monitor;
};
private probeTarget = async (target: string): Promise<EgressProbeResult> => {
const monitor = this.toProbeMonitor(target);
private probeTarget = async (raw: string): Promise<EgressProbeResult> => {
const target = parseEgressTarget(raw);
if (!target) {
// Validation rejects these; a stored value can still be stale, and it must not count as reachable.
return { target: raw, reachable: false, responseTime: 0, message: "Invalid egress target" };
}
const monitor = this.toProbeMonitor(target, raw);
let timer: NodeJS.Timeout | undefined;
const { response, responseTime, error } = await timeRequest(() =>
Promise.race([
this.networkService.requestStatus(monitor),
const { response, responseTime, error } = await timeRequest(async () => {
const proxyUrl = await this.proxyResolver.resolve(monitor);
return Promise.race([
this.networkService.requestStatus(monitor, { proxyUrl }),
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS);
}),
]).finally(() => clearTimeout(timer))
);
]).finally(() => clearTimeout(timer));
});
if (error || !response) {
return {
target,
target: raw,
reachable: false,
responseTime,
message: error instanceof Error ? error.message : error ? String(error) : "No response",
@@ -101,7 +114,7 @@ export class EgressService implements IEgressService {
}
return {
target,
target: raw,
reachable: response.status === true,
responseTime: response.responseTime ?? responseTime,
message: response.message,
@@ -127,32 +140,43 @@ export class EgressService implements IEgressService {
// Event-triggered assessment
// ****************************
assessAfterFailure = (): Promise<EgressStatus | null> => {
const now = Date.now();
if (this.assessment && this.assessment.expiresAt > now) {
return this.assessment.value;
}
// Cache the in-flight promise so concurrent failures share one probe
const value = this.assess();
this.assessment = { value, expiresAt: now + ASSESSMENT_TTL_MS };
// Don't cache an internal failure, so the next failing check retries
void value.then((status) => {
if (status === null && this.assessment?.value === value) this.assessment = null;
});
return value;
assessAfterFailure = async (): Promise<EgressStatus | null> => {
const assessment = await this.sharedAssessment();
return assessment === "disabled" ? null : assessment;
};
private assess = async (): Promise<EgressStatus | null> => {
// When egress is down every monitor fails at once. Concurrent failures share the in-flight assessment
// however long it takes, and a settled one answers for ASSESSMENT_REUSE_MS. Same shape as ProxyResolver's cache.
private sharedAssessment = (): Promise<EgressAssessment | null> => {
const cached = this.assessment;
if (cached && (cached.expiresAt === null || cached.expiresAt > Date.now())) {
return cached.value;
}
const entry: CachedAssessment = { value: this.assess(), expiresAt: null };
this.assessment = entry;
void entry.value.then((result) => {
if (this.assessment !== entry) return;
if (result === null) {
this.assessment = null; // an internal failure is not cached, so the next failing check retries
} else {
entry.expiresAt = Date.now() + ASSESSMENT_REUSE_MS;
}
});
return entry.value;
};
private assess = async (): Promise<EgressAssessment | null> => {
try {
const settings = await this.settingsService.getDBSettings();
if (!settings.egressCheckEnabled) {
return null;
return "disabled";
}
const state = await this.egressStateRepository.findSingleton();
if (state.status === "degraded") {
// Recovery is detected by the scheduled job. Make sure it exists in case its row was lost.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds);
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
return "degraded";
}
@@ -173,7 +197,8 @@ export class EgressService implements IEgressService {
method: "assessAfterFailure",
details: { results },
});
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds);
// The worker that performed the transition owns the job's schedule; anyone else only makes sure the row exists.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, degraded !== null);
return "degraded";
} catch (error: unknown) {
// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
@@ -191,35 +216,42 @@ export class EgressService implements IEgressService {
// Recovery job
// ****************************
// upsertJob only sets nextScheduledAt on insert, so re-arming while degraded never pushes a pending run back.
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined) => {
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
// so re-arming while degraded never pushes a pending run back.
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
const seed: JobSeed = {
id: RECOVERY_JOB_ID,
type: "egress",
type: RECOVERY_JOB_TYPE,
refId: null,
isActive: true,
nextScheduledAt: Date.now() + intervalMs,
intervalMs,
};
await this.jobsRepository.upsertJob(seed);
if (reschedule) {
await this.jobsRepository.upsertCleanupJob(seed); // global-row upsert that also sets the schedule
} else {
await this.jobsRepository.upsertJob(seed);
}
};
private removeRecoveryJob = async () => {
await this.jobsRepository.deleteByIdAndType(null, "egress");
// Removes the row only if nobody has rescheduled it since this run claimed it. Another worker may have seen
// egress fail again in the meantime and started a new episode; its schedule must survive.
private releaseRecoveryJob = async (job: Job) => {
await this.jobsRepository.deleteGlobalJobIfUnchanged(RECOVERY_JOB_TYPE, job.nextScheduledAt);
};
// Runs on the queue at the configured interval while degraded. Errors propagate so the queue records
// the failure and retries with its usual backoff.
checkRecovery = async (): Promise<void> => {
checkRecovery = async (job: Job): Promise<void> => {
const settings = await this.settingsService.getDBSettings();
const state = await this.egressStateRepository.findSingleton();
if (state.status !== "degraded") {
// Another worker recorded the recovery, or the state was reset; nothing left to poll for.
await this.removeRecoveryJob();
if (!settings.egressCheckEnabled || state.status !== "degraded") {
// Feature switched off, recovery recorded by another worker, or state reset: nothing left to poll for.
await this.releaseRecoveryJob(job);
return;
}
const settings = await this.settingsService.getDBSettings();
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
const now = new Date();
@@ -229,7 +261,7 @@ export class EgressService implements IEgressService {
}
const recovered = await this.egressStateRepository.markRecovered(results, now);
await this.removeRecoveryJob();
await this.releaseRecoveryJob(job);
if (!recovered) {
// Another process performed the transition.
return;
+2
View File
@@ -3,6 +3,8 @@
export const EgressStatuses = ["ok", "degraded"] as const;
export type EgressStatus = (typeof EgressStatuses)[number];
// Outcome of one assessment: a status, or "disabled" when the feature is switched off (distinct from an internal failure).
export type EgressAssessment = EgressStatus | "disabled";
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
@@ -61,6 +61,9 @@ export interface IJobsRepository {
// Delete a single job of type. refId null addresses a global row
deleteByIdAndType(refId: string | null, type: JobType): Promise<boolean>;
// Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged)
deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise<boolean>;
// Delete jobs that reference monitors that no longer exist
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
@@ -278,6 +278,11 @@ class MongoJobsRepository implements IJobsRepository {
return res.deletedCount > 0;
};
deleteGlobalJobIfUnchanged = async (type: JobType, nextScheduledAt: number) => {
const res = await JobModel.deleteOne({ _id: jobId(type, null), nextScheduledAt });
return res.deletedCount > 0;
};
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
@@ -178,7 +178,18 @@ export class NotificationsService implements INotificationsService {
deleteById = async (id: string, teamId: string): Promise<Notification> => {
await this.monitorsRepository.removeNotificationFromMonitors(id);
await this.removeFromEgressNotifications(id);
const deleted = await this.notificationsRepository.deleteById(id, teamId);
return deleted;
};
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles
private removeFromEgressNotifications = async (id: string) => {
const settings = await this.settingsService.getDBSettings();
const current = settings.egressNotifications ?? [];
const remaining = current.filter((notificationId) => notificationId !== id);
if (remaining.length !== current.length) {
await this.settingsService.updateDbSettings({ egressNotifications: remaining });
}
};
}