Harden the egress recovery job after a second review pass (#3937)

- Schedule the recovery job before recording the degraded transition,
  so a crash between the two leaves a stray row (removed on the job's
  first run) rather than a degraded state nothing is polling
- Count any HTTP response, including 4xx/5xx, as reachable for http
  targets; only a transport failure counts against egress
- Cascade a deleted notification out of egressNotifications with an
  atomic $pull in the settings repository, after the team-scoped delete
- Handle two processes racing the first egress state insert
- Address global job rows by one key (deleteGlobalJob on _id) and
  restore the deleteByIdAndType signature
- Inject net into PingProvider like PortProvider
- Integration tests for MongoEgressStateRepository and the global-row
  delete methods on the jobs repository

🔧
This commit is contained in:
Craig Lawson committed 2026-09-15 08:59:13 +01:00
1 parent 08f104e283
commit b68367b04f
18 files changed
+319 -62

No files matched your search

+1 -1
View File
@@ -78,7 +78,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
// ***********************
// Network providers
// ***********************
const pingProvider = new PingProvider(ping);
const pingProvider = new PingProvider(ping, net);
const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
const hardwareProvider = new HardwareProvider(httpProvider);
@@ -6,6 +6,7 @@ export interface ISettingsRepository {
findSingleton(): Promise<Settings | null>;
// update
update(settings: SettingsUpdate): Promise<Settings>;
removeEgressNotification(notificationId: string): Promise<void>; // $pull, so it cannot race a concurrent settings update
// delete
deleteLegacy: () => Promise<boolean>;
}
@@ -84,6 +84,10 @@ class MongoSettingsRepository implements ISettingsRepository {
return this.toEntity(updatedSettings);
};
removeEgressNotification = async (notificationId: string) => {
await AppSettingsModel.updateMany({ egressNotifications: notificationId }, { $pull: { egressNotifications: notificationId } });
};
deleteLegacy = async () => {
const res = await AppSettingsModel.deleteMany({ version: { $exists: false } });
return res.deletedCount > 0;
@@ -24,6 +24,7 @@ export interface ISettingsService {
areStatusPageThemesEnabled(): boolean;
getDBSettings(): Promise<Settings>;
updateDbSettings(newSettings: SettingsUpdate): Promise<Settings>;
removeEgressNotification(notificationId: string): Promise<void>;
}
export class SettingsService implements ISettingsService {
@@ -78,6 +79,10 @@ export class SettingsService implements ISettingsService {
return await this.getRepository().update(newSettings);
};
removeEgressNotification = async (notificationId: string) => {
await this.getRepository().removeEgressNotification(notificationId);
};
getDBSettings = async () => {
const repo = this.getRepository();
// Remove any old settings
@@ -30,12 +30,21 @@ class MongoEgressStateRepository implements IEgressStateRepository {
// Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write.
const existing = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
if (existing) return this.toEntity(existing);
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
{ $setOnInsert: { singleton: true, status: "ok" } },
{ upsert: true, new: true, setDefaultsOnInsert: true }
).lean<EgressStateDocument>();
return this.toEntity(doc);
try {
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
{ $setOnInsert: { singleton: true, status: "ok" } },
{ upsert: true, new: true, setDefaultsOnInsert: true }
).lean<EgressStateDocument>();
return this.toEntity(doc);
} catch (error: unknown) {
// Two processes can race the very first insert; the unique index rejects the loser, who reads the winner's row.
if ((error as { code?: number }).code === 11000) {
const doc = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
if (doc) return this.toEntity(doc);
}
throw error;
}
};
reset = async () => {
@@ -25,7 +25,7 @@ export class EgressStateService implements IEgressStateService {
// Called when the feature is switched on or off: drop any pending recovery job and start from "ok".
reset = async (): Promise<EgressState> => {
await this.jobsRepository.deleteByIdAndType(null, "egress");
await this.jobsRepository.deleteGlobalJob("egress");
return this.egressStateRepository.reset();
};
}
+9 -3
View File
@@ -25,6 +25,9 @@ const ASSESSMENT_REUSE_MS = 5000;
const RECOVERY_JOB_TYPE = "egress" as const;
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
const PROBE_MONITOR_ID = "egress-probe";
const PROBE_TEAM_ID = "system";
@@ -115,7 +118,9 @@ export class EgressService implements IEgressService {
return {
target: raw,
reachable: response.status === true,
// Any HTTP response, including 4xx/5xx, proves the instance can reach the target; only a transport
// failure counts against egress. Ping and port providers have no such distinction.
reachable: response.status === true || (target.kind === "http" && isHttpStatusCode(response.code)),
responseTime: response.responseTime ?? responseTime,
message: response.message,
};
@@ -188,6 +193,9 @@ export class EgressService implements IEgressService {
return "ok";
}
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
const degraded = await this.egressStateRepository.markDegraded(results, now);
this.logger.warn({
message: degraded
@@ -197,8 +205,6 @@ export class EgressService implements IEgressService {
method: "assessAfterFailure",
details: { results },
});
// The worker that performed the transition owns the job's schedule; anyone else only makes sure the row exists.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, degraded !== null);
return "degraded";
} catch (error: unknown) {
// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
@@ -58,9 +58,11 @@ export interface IJobsRepository {
// Delete, drop all rows
deleteById(refId: string): Promise<boolean>;
// Delete a single job of type. refId null addresses a global row
deleteByIdAndType(refId: string | null, type: JobType): Promise<boolean>;
// Delete a single job of type
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
// Global rows (refId null) are addressed by their canonical _id, see jobId
deleteGlobalJob(type: JobType): Promise<boolean>;
// Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged)
deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise<boolean>;
@@ -273,8 +273,13 @@ class MongoJobsRepository implements IJobsRepository {
return res.deletedCount > 0;
};
deleteByIdAndType = async (refId: string | null, type: JobType) => {
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row or the global egress row
deleteByIdAndType = async (refId: string, type: JobType) => {
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
return res.deletedCount > 0;
};
deleteGlobalJob = async (type: JobType) => {
const res = await JobModel.deleteOne({ _id: jobId(type, null) });
return res.deletedCount > 0;
};
@@ -178,18 +178,10 @@ export class NotificationsService implements INotificationsService {
deleteById = async (id: string, teamId: string): Promise<Notification> => {
await this.monitorsRepository.removeNotificationFromMonitors(id);
await this.removeFromEgressNotifications(id);
const deleted = await this.notificationsRepository.deleteById(id, teamId);
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles.
// After the team-scoped delete, so a request for another team's id (404 above) cannot touch instance settings.
await this.settingsService.removeEgressNotification(id);
return deleted;
};
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles
private removeFromEgressNotifications = async (id: string) => {
const settings = await this.settingsService.getDBSettings();
const current = settings.egressNotifications ?? [];
const remaining = current.filter((notificationId) => notificationId !== id);
if (remaining.length !== current.length) {
await this.settingsService.updateDbSettings({ egressNotifications: remaining });
}
};
}
+7 -3
View File
@@ -4,16 +4,20 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js";
import { MonitorStatusResponse } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import ping from "ping";
import net from "net";
import * as net from "net";
import { timeRequest } from "@/service/network/utils.js";
const SERVICE_NAME = "PingProvider";
type Ping = typeof ping;
type NetType = typeof net;
export class PingProvider implements IStatusProvider<PingStatusPayload> {
readonly type = "ping";
constructor(private ping: Ping) {}
constructor(
private ping: Ping,
private net: NetType
) {}
supports(type: MonitorType): boolean {
return type === "ping";
@@ -25,7 +29,7 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host);
if (bracketed) return bracketed[1]!;
// A bare IPv6 address contains colons that are not a port separator
if (net.isIPv6(host)) return host;
if (this.net.isIPv6(host)) return host;
return host.replace(/:.*/, "");
}