Harden the egress recovery job after a second review pass (#3937)
- Schedule the recovery job before recording the degraded transition,
so a crash between the two leaves a stray row (removed on the job's
first run) rather than a degraded state nothing is polling
- Count any HTTP response, including 4xx/5xx, as reachable for http
targets; only a transport failure counts against egress
- Cascade a deleted notification out of egressNotifications with an
atomic $pull in the settings repository, after the team-scoped delete
- Handle two processes racing the first egress state insert
- Address global job rows by one key (deleteGlobalJob on _id) and
restore the deleteByIdAndType signature
- Inject net into PingProvider like PortProvider
- Integration tests for MongoEgressStateRepository and the global-row
delete methods on the jobs repository
🔧
This commit is contained in:
1 parent
08f104e283
commit
b68367b04f
18 files changed
+319
-62
No files matched your search
@@ -78,7 +78,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
// ***********************
|
||||
// Network providers
|
||||
// ***********************
|
||||
const pingProvider = new PingProvider(ping);
|
||||
const pingProvider = new PingProvider(ping, net);
|
||||
const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
|
||||
const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
|
||||
const hardwareProvider = new HardwareProvider(httpProvider);
|
||||
|
||||
@@ -6,6 +6,7 @@ export interface ISettingsRepository {
|
||||
findSingleton(): Promise<Settings | null>;
|
||||
// update
|
||||
update(settings: SettingsUpdate): Promise<Settings>;
|
||||
removeEgressNotification(notificationId: string): Promise<void>; // $pull, so it cannot race a concurrent settings update
|
||||
// delete
|
||||
deleteLegacy: () => Promise<boolean>;
|
||||
}
|
||||
@@ -84,6 +84,10 @@ class MongoSettingsRepository implements ISettingsRepository {
|
||||
return this.toEntity(updatedSettings);
|
||||
};
|
||||
|
||||
removeEgressNotification = async (notificationId: string) => {
|
||||
await AppSettingsModel.updateMany({ egressNotifications: notificationId }, { $pull: { egressNotifications: notificationId } });
|
||||
};
|
||||
|
||||
deleteLegacy = async () => {
|
||||
const res = await AppSettingsModel.deleteMany({ version: { $exists: false } });
|
||||
return res.deletedCount > 0;
|
||||
|
||||
@@ -24,6 +24,7 @@ export interface ISettingsService {
|
||||
areStatusPageThemesEnabled(): boolean;
|
||||
getDBSettings(): Promise<Settings>;
|
||||
updateDbSettings(newSettings: SettingsUpdate): Promise<Settings>;
|
||||
removeEgressNotification(notificationId: string): Promise<void>;
|
||||
}
|
||||
|
||||
export class SettingsService implements ISettingsService {
|
||||
@@ -78,6 +79,10 @@ export class SettingsService implements ISettingsService {
|
||||
return await this.getRepository().update(newSettings);
|
||||
};
|
||||
|
||||
removeEgressNotification = async (notificationId: string) => {
|
||||
await this.getRepository().removeEgressNotification(notificationId);
|
||||
};
|
||||
|
||||
getDBSettings = async () => {
|
||||
const repo = this.getRepository();
|
||||
// Remove any old settings
|
||||
|
||||
@@ -30,12 +30,21 @@ class MongoEgressStateRepository implements IEgressStateRepository {
|
||||
// Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write.
|
||||
const existing = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
|
||||
if (existing) return this.toEntity(existing);
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $setOnInsert: { singleton: true, status: "ok" } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
try {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $setOnInsert: { singleton: true, status: "ok" } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
} catch (error: unknown) {
|
||||
// Two processes can race the very first insert; the unique index rejects the loser, who reads the winner's row.
|
||||
if ((error as { code?: number }).code === 11000) {
|
||||
const doc = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
|
||||
if (doc) return this.toEntity(doc);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
reset = async () => {
|
||||
|
||||
@@ -25,7 +25,7 @@ export class EgressStateService implements IEgressStateService {
|
||||
|
||||
// Called when the feature is switched on or off: drop any pending recovery job and start from "ok".
|
||||
reset = async (): Promise<EgressState> => {
|
||||
await this.jobsRepository.deleteByIdAndType(null, "egress");
|
||||
await this.jobsRepository.deleteGlobalJob("egress");
|
||||
return this.egressStateRepository.reset();
|
||||
};
|
||||
}
|
||||
@@ -25,6 +25,9 @@ const ASSESSMENT_REUSE_MS = 5000;
|
||||
const RECOVERY_JOB_TYPE = "egress" as const;
|
||||
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
|
||||
|
||||
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
|
||||
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
|
||||
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
|
||||
const PROBE_MONITOR_ID = "egress-probe";
|
||||
const PROBE_TEAM_ID = "system";
|
||||
@@ -115,7 +118,9 @@ export class EgressService implements IEgressService {
|
||||
|
||||
return {
|
||||
target: raw,
|
||||
reachable: response.status === true,
|
||||
// Any HTTP response, including 4xx/5xx, proves the instance can reach the target; only a transport
|
||||
// failure counts against egress. Ping and port providers have no such distinction.
|
||||
reachable: response.status === true || (target.kind === "http" && isHttpStatusCode(response.code)),
|
||||
responseTime: response.responseTime ?? responseTime,
|
||||
message: response.message,
|
||||
};
|
||||
@@ -188,6 +193,9 @@ export class EgressService implements IEgressService {
|
||||
return "ok";
|
||||
}
|
||||
|
||||
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
|
||||
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
|
||||
const degraded = await this.egressStateRepository.markDegraded(results, now);
|
||||
this.logger.warn({
|
||||
message: degraded
|
||||
@@ -197,8 +205,6 @@ export class EgressService implements IEgressService {
|
||||
method: "assessAfterFailure",
|
||||
details: { results },
|
||||
});
|
||||
// The worker that performed the transition owns the job's schedule; anyone else only makes sure the row exists.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, degraded !== null);
|
||||
return "degraded";
|
||||
} catch (error: unknown) {
|
||||
// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
|
||||
|
||||
@@ -58,9 +58,11 @@ export interface IJobsRepository {
|
||||
// Delete, drop all rows
|
||||
deleteById(refId: string): Promise<boolean>;
|
||||
|
||||
// Delete a single job of type. refId null addresses a global row
|
||||
deleteByIdAndType(refId: string | null, type: JobType): Promise<boolean>;
|
||||
// Delete a single job of type
|
||||
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
|
||||
|
||||
// Global rows (refId null) are addressed by their canonical _id, see jobId
|
||||
deleteGlobalJob(type: JobType): Promise<boolean>;
|
||||
// Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged)
|
||||
deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise<boolean>;
|
||||
|
||||
|
||||
@@ -273,8 +273,13 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByIdAndType = async (refId: string | null, type: JobType) => {
|
||||
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row or the global egress row
|
||||
deleteByIdAndType = async (refId: string, type: JobType) => {
|
||||
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteGlobalJob = async (type: JobType) => {
|
||||
const res = await JobModel.deleteOne({ _id: jobId(type, null) });
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
|
||||
@@ -178,18 +178,10 @@ export class NotificationsService implements INotificationsService {
|
||||
|
||||
deleteById = async (id: string, teamId: string): Promise<Notification> => {
|
||||
await this.monitorsRepository.removeNotificationFromMonitors(id);
|
||||
await this.removeFromEgressNotifications(id);
|
||||
const deleted = await this.notificationsRepository.deleteById(id, teamId);
|
||||
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles.
|
||||
// After the team-scoped delete, so a request for another team's id (404 above) cannot touch instance settings.
|
||||
await this.settingsService.removeEgressNotification(id);
|
||||
return deleted;
|
||||
};
|
||||
|
||||
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles
|
||||
private removeFromEgressNotifications = async (id: string) => {
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
const current = settings.egressNotifications ?? [];
|
||||
const remaining = current.filter((notificationId) => notificationId !== id);
|
||||
if (remaining.length !== current.length) {
|
||||
await this.settingsService.updateDbSettings({ egressNotifications: remaining });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -4,16 +4,20 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import ping from "ping";
|
||||
import net from "net";
|
||||
import * as net from "net";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
const SERVICE_NAME = "PingProvider";
|
||||
|
||||
type Ping = typeof ping;
|
||||
type NetType = typeof net;
|
||||
|
||||
export class PingProvider implements IStatusProvider<PingStatusPayload> {
|
||||
readonly type = "ping";
|
||||
|
||||
constructor(private ping: Ping) {}
|
||||
constructor(
|
||||
private ping: Ping,
|
||||
private net: NetType
|
||||
) {}
|
||||
|
||||
supports(type: MonitorType): boolean {
|
||||
return type === "ping";
|
||||
@@ -25,7 +29,7 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
|
||||
const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host);
|
||||
if (bracketed) return bracketed[1]!;
|
||||
// A bare IPv6 address contains colons that are not a port separator
|
||||
if (net.isIPv6(host)) return host;
|
||||
if (this.net.isIPv6(host)) return host;
|
||||
return host.replace(/:.*/, "");
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user