Address review on the egress recovery job (#3937)
- The egress job runs through WorkerHelper.getEgressRecoveryJob(), like
the cleanup jobs, instead of a direct service call from the queue
- Toggling egressCheckEnabled resets the egress state and removes any
pending recovery job
- Probes resolve and pass the proxy, so http targets honour the global
proxy as monitors do
- PingProvider keeps IPv6 addresses intact instead of truncating at the
first colon
- Egress targets are parsed by one function shared by the validator and
the probe; malformed addresses such as unbalanced brackets are rejected
- Notification id pattern drops the /i flag, which produced an invalid
OpenAPI pattern
- EgressState findSingleton reads before it writes, so a read no longer
bumps updatedAt
- "disabled" is a distinct assessment result and is cached like any other
- Concurrent failures share an in-flight assessment for as long as it
runs, then reuse the settled result for 5 s
- The recovery job row is only removed if nobody rescheduled it since
this run claimed it, so a concurrent new episode keeps its job
- Deleting a notification removes it from egressNotifications
🔧
This commit is contained in:
1 parent
46eb0b4063
commit
08f104e283
26 files changed
+600
-157
No files matched your search
@@ -6,6 +6,7 @@ import { AppError } from "@/utils/AppError.js";
|
||||
import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import { IEmailService } from "@/service/emailService.js";
|
||||
import { IProxiesService } from "@/domain/proxies/proxy.service.js";
|
||||
import { IEgressStateService } from "@/domain/egress/egress-state.service.js";
|
||||
import { Settings } from "@/domain/app-settings/app-settings.type.js";
|
||||
|
||||
export interface ISettingsController {
|
||||
@@ -18,10 +19,17 @@ class SettingsController implements ISettingsController {
|
||||
private settingsService: ISettingsService;
|
||||
private emailService: IEmailService;
|
||||
private proxiesService: IProxiesService;
|
||||
constructor(settingsService: ISettingsService, emailService: IEmailService, proxiesService: IProxiesService) {
|
||||
private egressStateService: IEgressStateService;
|
||||
constructor(
|
||||
settingsService: ISettingsService,
|
||||
emailService: IEmailService,
|
||||
proxiesService: IProxiesService,
|
||||
egressStateService: IEgressStateService
|
||||
) {
|
||||
this.settingsService = settingsService;
|
||||
this.emailService = emailService;
|
||||
this.proxiesService = proxiesService;
|
||||
this.egressStateService = egressStateService;
|
||||
}
|
||||
|
||||
buildAppSettings = async (dbSettings: Settings) => {
|
||||
@@ -70,7 +78,14 @@ class SettingsController implements ISettingsController {
|
||||
}
|
||||
}
|
||||
|
||||
const previousSettings = await this.settingsService.getDBSettings();
|
||||
const updatedSettings = await this.settingsService.updateDbSettings(validatedBody);
|
||||
|
||||
// Switching the egress check on or off starts from a clean state: no degraded episode, no pending recovery job
|
||||
if (validatedBody.egressCheckEnabled !== undefined && validatedBody.egressCheckEnabled !== previousSettings.egressCheckEnabled) {
|
||||
await this.egressStateService.reset();
|
||||
}
|
||||
|
||||
const returnSettings = await this.buildAppSettings(updatedSettings);
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
|
||||
import { parseEgressTarget } from "@/utils/egressTarget.js";
|
||||
import { z } from "zod";
|
||||
|
||||
// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL.
|
||||
// Parsed with the same function the probe uses, so nothing the validator accepts can fail to probe.
|
||||
const egressTargetValidation = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(253)
|
||||
.regex(/^(https?:\/\/\S+|[A-Za-z0-9.\-:[\]]+)$/, "Invalid egress target");
|
||||
.refine((target) => parseEgressTarget(target) !== null, "Enter a hostname or IP address, host:port, or http(s) URL");
|
||||
|
||||
//****************************************
|
||||
// Settings Validations
|
||||
@@ -55,7 +57,7 @@ export const updateAppSettingsBodyValidation = z
|
||||
egressCheckEnabled: z.boolean().optional(),
|
||||
egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
|
||||
egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
|
||||
egressNotifications: z.array(z.string().regex(/^[a-f\d]{24}$/i, "Invalid notification id")).optional(),
|
||||
egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(),
|
||||
})
|
||||
.strip()
|
||||
.superRefine((body, ctx) => {
|
||||
|
||||
@@ -36,7 +36,12 @@ export const initializeControllers = (apiServices: ApiServices): InitializedCont
|
||||
return {
|
||||
authController: new AuthController(apiServices.userService),
|
||||
monitorController: new MonitorController(apiServices.monitorService, apiServices.notificationsService),
|
||||
settingsController: new SettingsController(apiServices.settingsService, apiServices.emailService, apiServices.proxiesService),
|
||||
settingsController: new SettingsController(
|
||||
apiServices.settingsService,
|
||||
apiServices.emailService,
|
||||
apiServices.proxiesService,
|
||||
apiServices.egressStateService
|
||||
),
|
||||
checkController: new CheckController(apiServices.checkService),
|
||||
geoCheckController: new GeoCheckController(apiServices.geoChecksService),
|
||||
inviteController: new InviteController(apiServices.inviteService),
|
||||
|
||||
@@ -103,7 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
const tagsService = new TagsService(tagsRepository, monitorsRepository);
|
||||
const diagnosticService = new DiagnosticService(db);
|
||||
const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService);
|
||||
const egressStateService = new EgressStateService(egressStateRepository);
|
||||
const egressStateService = new EgressStateService(egressStateRepository, jobsRepository);
|
||||
return {
|
||||
...shared,
|
||||
worker: jobScheduler,
|
||||
|
||||
@@ -106,7 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
|
||||
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
|
||||
const monitorStatusPolicy = new MonitorStatusPolicy();
|
||||
const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, logger);
|
||||
const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger);
|
||||
|
||||
// ***********************
|
||||
// Reactors and dispatcher
|
||||
@@ -149,7 +149,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
checksRepository,
|
||||
incidentsRepository,
|
||||
geoChecksRepository,
|
||||
dockerLogsRepository
|
||||
dockerLogsRepository,
|
||||
egressService
|
||||
);
|
||||
|
||||
const worker = await DBQueueWorker.create({
|
||||
@@ -165,7 +166,6 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
geoCheckPipeline,
|
||||
dispatcher: reactorDispatcher,
|
||||
helper: workerHelper,
|
||||
egressService,
|
||||
queueWorkersRepository,
|
||||
queueMode: envSettings.queueMode,
|
||||
queuePrimaryProcesses: envSettings.queuePrimaryProcesses,
|
||||
|
||||
@@ -2,9 +2,10 @@ import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type
|
||||
|
||||
export interface IEgressStateRepository {
|
||||
// read
|
||||
findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call
|
||||
findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call, a plain read after that
|
||||
// update
|
||||
recordProbe(results: EgressProbeResult[], now: Date): Promise<EgressState>;
|
||||
reset(): Promise<EgressState>; // back to "ok" with no episode; used when the feature is toggled
|
||||
// Atomic transitions. Each returns the updated state when this caller performed the transition,
|
||||
// or null when the state was already on the target side (another worker got there first).
|
||||
markDegraded(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
|
||||
|
||||
@@ -27,6 +27,9 @@ class MongoEgressStateRepository implements IEgressStateRepository {
|
||||
};
|
||||
|
||||
findSingleton = async () => {
|
||||
// Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write.
|
||||
const existing = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
|
||||
if (existing) return this.toEntity(existing);
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $setOnInsert: { singleton: true, status: "ok" } },
|
||||
@@ -35,6 +38,15 @@ class MongoEgressStateRepository implements IEgressStateRepository {
|
||||
return this.toEntity(doc);
|
||||
};
|
||||
|
||||
reset = async () => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $set: { status: "ok", degradedSince: null, lastProbeAt: null, lastProbeResults: [] }, $setOnInsert: { singleton: true } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
};
|
||||
|
||||
recordProbe = async (results: EgressProbeResult[], now: Date) => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
|
||||
@@ -1,20 +1,31 @@
|
||||
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import type { EgressState } from "@/domain/egress/egress.type.js";
|
||||
|
||||
export interface IEgressStateService {
|
||||
getState(): Promise<EgressState>;
|
||||
reset(): Promise<EgressState>;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "EgressStateService";
|
||||
|
||||
// Read-only view of the instance-global egress state for the API process. Transitions are
|
||||
// owned by the worker (egress.service.ts); this exists so the controller never touches the repository.
|
||||
// API-process view of the instance-global egress state. Probe-driven transitions are owned by the
|
||||
// worker (egress.service.ts); this exists so the controller never touches the repositories.
|
||||
export class EgressStateService implements IEgressStateService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor(private egressStateRepository: IEgressStateRepository) {}
|
||||
constructor(
|
||||
private egressStateRepository: IEgressStateRepository,
|
||||
private jobsRepository: IJobsRepository
|
||||
) {}
|
||||
|
||||
getState = async (): Promise<EgressState> => {
|
||||
return this.egressStateRepository.findSingleton();
|
||||
};
|
||||
|
||||
// Called when the feature is switched on or off: drop any pending recovery job and start from "ok".
|
||||
reset = async (): Promise<EgressState> => {
|
||||
await this.jobsRepository.deleteByIdAndType(null, "egress");
|
||||
return this.egressStateRepository.reset();
|
||||
};
|
||||
}
|
||||
@@ -2,50 +2,57 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { jobId, type JobSeed } from "@/domain/jobs/job.type.js";
|
||||
import { jobId, type Job, type JobSeed } from "@/domain/jobs/job.type.js";
|
||||
import type { INetworkService } from "@/service/networkService.js";
|
||||
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import {
|
||||
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
DEFAULT_EGRESS_TARGETS,
|
||||
type EgressAssessment,
|
||||
type EgressProbeResult,
|
||||
type EgressStatus,
|
||||
} from "@/domain/egress/egress.type.js";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { parseEgressTarget, type EgressTarget } from "@/utils/egressTarget.js";
|
||||
|
||||
const SERVICE_NAME = "EgressService";
|
||||
const PROBE_TIMEOUT_MS = 5000;
|
||||
// When egress is down every monitor fails at once, so concurrent failures share one assessment
|
||||
// rather than each probing. Same in-flight cache shape as ProxyResolver.
|
||||
const ASSESSMENT_TTL_MS = PROBE_TIMEOUT_MS;
|
||||
// How long a settled assessment answers for later failures. An assessment still in flight is shared
|
||||
// for as long as it takes, so this only bounds how stale a reused answer can be.
|
||||
const ASSESSMENT_REUSE_MS = 5000;
|
||||
// Global job row (refId null) that re-probes while degraded. Inserted when egress degrades, removed on recovery.
|
||||
const RECOVERY_JOB_ID = jobId("egress", null);
|
||||
const RECOVERY_JOB_TYPE = "egress" as const;
|
||||
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
|
||||
|
||||
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
|
||||
const PROBE_MONITOR_ID = "egress-probe";
|
||||
const PROBE_TEAM_ID = "system";
|
||||
|
||||
// "[host]:port" or "host:port" where host has no colons (so a bare IPv6 address is not mistaken for host:port).
|
||||
const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/;
|
||||
|
||||
export interface IEgressService {
|
||||
probeTargets(targets: string[]): Promise<EgressProbeResult[]>;
|
||||
// Called by the check producer when a check fails. Null when the feature is off or the assessment itself failed.
|
||||
assessAfterFailure(): Promise<EgressStatus | null>;
|
||||
// Handler for the "egress" job: re-probes while degraded and removes the job once egress is back.
|
||||
checkRecovery(): Promise<void>;
|
||||
checkRecovery(job: Job): Promise<void>;
|
||||
}
|
||||
|
||||
type CachedAssessment = {
|
||||
value: Promise<EgressAssessment | null>;
|
||||
expiresAt: number | null; // null while the assessment is still in flight
|
||||
};
|
||||
|
||||
export class EgressService implements IEgressService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private assessment: { value: Promise<EgressStatus | null>; expiresAt: number } | null = null;
|
||||
private assessment: CachedAssessment | null = null;
|
||||
|
||||
constructor(
|
||||
private settingsService: ISettingsService,
|
||||
private egressStateRepository: IEgressStateRepository,
|
||||
private jobsRepository: IJobsRepository,
|
||||
private networkService: INetworkService,
|
||||
private proxyResolver: IProxyResolver,
|
||||
private logger: ILogger
|
||||
) {}
|
||||
|
||||
@@ -54,46 +61,52 @@ export class EgressService implements IEgressService {
|
||||
// ****************************
|
||||
|
||||
// Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor.
|
||||
private toProbeMonitor = (target: string): Monitor => {
|
||||
// proxyMode "inherit" lets the resolver apply the global proxy to http probes, as it does for monitors.
|
||||
private toProbeMonitor = (target: EgressTarget, raw: string): Monitor => {
|
||||
const base = {
|
||||
id: PROBE_MONITOR_ID,
|
||||
teamId: PROBE_TEAM_ID,
|
||||
name: `Egress probe: ${target}`,
|
||||
name: `Egress probe: ${raw}`,
|
||||
method: "GET",
|
||||
proxyMode: "inherit",
|
||||
useAdvancedMatching: false,
|
||||
ignoreTlsErrors: false,
|
||||
customUpCodes: [],
|
||||
};
|
||||
|
||||
if (/^https?:\/\//i.test(target)) {
|
||||
return { ...base, type: "http", url: target } as unknown as Monitor;
|
||||
switch (target.kind) {
|
||||
case "http":
|
||||
return { ...base, type: "http", url: target.url } as unknown as Monitor;
|
||||
case "port":
|
||||
return { ...base, type: "port", url: target.host, port: target.port } as unknown as Monitor;
|
||||
case "ping":
|
||||
return { ...base, type: "ping", url: target.host } as unknown as Monitor;
|
||||
}
|
||||
|
||||
const hostPort = HOST_PORT_PATTERN.exec(target);
|
||||
if (hostPort) {
|
||||
const host = hostPort[1] ?? hostPort[2];
|
||||
return { ...base, type: "port", url: host, port: Number(hostPort[3]) } as unknown as Monitor;
|
||||
}
|
||||
|
||||
return { ...base, type: "ping", url: target } as unknown as Monitor;
|
||||
};
|
||||
|
||||
private probeTarget = async (target: string): Promise<EgressProbeResult> => {
|
||||
const monitor = this.toProbeMonitor(target);
|
||||
private probeTarget = async (raw: string): Promise<EgressProbeResult> => {
|
||||
const target = parseEgressTarget(raw);
|
||||
if (!target) {
|
||||
// Validation rejects these; a stored value can still be stale, and it must not count as reachable.
|
||||
return { target: raw, reachable: false, responseTime: 0, message: "Invalid egress target" };
|
||||
}
|
||||
|
||||
const monitor = this.toProbeMonitor(target, raw);
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const { response, responseTime, error } = await timeRequest(() =>
|
||||
Promise.race([
|
||||
this.networkService.requestStatus(monitor),
|
||||
const { response, responseTime, error } = await timeRequest(async () => {
|
||||
const proxyUrl = await this.proxyResolver.resolve(monitor);
|
||||
return Promise.race([
|
||||
this.networkService.requestStatus(monitor, { proxyUrl }),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS);
|
||||
}),
|
||||
]).finally(() => clearTimeout(timer))
|
||||
);
|
||||
]).finally(() => clearTimeout(timer));
|
||||
});
|
||||
|
||||
if (error || !response) {
|
||||
return {
|
||||
target,
|
||||
target: raw,
|
||||
reachable: false,
|
||||
responseTime,
|
||||
message: error instanceof Error ? error.message : error ? String(error) : "No response",
|
||||
@@ -101,7 +114,7 @@ export class EgressService implements IEgressService {
|
||||
}
|
||||
|
||||
return {
|
||||
target,
|
||||
target: raw,
|
||||
reachable: response.status === true,
|
||||
responseTime: response.responseTime ?? responseTime,
|
||||
message: response.message,
|
||||
@@ -127,32 +140,43 @@ export class EgressService implements IEgressService {
|
||||
// Event-triggered assessment
|
||||
// ****************************
|
||||
|
||||
assessAfterFailure = (): Promise<EgressStatus | null> => {
|
||||
const now = Date.now();
|
||||
if (this.assessment && this.assessment.expiresAt > now) {
|
||||
return this.assessment.value;
|
||||
}
|
||||
// Cache the in-flight promise so concurrent failures share one probe
|
||||
const value = this.assess();
|
||||
this.assessment = { value, expiresAt: now + ASSESSMENT_TTL_MS };
|
||||
// Don't cache an internal failure, so the next failing check retries
|
||||
void value.then((status) => {
|
||||
if (status === null && this.assessment?.value === value) this.assessment = null;
|
||||
});
|
||||
return value;
|
||||
assessAfterFailure = async (): Promise<EgressStatus | null> => {
|
||||
const assessment = await this.sharedAssessment();
|
||||
return assessment === "disabled" ? null : assessment;
|
||||
};
|
||||
|
||||
private assess = async (): Promise<EgressStatus | null> => {
|
||||
// When egress is down every monitor fails at once. Concurrent failures share the in-flight assessment
|
||||
// however long it takes, and a settled one answers for ASSESSMENT_REUSE_MS. Same shape as ProxyResolver's cache.
|
||||
private sharedAssessment = (): Promise<EgressAssessment | null> => {
|
||||
const cached = this.assessment;
|
||||
if (cached && (cached.expiresAt === null || cached.expiresAt > Date.now())) {
|
||||
return cached.value;
|
||||
}
|
||||
|
||||
const entry: CachedAssessment = { value: this.assess(), expiresAt: null };
|
||||
this.assessment = entry;
|
||||
void entry.value.then((result) => {
|
||||
if (this.assessment !== entry) return;
|
||||
if (result === null) {
|
||||
this.assessment = null; // an internal failure is not cached, so the next failing check retries
|
||||
} else {
|
||||
entry.expiresAt = Date.now() + ASSESSMENT_REUSE_MS;
|
||||
}
|
||||
});
|
||||
return entry.value;
|
||||
};
|
||||
|
||||
private assess = async (): Promise<EgressAssessment | null> => {
|
||||
try {
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
if (!settings.egressCheckEnabled) {
|
||||
return null;
|
||||
return "disabled";
|
||||
}
|
||||
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status === "degraded") {
|
||||
// Recovery is detected by the scheduled job. Make sure it exists in case its row was lost.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds);
|
||||
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
|
||||
return "degraded";
|
||||
}
|
||||
|
||||
@@ -173,7 +197,8 @@ export class EgressService implements IEgressService {
|
||||
method: "assessAfterFailure",
|
||||
details: { results },
|
||||
});
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds);
|
||||
// The worker that performed the transition owns the job's schedule; anyone else only makes sure the row exists.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, degraded !== null);
|
||||
return "degraded";
|
||||
} catch (error: unknown) {
|
||||
// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
|
||||
@@ -191,35 +216,42 @@ export class EgressService implements IEgressService {
|
||||
// Recovery job
|
||||
// ****************************
|
||||
|
||||
// upsertJob only sets nextScheduledAt on insert, so re-arming while degraded never pushes a pending run back.
|
||||
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined) => {
|
||||
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
|
||||
// so re-arming while degraded never pushes a pending run back.
|
||||
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
|
||||
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
|
||||
const seed: JobSeed = {
|
||||
id: RECOVERY_JOB_ID,
|
||||
type: "egress",
|
||||
type: RECOVERY_JOB_TYPE,
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: Date.now() + intervalMs,
|
||||
intervalMs,
|
||||
};
|
||||
await this.jobsRepository.upsertJob(seed);
|
||||
if (reschedule) {
|
||||
await this.jobsRepository.upsertCleanupJob(seed); // global-row upsert that also sets the schedule
|
||||
} else {
|
||||
await this.jobsRepository.upsertJob(seed);
|
||||
}
|
||||
};
|
||||
|
||||
private removeRecoveryJob = async () => {
|
||||
await this.jobsRepository.deleteByIdAndType(null, "egress");
|
||||
// Removes the row only if nobody has rescheduled it since this run claimed it. Another worker may have seen
|
||||
// egress fail again in the meantime and started a new episode; its schedule must survive.
|
||||
private releaseRecoveryJob = async (job: Job) => {
|
||||
await this.jobsRepository.deleteGlobalJobIfUnchanged(RECOVERY_JOB_TYPE, job.nextScheduledAt);
|
||||
};
|
||||
|
||||
// Runs on the queue at the configured interval while degraded. Errors propagate so the queue records
|
||||
// the failure and retries with its usual backoff.
|
||||
checkRecovery = async (): Promise<void> => {
|
||||
checkRecovery = async (job: Job): Promise<void> => {
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status !== "degraded") {
|
||||
// Another worker recorded the recovery, or the state was reset; nothing left to poll for.
|
||||
await this.removeRecoveryJob();
|
||||
if (!settings.egressCheckEnabled || state.status !== "degraded") {
|
||||
// Feature switched off, recovery recorded by another worker, or state reset: nothing left to poll for.
|
||||
await this.releaseRecoveryJob(job);
|
||||
return;
|
||||
}
|
||||
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
|
||||
const now = new Date();
|
||||
|
||||
@@ -229,7 +261,7 @@ export class EgressService implements IEgressService {
|
||||
}
|
||||
|
||||
const recovered = await this.egressStateRepository.markRecovered(results, now);
|
||||
await this.removeRecoveryJob();
|
||||
await this.releaseRecoveryJob(job);
|
||||
if (!recovered) {
|
||||
// Another process performed the transition.
|
||||
return;
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
export const EgressStatuses = ["ok", "degraded"] as const;
|
||||
export type EgressStatus = (typeof EgressStatuses)[number];
|
||||
// Outcome of one assessment: a status, or "disabled" when the feature is switched off (distinct from an internal failure).
|
||||
export type EgressAssessment = EgressStatus | "disabled";
|
||||
|
||||
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
|
||||
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
|
||||
|
||||
@@ -61,6 +61,9 @@ export interface IJobsRepository {
|
||||
// Delete a single job of type. refId null addresses a global row
|
||||
deleteByIdAndType(refId: string | null, type: JobType): Promise<boolean>;
|
||||
|
||||
// Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged)
|
||||
deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise<boolean>;
|
||||
|
||||
// Delete jobs that reference monitors that no longer exist
|
||||
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
|
||||
|
||||
|
||||
@@ -278,6 +278,11 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteGlobalJobIfUnchanged = async (type: JobType, nextScheduledAt: number) => {
|
||||
const res = await JobModel.deleteOne({ _id: jobId(type, null), nextScheduledAt });
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
|
||||
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
|
||||
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
|
||||
|
||||
@@ -178,7 +178,18 @@ export class NotificationsService implements INotificationsService {
|
||||
|
||||
deleteById = async (id: string, teamId: string): Promise<Notification> => {
|
||||
await this.monitorsRepository.removeNotificationFromMonitors(id);
|
||||
await this.removeFromEgressNotifications(id);
|
||||
const deleted = await this.notificationsRepository.deleteById(id, teamId);
|
||||
return deleted;
|
||||
};
|
||||
|
||||
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles
|
||||
private removeFromEgressNotifications = async (id: string) => {
|
||||
const settings = await this.settingsService.getDBSettings();
|
||||
const current = settings.egressNotifications ?? [];
|
||||
const remaining = current.filter((notificationId) => notificationId !== id);
|
||||
if (remaining.length !== current.length) {
|
||||
await this.settingsService.updateDbSettings({ egressNotifications: remaining });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import ping from "ping";
|
||||
import net from "net";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
const SERVICE_NAME = "PingProvider";
|
||||
|
||||
@@ -19,10 +20,13 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
|
||||
}
|
||||
|
||||
private sanitizeHost(url: string): string {
|
||||
return url
|
||||
.replace(/^https?:\/\//, "")
|
||||
.replace(/\/.*$/, "")
|
||||
.replace(/:.*/, "");
|
||||
const host = url.replace(/^https?:\/\//, "").replace(/\/.*$/, "");
|
||||
// "[2001:db8::1]" or "[2001:db8::1]:53": the address is inside the brackets
|
||||
const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host);
|
||||
if (bracketed) return bracketed[1]!;
|
||||
// A bare IPv6 address contains colons that are not a port separator
|
||||
if (net.isIPv6(host)) return host;
|
||||
return host.replace(/:.*/, "");
|
||||
}
|
||||
|
||||
async handle(monitor: Monitor): Promise<MonitorStatusResponse<PingStatusPayload>> {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import { z } from "zod";
|
||||
import { dnsHostnameRegex } from "@/api/validation/shared.js";
|
||||
|
||||
// A reliability target for the egress self-check, in one of three shapes. Parsed here so the
|
||||
// settings validator and the probe agree on exactly what is accepted.
|
||||
export type EgressTarget =
|
||||
| { kind: "http"; url: string } // http(s):// URL, fetched
|
||||
| { kind: "port"; host: string; port: number } // host:port or [ipv6]:port, TCP connect
|
||||
| { kind: "ping"; host: string }; // bare hostname or IP address, ICMP
|
||||
|
||||
// "[ipv6]:port" or "host:port" where host has no colons, so a bare IPv6 address is never read as host:port.
|
||||
const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s[\]]+)):(\d{1,5})$/;
|
||||
const BRACKETED_PATTERN = /^\[([^\]]+)\]$/;
|
||||
|
||||
const isIpv6 = (value: string): boolean => z.ipv6().safeParse(value).success;
|
||||
const isHostOrIpv4 = (value: string): boolean => value === "localhost" || z.ipv4().safeParse(value).success || dnsHostnameRegex.test(value);
|
||||
|
||||
const parseHttp = (target: string): EgressTarget | null => {
|
||||
try {
|
||||
const { protocol, hostname } = new URL(target);
|
||||
return (protocol === "http:" || protocol === "https:") && hostname.length > 0 ? { kind: "http", url: target } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const parseEgressTarget = (raw: string): EgressTarget | null => {
|
||||
const target = raw.trim();
|
||||
if (target.length === 0) return null;
|
||||
|
||||
if (/^https?:\/\//i.test(target)) return parseHttp(target);
|
||||
|
||||
const hostPort = HOST_PORT_PATTERN.exec(target);
|
||||
if (hostPort) {
|
||||
const port = Number(hostPort[3]);
|
||||
if (port < 1 || port > 65535) return null;
|
||||
const bracketedHost = hostPort[1];
|
||||
if (bracketedHost !== undefined) return isIpv6(bracketedHost) ? { kind: "port", host: bracketedHost, port } : null;
|
||||
const host = hostPort[2]!;
|
||||
return isHostOrIpv4(host) ? { kind: "port", host, port } : null;
|
||||
}
|
||||
|
||||
const bracketed = BRACKETED_PATTERN.exec(target);
|
||||
if (bracketed) return isIpv6(bracketed[1]!) ? { kind: "ping", host: bracketed[1]! } : null;
|
||||
|
||||
if (isIpv6(target) || isHostOrIpv4(target)) return { kind: "ping", host: target };
|
||||
return null;
|
||||
};
|
||||
@@ -10,7 +10,6 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
@@ -45,7 +44,6 @@ export interface DBQueueWorkerDependencies {
|
||||
geoCheckPipeline: ICheckPipeline;
|
||||
dispatcher: IReactorDispatcher;
|
||||
helper: IWorkerHelper;
|
||||
egressService: IEgressService;
|
||||
queueWorkersRepository: IQueueWorkersRepository; // forwarded
|
||||
queueMode: QueueMode; // forwarded
|
||||
queuePrimaryProcesses: boolean; // consumed immediately
|
||||
@@ -74,7 +72,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
private dispatcher: IReactorDispatcher;
|
||||
private geoCheckPipeline: ICheckPipeline;
|
||||
private helper: IWorkerHelper;
|
||||
private egressService: IEgressService;
|
||||
private bufferService: IBufferService;
|
||||
private isDbConnected: () => boolean;
|
||||
|
||||
@@ -95,7 +92,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
this.dispatcher = dependencies.dispatcher;
|
||||
this.geoCheckPipeline = dependencies.geoCheckPipeline;
|
||||
this.helper = dependencies.helper;
|
||||
this.egressService = dependencies.egressService;
|
||||
this.bufferService = dependencies.bufferService;
|
||||
this.isDbConnected = dependencies.isDbConnected;
|
||||
}
|
||||
@@ -182,7 +178,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
await this.helper.getCleanupRetentionJob()(); // Get job and execute
|
||||
break;
|
||||
case "egress":
|
||||
await this.egressService.checkRecovery(); // Re-probe while degraded; removes its own row on recovery
|
||||
await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,10 +11,13 @@ import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import type { Job } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export interface IWorkerHelper {
|
||||
getCleanupOrphanedJob(): () => Promise<void>;
|
||||
getCleanupRetentionJob(): () => Promise<void>;
|
||||
getEgressRecoveryJob(): (job: Job) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface MonitorActionDecision {
|
||||
@@ -45,6 +48,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
private incidentsRepository: IIncidentsRepository;
|
||||
private geoChecksRepository: IGeoChecksRepository;
|
||||
private dockerLogsRepository: IDockerLogsRepository;
|
||||
private egressService: IEgressService;
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
@@ -57,7 +61,8 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
checksRepository: IChecksRepository,
|
||||
incidentsRepository: IIncidentsRepository,
|
||||
geoChecksRepository: IGeoChecksRepository,
|
||||
dockerLogsRepository: IDockerLogsRepository
|
||||
dockerLogsRepository: IDockerLogsRepository,
|
||||
egressService: IEgressService
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.checkService = checkService;
|
||||
@@ -70,8 +75,17 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
this.incidentsRepository = incidentsRepository;
|
||||
this.geoChecksRepository = geoChecksRepository;
|
||||
this.dockerLogsRepository = dockerLogsRepository;
|
||||
this.egressService = egressService;
|
||||
}
|
||||
|
||||
// Runs while instance egress is degraded; the service removes the row once egress is back.
|
||||
// Errors propagate so the queue records the failure and retries with its usual backoff.
|
||||
getEgressRecoveryJob = () => {
|
||||
return async (job: Job) => {
|
||||
await this.egressService.checkRecovery(job);
|
||||
};
|
||||
};
|
||||
|
||||
getCleanupOrphanedJob = () => {
|
||||
return async () => {
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user