port egress to pipeline
This commit is contained in:
1 parent
01e9655f42
commit
8dd57d45e2
27 files changed
+19071
-15939
No files matched your search
@@ -0,0 +1,336 @@
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { isEgressAttributable, Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { IBufferService } from "@/service/bufferService.js";
|
||||
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import { INetworkService } from "@/service/networkService.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
import { DockerStatusPayload, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
|
||||
const SERVICE_NAME = "WorkerPipeline";
|
||||
|
||||
// The three entry points: the queue calls the two handlers, the buffer calls ingestChecks.
|
||||
// produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer.
|
||||
export interface IWorkerPipeline {
|
||||
handleCheck: JobHandler;
|
||||
ingestChecks(checks: Check[]): Promise<void>;
|
||||
handleEvaluate: JobHandler;
|
||||
}
|
||||
|
||||
export interface WorkerPipelineDependencies {
|
||||
logger: ILogger;
|
||||
monitorsRepository: IMonitorsRepository;
|
||||
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
checksRepository: IChecksRepository;
|
||||
jobsRepository: IJobsRepository;
|
||||
checkService: ICheckService;
|
||||
networkService: INetworkService;
|
||||
proxyResolver: IProxyResolver;
|
||||
bufferService: IBufferService;
|
||||
dockerLogsService: IDockerLogsService;
|
||||
egressService: IEgressService;
|
||||
statusService: IStatusService;
|
||||
dispatcher: IReactorDispatcher;
|
||||
}
|
||||
|
||||
export class WorkerPipeline implements IWorkerPipeline {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private unarmed: Map<string, PendingCheck[]> = new Map();
|
||||
|
||||
constructor(private deps: WorkerPipelineDependencies) {}
|
||||
|
||||
private isInMaintenanceWindow = async (monitorId: string, teamId: string) => {
|
||||
const windows = await this.deps.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
|
||||
const now = new Date();
|
||||
return windows.some((window) => isWindowActive(window, now));
|
||||
};
|
||||
|
||||
// ****************************************************************
|
||||
// Stage 1: Check job. Probe the host and buffer the result.
|
||||
// ****************************************************************
|
||||
|
||||
handleCheck: JobHandler = async (job) => {
|
||||
if (!job.refId) return;
|
||||
const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId);
|
||||
if (!monitor) return;
|
||||
await this.produce(monitor);
|
||||
};
|
||||
|
||||
produce = async (monitor: Monitor) => {
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
|
||||
}
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
// ****************************
|
||||
|
||||
// Step 1a: Maintenance window gate - skip if in maintenance
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.deps.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
});
|
||||
if (monitor.status !== "maintenance") {
|
||||
// Clear the status window to avoid incidents being created on next check
|
||||
await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the
|
||||
// degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is
|
||||
// evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage.
|
||||
if (monitor.status === "maintenance") {
|
||||
monitor = await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" });
|
||||
}
|
||||
|
||||
// Step 1b: Acquire status
|
||||
const proxyUrl = await this.deps.proxyResolver.resolve(monitor);
|
||||
const dockerTlsKey = await this.resolveDockerTlsKey(monitor);
|
||||
|
||||
const status = await this.deps.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey });
|
||||
if (!status) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
// ****************************
|
||||
|
||||
// Step 2a: Create & record a check, return null if fail.
|
||||
// Built before the egress assessment below so that the probe's duration stays out of `createdAt`.
|
||||
const check = this.deps.checkService.toCheck(status);
|
||||
if (!check) {
|
||||
this.deps.logger.warn({
|
||||
message: `No check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
details: { code: status.code, message: status.message },
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
|
||||
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
|
||||
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
|
||||
if (egressStatus !== null) {
|
||||
check.egressStatus = egressStatus;
|
||||
}
|
||||
|
||||
// Step 2c: Add to buffer
|
||||
this.deps.bufferService.addToBuffer(check);
|
||||
|
||||
// Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops
|
||||
// being evaluated, so the results either side of a spell never end up adjacent and cross the threshold
|
||||
// on stale data. The length guard keeps this to one write per spell rather than one per degraded check,
|
||||
// and it runs after the check is buffered so that a failed write cannot cost us the check.
|
||||
if (egressStatus === "degraded" && monitor.statusWindow?.length) {
|
||||
await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] });
|
||||
}
|
||||
|
||||
// Step 2e: Handle docker logs
|
||||
if (status.type === "docker") {
|
||||
const dockerLogs = await this.deps.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
|
||||
for (const dockerLog of dockerLogs) {
|
||||
this.deps.bufferService.addDockerLogToBuffer(dockerLog);
|
||||
}
|
||||
}
|
||||
return { status, check };
|
||||
};
|
||||
|
||||
private async resolveDockerTlsKey(monitor: Monitor): Promise<string | undefined> {
|
||||
if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined;
|
||||
const dockerTlsKey = await this.deps.monitorsRepository.findDockerTlsKeyById(monitor.id);
|
||||
return dockerTlsKey ?? undefined;
|
||||
}
|
||||
|
||||
// Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the
|
||||
// two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx,
|
||||
// or a 200 with a content mismatch).
|
||||
private peerAnswered(status: MonitorStatusResponse): boolean {
|
||||
return status.peerResponded ?? isHttpStatusCode(status.code);
|
||||
}
|
||||
|
||||
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
|
||||
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
|
||||
}
|
||||
|
||||
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
|
||||
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
|
||||
try {
|
||||
return await this.deps.egressService.assessAfterFailure();
|
||||
} catch (error: unknown) {
|
||||
this.deps.logger.warn({
|
||||
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "assessEgress",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ****************************************************************
|
||||
// Stage 2: Ingestion
|
||||
// Called by the buffer, checks are inserted and evaluation jobs created
|
||||
// ****************************************************************
|
||||
|
||||
ingestChecks = async (checks: Check[]) => {
|
||||
if (checks.length > 0) await this.deps.checkService.createChecks(checks);
|
||||
|
||||
// Group by monitor, starting from any entries a previous flush failed to arm
|
||||
const byMonitor = this.unarmed;
|
||||
this.unarmed = new Map();
|
||||
for (const check of checks) {
|
||||
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
|
||||
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
|
||||
byMonitor.set(check.metadata.monitorId, pending);
|
||||
}
|
||||
if (byMonitor.size === 0) return;
|
||||
|
||||
const now = Date.now();
|
||||
const ops: Promise<boolean>[] = [];
|
||||
for (const [monitorId, pendingChecks] of byMonitor) {
|
||||
ops.push(
|
||||
this.deps.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
|
||||
// The checks are already stored, so a failed arm is retried on the next ingest rather than dropped
|
||||
this.unarmed.set(monitorId, pendingChecks);
|
||||
this.deps.logger.error({
|
||||
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next ingest: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "ingestChecks",
|
||||
});
|
||||
return false;
|
||||
})
|
||||
);
|
||||
}
|
||||
await Promise.all(ops);
|
||||
};
|
||||
|
||||
// ****************************************************************
|
||||
// Stage 3: Evaluation
|
||||
// Apply each pending check to the monitor -> decide -> react.
|
||||
// ****************************************************************
|
||||
|
||||
handleEvaluate: JobHandler = async (job) => {
|
||||
if (!job.refId || job.pendingChecks.length === 0) return;
|
||||
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
|
||||
|
||||
const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||
if (!monitor) {
|
||||
await this.deps.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
|
||||
return;
|
||||
}
|
||||
const checks = await this.deps.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
|
||||
|
||||
let current = monitor;
|
||||
for (const check of checks) {
|
||||
const status = this.deps.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.evaluateCheck(status, check, current);
|
||||
await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
if (!leaseHeld) {
|
||||
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
|
||||
this.deps.logger.warn({
|
||||
message: `Lost lease on ${job.id} after check ${check.id}, stopping`,
|
||||
service: SERVICE_NAME,
|
||||
method: "handleEvaluate",
|
||||
});
|
||||
return;
|
||||
}
|
||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||
}
|
||||
|
||||
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
|
||||
const found = new Set(checks.map((check) => check.id));
|
||||
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
|
||||
if (missing.length > 0) {
|
||||
this.deps.logger.warn({
|
||||
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "handleEvaluate",
|
||||
});
|
||||
await this.deps.jobsRepository.pullEvaluated(job.id, missing);
|
||||
}
|
||||
};
|
||||
|
||||
// A check taken while the instance had no outbound connectivity says nothing about the target.
|
||||
// It is left out of the status window, running stats and monitor status entirely, so that no incident
|
||||
// opens for it and no spurious "resolved"/"up" fires once egress returns.
|
||||
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
|
||||
this.deps.logger.debug({
|
||||
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
|
||||
service: SERVICE_NAME,
|
||||
method: "evaluateCheck",
|
||||
});
|
||||
const decision: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
return {
|
||||
monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
|
||||
decision,
|
||||
};
|
||||
};
|
||||
|
||||
evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
|
||||
if (check.egressStatus === "degraded") {
|
||||
return this.skipDegradedEgressCheck(status, check, monitor);
|
||||
}
|
||||
|
||||
const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor);
|
||||
|
||||
const decision = this.decide(statusChange);
|
||||
return { monitor: statusChange.monitor, status, check, statusChange, decision };
|
||||
};
|
||||
|
||||
private decide = (statusChange: StatusChangeResult): MonitorActionDecision => {
|
||||
const { monitor, statusChanged, prevStatus } = statusChange;
|
||||
const decision: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
if (!statusChanged) return decision;
|
||||
|
||||
if (monitor.status === "down") {
|
||||
decision.shouldCreateIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "status_down";
|
||||
decision.notificationReason = "status_change";
|
||||
} else if (monitor.status === "breached") {
|
||||
decision.shouldCreateIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "threshold_breach";
|
||||
decision.notificationReason = "threshold_breach";
|
||||
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
|
||||
decision.shouldResolveIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.notificationReason = "status_change";
|
||||
}
|
||||
return decision;
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user