Address review of degraded-egress check persistence
CheckProducer consults egress only for a transport failure: the check
failed, no HTTP status came back (isHttpStatusCode from PR 1), and the
monitor type is one whose failure can be a loss of egress
(EgressAttributableTypes; hardware and docker are excluded). A 4xx/5xx
or a content mismatch is evaluated as usual while the instance is
degraded. A rejected assessment is logged and treated as unknown so the
check is still buffered.
EXCLUDE_DEGRADED_EGRESS_MATCH moves out of the model into
check.query.ts, with an expression form for $group accumulators.
Daily status buckets and docker stats buckets exclude degraded checks
from their up/total counts while keeping them in the response-time
average, matching the uptime facet. toEntity maps egressStatus like its
siblings.
Adds heartbeatJobDegradedEgress integration tests, extends the
repository exclusion tests, and makes the harness egress stub settable.
🔧
This commit is contained in:
1 parent
4f5dba560d
commit
4335e6b7b6
13 files changed
+334
-29
No files matched your search
@@ -11,7 +11,9 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { isEgressAttributable } from "@/domain/monitors/monitor.type.js";
|
||||
|
||||
export interface ICheckProducer {
|
||||
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
|
||||
@@ -44,6 +46,26 @@ export class CheckProducer implements ICheckProducer {
|
||||
return dockerTlsKey ?? undefined;
|
||||
}
|
||||
|
||||
// Only a failure to reach the target at all can be the instance's own fault. Any HTTP response (4xx, 5xx,
|
||||
// or a 200 with a content mismatch) proves the target was reached, and hardware/docker failures are local.
|
||||
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
|
||||
return status.status === false && isEgressAttributable(monitor.type) && !isHttpStatusCode(status.code);
|
||||
}
|
||||
|
||||
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
|
||||
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
|
||||
try {
|
||||
return await this.egressService.assessAfterFailure();
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "assessEgress",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
produce = async (monitor: Monitor) => {
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
|
||||
@@ -76,9 +98,9 @@ export class CheckProducer implements ICheckProducer {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target.
|
||||
// Step 1c: On a transport failure, ask whether the instance itself can reach anything before blaming the target.
|
||||
// Null means the egress check is disabled (or failed internally) and the check is treated as usual.
|
||||
const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null;
|
||||
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
|
||||
Reference in new issue
Block a user