Address review of degraded-egress check persistence
CheckProducer consults egress only for a transport failure: the check
failed, no HTTP status came back (isHttpStatusCode from PR 1), and the
monitor type is one whose failure can be a loss of egress
(EgressAttributableTypes; hardware and docker are excluded). A 4xx/5xx
or a content mismatch is evaluated as usual while the instance is
degraded. A rejected assessment is logged and treated as unknown so the
check is still buffered.
EXCLUDE_DEGRADED_EGRESS_MATCH moves out of the model into
check.query.ts, with an expression form for $group accumulators.
Daily status buckets and docker stats buckets exclude degraded checks
from their up/total counts while keeping them in the response-time
average, matching the uptime facet. toEntity maps egressStatus like its
siblings.
Adds heartbeatJobDegradedEgress integration tests, extends the
repository exclusion tests, and makes the harness egress stub settable.
🔧
This commit is contained in:
1 parent
4f5dba560d
commit
4335e6b7b6
13 files changed
+334
-29
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
|
||||
import CheckModel from "@/domain/checks/check.model.js";
|
||||
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
|
||||
import mongoose from "mongoose";
|
||||
|
||||
type DateRange = { start: Date; end: Date };
|
||||
@@ -35,8 +36,9 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
|
||||
$group: {
|
||||
_id: { $dateToString: { format: dateString, date: "$createdAt" } },
|
||||
avgResponseTime: { $avg: "$responseTime" },
|
||||
upCount: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
|
||||
totalCount: { $sum: 1 },
|
||||
// Counts exclude degraded-egress checks like aggregateData.totalChecks does; the response-time average keeps them.
|
||||
upCount: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
|
||||
totalCount: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
|
||||
avgRunning: { $avg: "$containerSummary.running" },
|
||||
avgTotal: { $avg: "$containerSummary.total" },
|
||||
avgUnhealthy: { $avg: "$containerSummary.unhealthy" },
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
|
||||
import { CheckModel } from "@/domain/checks/check.model.js";
|
||||
import mongoose from "mongoose";
|
||||
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
|
||||
|
||||
type DateRange = { start: Date; end: Date };
|
||||
type HardwareUpChecks = { totalChecks: number };
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Schema, model, Types } from "mongoose";
|
||||
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { EgressStatuses } from "@/domain/egress/egress.type.js";
|
||||
import type {
|
||||
Check,
|
||||
CheckAudits,
|
||||
@@ -365,10 +365,6 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
|
||||
|
||||
const CheckModel = model<CheckDocument>("Check", CheckSchema);
|
||||
|
||||
// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
|
||||
// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query.
|
||||
const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
|
||||
|
||||
export type { CheckDocument, CheckMetadataDocument };
|
||||
export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH };
|
||||
export { CheckModel, containerSummarySchema };
|
||||
export default CheckModel;
|
||||
@@ -0,0 +1,8 @@
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
|
||||
// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
|
||||
// are left out of every uptime percentage and up/down count. Response-time series keep them.
|
||||
// Spread this into the $match / filter of a query, or use the expression form inside a $group accumulator.
|
||||
export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
|
||||
|
||||
export const IS_NOT_DEGRADED_EGRESS_EXPR = { $ne: ["$egressStatus", "degraded" satisfies EgressStatus] } as const;
|
||||
@@ -15,7 +15,8 @@ import type {
|
||||
HardwareCheckStats,
|
||||
} from "@/domain/checks/check.type.js";
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js";
|
||||
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
|
||||
import mongoose from "mongoose";
|
||||
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
@@ -179,7 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
audits: mapAudits(doc.audits),
|
||||
containers: doc.containers,
|
||||
containerSummary: doc.containerSummary,
|
||||
...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }),
|
||||
egressStatus: doc.egressStatus,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
@@ -395,7 +396,6 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
$match: {
|
||||
"metadata.monitorId": { $in: objectIds },
|
||||
createdAt: { $gte: windowStart },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -404,11 +404,14 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
monitorId: "$metadata.monitorId",
|
||||
day: { $dateTrunc: { date: "$createdAt", unit: "day", timezone } },
|
||||
},
|
||||
totalChecks: { $sum: 1 },
|
||||
upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
|
||||
// Counts exclude degraded-egress checks; the response-time average keeps them.
|
||||
totalChecks: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
|
||||
upChecks: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
|
||||
avgResponseTime: { $avg: "$responseTime" },
|
||||
},
|
||||
},
|
||||
// A day with nothing attributable to the target renders as an empty bar; the client divides by totalChecks for the tooltip.
|
||||
{ $match: { totalChecks: { $gt: 0 } } },
|
||||
{ $sort: { "_id.day": 1 } },
|
||||
{
|
||||
$project: {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
|
||||
import type { Response } from "got";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
|
||||
export const CHECK_TTL_SENTINEL = 366;
|
||||
|
||||
@@ -117,8 +117,9 @@ export interface Check {
|
||||
seo?: number;
|
||||
performance?: number;
|
||||
audits?: CheckAudits;
|
||||
// Set only on checks produced while the reliability probe ran (a failing check).
|
||||
// "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target.
|
||||
// Set on failing checks while the egress check is enabled; absent otherwise.
|
||||
// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
|
||||
// "ok": a reliability target was reachable.
|
||||
egressStatus?: EgressStatus;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
|
||||
@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
|
||||
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
|
||||
|
||||
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
|
||||
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
|
||||
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
|
||||
const PROBE_MONITOR_ID = "egress-probe";
|
||||
|
||||
@@ -50,6 +50,21 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
|
||||
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
|
||||
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
|
||||
|
||||
// Types whose transport failure can be the instance's own loss of egress. Hardware (Capture agent) and docker
|
||||
// (local socket or host) failures are local, so they are never attributed to egress.
|
||||
export const EgressAttributableTypes = [
|
||||
"http",
|
||||
"ping",
|
||||
"pagespeed",
|
||||
"port",
|
||||
"game",
|
||||
"grpc",
|
||||
"websocket",
|
||||
"dns",
|
||||
] as const satisfies readonly MonitorType[];
|
||||
export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
|
||||
export const isEgressAttributable = (type: MonitorType): type is EgressAttributableType => EgressAttributableTypes.some((t) => t === type);
|
||||
|
||||
export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
|
||||
export type MonitorStatus = (typeof MonitorStatuses)[number];
|
||||
|
||||
|
||||
@@ -11,7 +11,9 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { isEgressAttributable } from "@/domain/monitors/monitor.type.js";
|
||||
|
||||
export interface ICheckProducer {
|
||||
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
|
||||
@@ -44,6 +46,26 @@ export class CheckProducer implements ICheckProducer {
|
||||
return dockerTlsKey ?? undefined;
|
||||
}
|
||||
|
||||
// Only a failure to reach the target at all can be the instance's own fault. Any HTTP response (4xx, 5xx,
|
||||
// or a 200 with a content mismatch) proves the target was reached, and hardware/docker failures are local.
|
||||
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
|
||||
return status.status === false && isEgressAttributable(monitor.type) && !isHttpStatusCode(status.code);
|
||||
}
|
||||
|
||||
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
|
||||
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
|
||||
try {
|
||||
return await this.egressService.assessAfterFailure();
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "assessEgress",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
produce = async (monitor: Monitor) => {
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
|
||||
@@ -76,9 +98,9 @@ export class CheckProducer implements ICheckProducer {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target.
|
||||
// Step 1c: On a transport failure, ask whether the instance itself can reach anything before blaming the target.
|
||||
// Null means the egress check is disabled (or failed internally) and the check is treated as usual.
|
||||
const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null;
|
||||
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
|
||||
Reference in new issue
Block a user