Flag checks taken during degraded egress and exclude them from uptime (#3937)

CheckProducer asks EgressService to assess the instance's own connectivity
after a failing check and stamps the result on the check as egressStatus.
CheckEvaluator short-circuits degraded checks so no incident opens and no
spurious recovery fires later. Uptime and down-count aggregations exclude
degraded checks; the response-time series keeps them.

🔧
This commit is contained in:
Craig Lawson committed 2026-09-17 22:06:16 +01:00
1 parent f4864b5455
commit f52acb3f68
11 files changed
+172 -8

No files matched your search

@@ -1,4 +1,4 @@
import CheckModel from "@/domain/checks/check.model.js";
import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
import mongoose from "mongoose";
@@ -9,6 +9,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
@@ -1,4 +1,4 @@
import { CheckModel } from "@/domain/checks/check.model.js";
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
import mongoose from "mongoose";
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
@@ -70,6 +70,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
};
+11 -1
View File
@@ -1,5 +1,6 @@
import { Schema, model, Types } from "mongoose";
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js";
import type {
Check,
CheckAudits,
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
type: containerSummarySchema,
default: undefined,
},
egressStatus: {
type: String,
enum: EgressStatuses,
default: undefined,
},
},
{
timestamps: true,
@@ -359,6 +365,10 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
const CheckModel = model<CheckDocument>("Check", CheckSchema);
// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query.
const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
export type { CheckDocument, CheckMetadataDocument };
export { CheckModel, containerSummarySchema };
export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH };
export default CheckModel;
@@ -15,7 +15,7 @@ import type {
HardwareCheckStats,
} from "@/domain/checks/check.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js";
import mongoose from "mongoose";
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
import { ILogger } from "@/utils/logger.js";
@@ -179,6 +179,7 @@ class MongoChecksRepository implements IChecksRepository {
audits: mapAudits(doc.audits),
containers: doc.containers,
containerSummary: doc.containerSummary,
...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }),
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
};
@@ -348,6 +349,7 @@ class MongoChecksRepository implements IChecksRepository {
const baseMatch = {
"metadata.teamId": new mongoose.Types.ObjectId(teamId),
createdAt: { $gte: getDateForRange(dateRange) },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [totalResult, downResult] = await Promise.all([
@@ -393,6 +395,7 @@ class MongoChecksRepository implements IChecksRepository {
$match: {
"metadata.monitorId": { $in: objectIds },
createdAt: { $gte: windowStart },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{
@@ -480,6 +483,8 @@ class MongoChecksRepository implements IChecksRepository {
{
$facet: {
uptimePercentage: [
// Response-time series (groupedUpChecks / groupedDownChecks) deliberately keep degraded checks so they still show as failures on the graph.
{ $match: EXCLUDE_DEGRADED_EGRESS_MATCH },
{
$group: {
_id: null,
+4
View File
@@ -1,6 +1,7 @@
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { Response } from "got";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
export const CHECK_TTL_SENTINEL = 366;
@@ -116,6 +117,9 @@ export interface Check {
seo?: number;
performance?: number;
audits?: CheckAudits;
// Set only on checks produced while the reliability probe ran (a failing check).
// "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target.
egressStatus?: EgressStatus;
createdAt: string;
updatedAt: string;
}