Flag checks taken during degraded egress and exclude them from uptime (#3937)
CheckProducer asks EgressService to assess the instance's own connectivity
after a failing check and stamps the result on the check as egressStatus.
CheckEvaluator short-circuits degraded checks so no incident opens and no
spurious recovery fires later. Uptime and down-count aggregations exclude
degraded checks; the response-time series keeps them.
🔧
This commit is contained in:
1 parent
f4864b5455
commit
f52acb3f68
11 files changed
+172
-8
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import CheckModel from "@/domain/checks/check.model.js";
|
||||
import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
|
||||
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
|
||||
import mongoose from "mongoose";
|
||||
|
||||
@@ -9,6 +9,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
});
|
||||
|
||||
export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { CheckModel } from "@/domain/checks/check.model.js";
|
||||
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js";
|
||||
import mongoose from "mongoose";
|
||||
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
|
||||
|
||||
@@ -70,6 +70,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "hardware",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Schema, model, Types } from "mongoose";
|
||||
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import type {
|
||||
Check,
|
||||
CheckAudits,
|
||||
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
|
||||
type: containerSummarySchema,
|
||||
default: undefined,
|
||||
},
|
||||
egressStatus: {
|
||||
type: String,
|
||||
enum: EgressStatuses,
|
||||
default: undefined,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
@@ -359,6 +365,10 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
|
||||
|
||||
const CheckModel = model<CheckDocument>("Check", CheckSchema);
|
||||
|
||||
// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
|
||||
// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query.
|
||||
const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
|
||||
|
||||
export type { CheckDocument, CheckMetadataDocument };
|
||||
export { CheckModel, containerSummarySchema };
|
||||
export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH };
|
||||
export default CheckModel;
|
||||
@@ -15,7 +15,7 @@ import type {
|
||||
HardwareCheckStats,
|
||||
} from "@/domain/checks/check.type.js";
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
|
||||
import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js";
|
||||
import mongoose from "mongoose";
|
||||
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
@@ -179,6 +179,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
audits: mapAudits(doc.audits),
|
||||
containers: doc.containers,
|
||||
containerSummary: doc.containerSummary,
|
||||
...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }),
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
@@ -348,6 +349,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const baseMatch = {
|
||||
"metadata.teamId": new mongoose.Types.ObjectId(teamId),
|
||||
createdAt: { $gte: getDateForRange(dateRange) },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
};
|
||||
|
||||
const [totalResult, downResult] = await Promise.all([
|
||||
@@ -393,6 +395,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
$match: {
|
||||
"metadata.monitorId": { $in: objectIds },
|
||||
createdAt: { $gte: windowStart },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -480,6 +483,8 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
{
|
||||
$facet: {
|
||||
uptimePercentage: [
|
||||
// Response-time series (groupedUpChecks / groupedDownChecks) deliberately keep degraded checks so they still show as failures on the graph.
|
||||
{ $match: EXCLUDE_DEGRADED_EGRESS_MATCH },
|
||||
{
|
||||
$group: {
|
||||
_id: null,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
|
||||
import type { Response } from "got";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
|
||||
export const CHECK_TTL_SENTINEL = 366;
|
||||
|
||||
@@ -116,6 +117,9 @@ export interface Check {
|
||||
seo?: number;
|
||||
performance?: number;
|
||||
audits?: CheckAudits;
|
||||
// Set only on checks produced while the reliability probe ran (a failing check).
|
||||
// "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target.
|
||||
egressStatus?: EgressStatus;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user