TRX/Checkmate
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Exclude degraded-egress checks from every aggregate

Browse Source 
A degraded episode now reads as a gap in monitoring rather than as downtime
or as a dip in response time. The exclusion is applied once per query, in the
top-level $match, so no pipeline can count a degraded check by omission.

- check.query.ts: EXCLUDE_DEGRADED_EGRESS_MATCH is the only helper;
  IS_NOT_DEGRADED_EGRESS_EXPR and the $cond accumulators it fed are gone.
- findUptimeDateRangeChecks: excludes at the match, so the response-time
  series, the up/down series and the averages drop degraded checks too.
- findPageSpeedDateRangeChecks: excludes as well; pagespeed is egress
  attributable and was previously missed.
- getDailyStatusBuckets: excludes at the match; the group shapes and the
  totalChecks > 0 guard revert to their original form.
- check.docker.aggregation.ts, check.hardware.aggregations.ts: reverted.
  Neither type is egress attributable, so their checks never carry the flag
  and the filters were dead code. A test now pins that decision.
- Tests: the gap behaviour, a window that was entirely degraded, pagespeed,
  the team listing and the resolve filter, and an incident that stays open
  when egress drops during it.

🔧
This commit is contained in:
Craig Lawson committed 2026-09-17 22:06:16 +01:00
1 parent 4335e6b7b6
commit d865103fb5
7 files changed
+99 -66
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/domain/checks/check.docker.aggregation.ts
+2 -5
View File
@@ -1,6 +1,5 @@
import CheckModel from "@/domain/checks/check.model.js";
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
type DateRange = { start: Date; end: Date };
@@ -10,7 +9,6 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "docker",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
@@ -36,9 +34,8 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
$group: {
_id: { $dateToString: { format: dateString, date: "$createdAt" } },
avgResponseTime: { $avg: "$responseTime" },
// Counts exclude degraded-egress checks like aggregateData.totalChecks does; the response-time average keeps them.
upCount: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
totalCount: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
upCount: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
totalCount: { $sum: 1 },
avgRunning: { $avg: "$containerSummary.running" },
avgTotal: { $avg: "$containerSummary.total" },
avgUnhealthy: { $avg: "$containerSummary.unhealthy" },
server/src/domain/checks/check.hardware.aggregations.ts
-2
View File
@@ -1,7 +1,6 @@
import { CheckModel } from "@/domain/checks/check.model.js";
import mongoose from "mongoose";
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
type DateRange = { start: Date; end: Date };
type HardwareUpChecks = { totalChecks: number };
@@ -71,7 +70,6 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.type": "hardware",
createdAt: { $gte: dates.start, $lte: dates.end },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
});
};
server/src/domain/checks/check.query.ts
+4 -5
View File
@@ -1,8 +1,7 @@
import type { EgressStatus } from "@/domain/egress/egress.type.js";
// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they
// are left out of every uptime percentage and up/down count. Response-time series keep them.
// Spread this into the $match / filter of a query, or use the expression form inside a $group accumulator.
// A check recorded while the instance's own egress was down says nothing about the target, so it is left out of
// every aggregate: uptime percentages, up/down counts and the response-time series alike. A degraded episode
// therefore reads as a gap in monitoring rather than as downtime or as a dip in response time.
// Spread this into the $match or filter of any query that reports statistics. Check listings keep degraded checks.
export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
export const IS_NOT_DEGRADED_EGRESS_EXPR = { $ne: ["$egressStatus", "degraded" satisfies EgressStatus] } as const;
server/src/domain/checks/check.repository.mongo.ts
+6 -8
View File
@@ -16,7 +16,7 @@ import type {
} from "@/domain/checks/check.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js";
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
import mongoose from "mongoose";
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
import { ILogger } from "@/utils/logger.js";
@@ -396,6 +396,7 @@ class MongoChecksRepository implements IChecksRepository {
$match: {
"metadata.monitorId": { $in: objectIds },
createdAt: { $gte: windowStart },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
},
},
{
@@ -404,14 +405,11 @@ class MongoChecksRepository implements IChecksRepository {
monitorId: "$metadata.monitorId",
day: { $dateTrunc: { date: "$createdAt", unit: "day", timezone } },
},
// Counts exclude degraded-egress checks; the response-time average keeps them.
totalChecks: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } },
upChecks: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } },
totalChecks: { $sum: 1 },
upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
avgResponseTime: { $avg: "$responseTime" },
},
},
// A day with nothing attributable to the target renders as an empty bar; the client divides by totalChecks for the tooltip.
{ $match: { totalChecks: { $gt: 0 } } },
{ $sort: { "_id.day": 1 } },
{
$project: {
@@ -479,6 +477,7 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [result] = await CheckModel.aggregate([
{ $match: matchStage },
@@ -486,8 +485,6 @@ class MongoChecksRepository implements IChecksRepository {
{
$facet: {
uptimePercentage: [
// Response-time series (groupedUpChecks / groupedDownChecks) deliberately keep degraded checks so they still show as failures on the graph.
{ $match: EXCLUDE_DEGRADED_EGRESS_MATCH },
{
$group: {
_id: null,
@@ -646,6 +643,7 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
...EXCLUDE_DEGRADED_EGRESS_MATCH,
};
const [result] = await CheckModel.aggregate([
server/src/domain/monitors/monitor.type.ts
+3 -2
View File
@@ -50,8 +50,9 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
// Types whose transport failure can be the instance's own loss of egress. Hardware (Capture agent) and docker
// (local socket or host) failures are local, so they are never attributed to egress.
// Types whose transport failure can be the instance's own loss of egress. Hardware and docker are left out:
// a Capture agent or a docker socket is normally reached locally, and a docker monitor pointed at a remote
// host over TCP is deliberately out of scope rather than assumed local.
export const EgressAttributableTypes = [
"http",
"ping",
server/test/integration/checksRepositoryEgressExclusion.test.ts
+60 -44
View File
@@ -3,14 +3,15 @@ import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
import { CheckModel } from "../../src/domain/checks/check.model.ts";
import type { Check, DockerChecksResult, HardwareChecksResult, UptimeChecksResult } from "../../src/domain/checks/check.type.ts";
import type { Check, DockerChecksResult, PageSpeedChecksResult, UptimeChecksResult } from "../../src/domain/checks/check.type.ts";
import { NETWORK_ERROR } from "../../src/types/network.ts";
import type { ILogger } from "../../src/utils/logger.ts";
import { createMockLogger } from "../helpers/createMockLogger.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The guarantee under test — a check flagged egressStatus "degraded" is left out of
// every uptime percentage and down-count while still appearing in listings and in
// the response-time series — lives in $match stages spread into several aggregation
// every aggregate, so a degraded episode reads as a gap in monitoring, while the check
// still appears in listings — lives in $match stages spread into several aggregation
// pipelines, so it can only be exercised against a live mongod.
let mongod: MongoMemoryServer;
@@ -85,7 +86,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
await expect(seedCheck({ status: false, egressStatus: "unknown" })).rejects.toThrow(/egressStatus/);
});
it("excludes degraded checks from the uptime percentage but keeps them in the response-time series", async () => {
it("excludes degraded checks from the uptime percentage and from the response-time series alike", async () => {
await seedCheck(); // up
await seedCheck({ status: false }); // real failure
await seedDegradedFailure();
@@ -95,13 +96,33 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
// 1 up of 2 attributable checks. Counting the degraded pair would give 25%.
expect(result.uptimePercentage).toBe(0.5);
// The graph still shows the degraded checks as failures, and their response times stay in the averages.
expect(result.avgResponseTime).toBe(200);
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4, avgResponseTime: 200 });
expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 3 });
// Every series sees the same two checks, so the episode is a gap rather than a dip.
expect(result.avgResponseTime).toBe(100);
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 2, avgResponseTime: 100 });
expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 1 });
expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 });
});
it("leaves a bucket out of the response-time series entirely when every check in it was degraded", async () => {
await seedDegradedFailure();
await seedDegradedFailure();
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
expect(result.groupedChecks).toEqual([]);
expect(result.groupedDownChecks).toEqual([]);
expect(result.uptimePercentage).toBe(0);
});
it("excludes degraded checks from the pagespeed series", async () => {
await seedCheck({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.9 });
await seedDegradedFailure({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.1 });
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "pagespeed" })) as PageSpeedChecksResult;
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 1, performance: 0.9 });
});
it("does not flag a failure that carries egressStatus ok", async () => {
await seedCheck();
await seedCheck({ status: false, egressStatus: "ok" });
@@ -121,7 +142,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
expect(summary).toEqual({ totalChecks: 2, downChecks: 1 });
});
it("excludes degraded checks from the daily status bucket counts but keeps them in its response-time average", async () => {
it("excludes degraded checks from the daily status buckets, dropping a day that saw nothing else", async () => {
const otherMonitor = new mongoose.Types.ObjectId();
await seedCheck();
await seedCheck({ status: false });
@@ -130,46 +151,12 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC");
// (100 + 100 + 300) / 3 = 167: the degraded check's response time counts, its failure does not.
// The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket.
expect(buckets).toEqual([
{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 167 },
{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 },
]);
});
it("excludes degraded checks from the hardware and docker totals", async () => {
const hardwareMonitor = new mongoose.Types.ObjectId();
const dockerMonitor = new mongoose.Types.ObjectId();
const hardwareMeta = { monitorId: hardwareMonitor, teamId: TEAM_ID, type: "hardware" };
const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
await seedCheck({ metadata: hardwareMeta });
await seedDegradedFailure({ metadata: hardwareMeta });
await seedCheck({ metadata: dockerMeta });
await seedDegradedFailure({ metadata: dockerMeta });
const hardware = (await repo.findByDateRangeAndMonitorId(hardwareMonitor.toString(), "day", { type: "hardware" })) as HardwareChecksResult;
const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
expect(hardware.aggregateData.totalChecks).toBe(1);
expect(hardware.upChecks.totalChecks).toBe(1);
expect(docker.aggregateData.totalChecks).toBe(1);
expect(docker.upChecks.totalChecks).toBe(1);
});
it("excludes degraded checks from the docker bucket counts but keeps them in its response-time average", async () => {
const dockerMonitor = new mongoose.Types.ObjectId();
const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
await seedCheck({ metadata: dockerMeta });
await seedDegradedFailure({ metadata: dockerMeta });
const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
expect(docker.aggregate).toHaveLength(1);
expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1, avgResponseTime: 200 });
});
it("keeps degraded checks in the paginated listing so they can be shown as such", async () => {
await seedCheck({ status: false });
await seedDegradedFailure();
@@ -179,4 +166,33 @@ describe("MongoChecksRepository degraded-egress exclusion", () => {
expect(checksCount).toBe(2);
expect(checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
});
it("keeps degraded checks in the team listing and under the resolve filter", async () => {
await seedCheck({ status: false, statusCode: NETWORK_ERROR });
await seedDegradedFailure({ statusCode: NETWORK_ERROR });
// The Checks page reads this listing when no single monitor is selected, and a degraded check has the
// same shape the resolve filter looks for, so neither may quietly drop it.
const team = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "down");
const resolve = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "resolve");
expect(team.checksCount).toBe(2);
expect(resolve.checksCount).toBe(2);
expect(resolve.checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
});
it("leaves the docker aggregates alone, since a docker check is never attributed to egress", async () => {
const dockerMonitor = new mongoose.Types.ObjectId();
const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
// The schema accepts the field on any type, so this pins the scope decision rather than the schema:
// the producer never sets it for docker or hardware, so their aggregations must not filter on it.
await seedCheck({ metadata: dockerMeta });
await seedDegradedFailure({ metadata: dockerMeta });
const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
expect(docker.aggregateData.totalChecks).toBe(2);
expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 2 });
});
});
server/test/integration/heartbeatJobDegradedEgress.test.ts
+24
View File
@@ -134,6 +134,30 @@ describe("Heartbeat job: degraded egress", () => {
expect(h.incidentsRepo.getAll()).toHaveLength(1);
});
it("leaves an open incident open when egress drops during it", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);
// The target goes down first and is detected normally
h.setEgressStatus(null);
h.setNextResponse(false, NETWORK_ERROR);
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
// Egress then drops. The monitor must stay down rather than be resolved by the silence.
h.setEgressStatus("degraded");
for (let i = 0; i < 3; i++) {
await h.heartbeatJob(monitor);
}
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
expect(h.incidentsRepo.getAll()).toHaveLength(1);
expect(h.notificationsService.handleNotifications).toHaveBeenCalledTimes(1);
});
it("keeps counting real failures taken either side of a degraded spell", async () => {
const monitor = makeMonitor();
h.monitorsRepo.seed(monitor);