diff --git a/server/src/domain/checks/check.docker.aggregation.ts b/server/src/domain/checks/check.docker.aggregation.ts index de6870e04..37d6d8e87 100644 --- a/server/src/domain/checks/check.docker.aggregation.ts +++ b/server/src/domain/checks/check.docker.aggregation.ts @@ -1,6 +1,5 @@ import CheckModel from "@/domain/checks/check.model.js"; import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js"; -import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js"; import mongoose from "mongoose"; type DateRange = { start: Date; end: Date }; @@ -10,7 +9,6 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange): "metadata.monitorId": new mongoose.Types.ObjectId(monitorId), "metadata.type": "docker", createdAt: { $gte: dates.start, $lte: dates.end }, - ...EXCLUDE_DEGRADED_EGRESS_MATCH, }); export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => { @@ -36,9 +34,8 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt $group: { _id: { $dateToString: { format: dateString, date: "$createdAt" } }, avgResponseTime: { $avg: "$responseTime" }, - // Counts exclude degraded-egress checks like aggregateData.totalChecks does; the response-time average keeps them. - upCount: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } }, - totalCount: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } }, + upCount: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } }, + totalCount: { $sum: 1 }, avgRunning: { $avg: "$containerSummary.running" }, avgTotal: { $avg: "$containerSummary.total" }, avgUnhealthy: { $avg: "$containerSummary.unhealthy" }, diff --git a/server/src/domain/checks/check.hardware.aggregations.ts b/server/src/domain/checks/check.hardware.aggregations.ts index 9a67a8d0a..6c672b630 100644 --- a/server/src/domain/checks/check.hardware.aggregations.ts +++ b/server/src/domain/checks/check.hardware.aggregations.ts @@ -1,7 +1,6 @@ import { CheckModel } from "@/domain/checks/check.model.js"; import mongoose from "mongoose"; import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js"; -import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js"; type DateRange = { start: Date; end: Date }; type HardwareUpChecks = { totalChecks: number }; @@ -71,7 +70,6 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange "metadata.monitorId": new mongoose.Types.ObjectId(monitorId), "metadata.type": "hardware", createdAt: { $gte: dates.start, $lte: dates.end }, - ...EXCLUDE_DEGRADED_EGRESS_MATCH, }); }; diff --git a/server/src/domain/checks/check.query.ts b/server/src/domain/checks/check.query.ts index 2ebaf59d3..8b07a30cf 100644 --- a/server/src/domain/checks/check.query.ts +++ b/server/src/domain/checks/check.query.ts @@ -1,8 +1,7 @@ import type { EgressStatus } from "@/domain/egress/egress.type.js"; -// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they -// are left out of every uptime percentage and up/down count. Response-time series keep them. -// Spread this into the $match / filter of a query, or use the expression form inside a $group accumulator. +// A check recorded while the instance's own egress was down says nothing about the target, so it is left out of +// every aggregate: uptime percentages, up/down counts and the response-time series alike. A degraded episode +// therefore reads as a gap in monitoring rather than as downtime or as a dip in response time. +// Spread this into the $match or filter of any query that reports statistics. Check listings keep degraded checks. export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const; - -export const IS_NOT_DEGRADED_EGRESS_EXPR = { $ne: ["$egressStatus", "degraded" satisfies EgressStatus] } as const; diff --git a/server/src/domain/checks/check.repository.mongo.ts b/server/src/domain/checks/check.repository.mongo.ts index 1a9707aa5..6a1648ce1 100644 --- a/server/src/domain/checks/check.repository.mongo.ts +++ b/server/src/domain/checks/check.repository.mongo.ts @@ -16,7 +16,7 @@ import type { } from "@/domain/checks/check.type.js"; import type { MonitorType } from "@/domain/monitors/monitor.type.js"; import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js"; -import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js"; +import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js"; import mongoose from "mongoose"; import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js"; import { ILogger } from "@/utils/logger.js"; @@ -396,6 +396,7 @@ class MongoChecksRepository implements IChecksRepository { $match: { "metadata.monitorId": { $in: objectIds }, createdAt: { $gte: windowStart }, + ...EXCLUDE_DEGRADED_EGRESS_MATCH, }, }, { @@ -404,14 +405,11 @@ class MongoChecksRepository implements IChecksRepository { monitorId: "$metadata.monitorId", day: { $dateTrunc: { date: "$createdAt", unit: "day", timezone } }, }, - // Counts exclude degraded-egress checks; the response-time average keeps them. - totalChecks: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } }, - upChecks: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } }, + totalChecks: { $sum: 1 }, + upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } }, avgResponseTime: { $avg: "$responseTime" }, }, }, - // A day with nothing attributable to the target renders as an empty bar; the client divides by totalChecks for the tooltip. - { $match: { totalChecks: { $gt: 0 } } }, { $sort: { "_id.day": 1 } }, { $project: { @@ -479,6 +477,7 @@ class MongoChecksRepository implements IChecksRepository { const matchStage = { "metadata.monitorId": monitorObjectId, createdAt: { $gte: startDate, $lte: endDate }, + ...EXCLUDE_DEGRADED_EGRESS_MATCH, }; const [result] = await CheckModel.aggregate([ { $match: matchStage }, @@ -486,8 +485,6 @@ class MongoChecksRepository implements IChecksRepository { { $facet: { uptimePercentage: [ - // Response-time series (groupedUpChecks / groupedDownChecks) deliberately keep degraded checks so they still show as failures on the graph. - { $match: EXCLUDE_DEGRADED_EGRESS_MATCH }, { $group: { _id: null, @@ -646,6 +643,7 @@ class MongoChecksRepository implements IChecksRepository { const matchStage = { "metadata.monitorId": monitorObjectId, createdAt: { $gte: startDate, $lte: endDate }, + ...EXCLUDE_DEGRADED_EGRESS_MATCH, }; const [result] = await CheckModel.aggregate([ diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index f2029bbb1..5f40d6d80 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -50,8 +50,9 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number]; export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type); -// Types whose transport failure can be the instance's own loss of egress. Hardware (Capture agent) and docker -// (local socket or host) failures are local, so they are never attributed to egress. +// Types whose transport failure can be the instance's own loss of egress. Hardware and docker are left out: +// a Capture agent or a docker socket is normally reached locally, and a docker monitor pointed at a remote +// host over TCP is deliberately out of scope rather than assumed local. export const EgressAttributableTypes = [ "http", "ping", diff --git a/server/test/integration/checksRepositoryEgressExclusion.test.ts b/server/test/integration/checksRepositoryEgressExclusion.test.ts index eea421019..0f0950928 100644 --- a/server/test/integration/checksRepositoryEgressExclusion.test.ts +++ b/server/test/integration/checksRepositoryEgressExclusion.test.ts @@ -3,14 +3,15 @@ import mongoose from "mongoose"; import { MongoMemoryServer } from "mongodb-memory-server"; import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts"; import { CheckModel } from "../../src/domain/checks/check.model.ts"; -import type { Check, DockerChecksResult, HardwareChecksResult, UptimeChecksResult } from "../../src/domain/checks/check.type.ts"; +import type { Check, DockerChecksResult, PageSpeedChecksResult, UptimeChecksResult } from "../../src/domain/checks/check.type.ts"; +import { NETWORK_ERROR } from "../../src/types/network.ts"; import type { ILogger } from "../../src/utils/logger.ts"; import { createMockLogger } from "../helpers/createMockLogger.ts"; // ── Real-Mongo harness ───────────────────────────────────────────────────────── // The guarantee under test — a check flagged egressStatus "degraded" is left out of -// every uptime percentage and down-count while still appearing in listings and in -// the response-time series — lives in $match stages spread into several aggregation +// every aggregate, so a degraded episode reads as a gap in monitoring, while the check +// still appears in listings — lives in $match stages spread into several aggregation // pipelines, so it can only be exercised against a live mongod. let mongod: MongoMemoryServer; @@ -85,7 +86,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { await expect(seedCheck({ status: false, egressStatus: "unknown" })).rejects.toThrow(/egressStatus/); }); - it("excludes degraded checks from the uptime percentage but keeps them in the response-time series", async () => { + it("excludes degraded checks from the uptime percentage and from the response-time series alike", async () => { await seedCheck(); // up await seedCheck({ status: false }); // real failure await seedDegradedFailure(); @@ -95,13 +96,33 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { // 1 up of 2 attributable checks. Counting the degraded pair would give 25%. expect(result.uptimePercentage).toBe(0.5); - // The graph still shows the degraded checks as failures, and their response times stay in the averages. - expect(result.avgResponseTime).toBe(200); - expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4, avgResponseTime: 200 }); - expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 3 }); + // Every series sees the same two checks, so the episode is a gap rather than a dip. + expect(result.avgResponseTime).toBe(100); + expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 2, avgResponseTime: 100 }); + expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 1 }); expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 }); }); + it("leaves a bucket out of the response-time series entirely when every check in it was degraded", async () => { + await seedDegradedFailure(); + await seedDegradedFailure(); + + const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult; + + expect(result.groupedChecks).toEqual([]); + expect(result.groupedDownChecks).toEqual([]); + expect(result.uptimePercentage).toBe(0); + }); + + it("excludes degraded checks from the pagespeed series", async () => { + await seedCheck({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.9 }); + await seedDegradedFailure({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.1 }); + + const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "pagespeed" })) as PageSpeedChecksResult; + + expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 1, performance: 0.9 }); + }); + it("does not flag a failure that carries egressStatus ok", async () => { await seedCheck(); await seedCheck({ status: false, egressStatus: "ok" }); @@ -121,7 +142,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { expect(summary).toEqual({ totalChecks: 2, downChecks: 1 }); }); - it("excludes degraded checks from the daily status bucket counts but keeps them in its response-time average", async () => { + it("excludes degraded checks from the daily status buckets, dropping a day that saw nothing else", async () => { const otherMonitor = new mongoose.Types.ObjectId(); await seedCheck(); await seedCheck({ status: false }); @@ -130,46 +151,12 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC"); - // (100 + 100 + 300) / 3 = 167: the degraded check's response time counts, its failure does not. // The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket. expect(buckets).toEqual([ - { monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 167 }, + { monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 }, ]); }); - it("excludes degraded checks from the hardware and docker totals", async () => { - const hardwareMonitor = new mongoose.Types.ObjectId(); - const dockerMonitor = new mongoose.Types.ObjectId(); - const hardwareMeta = { monitorId: hardwareMonitor, teamId: TEAM_ID, type: "hardware" }; - const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" }; - - await seedCheck({ metadata: hardwareMeta }); - await seedDegradedFailure({ metadata: hardwareMeta }); - await seedCheck({ metadata: dockerMeta }); - await seedDegradedFailure({ metadata: dockerMeta }); - - const hardware = (await repo.findByDateRangeAndMonitorId(hardwareMonitor.toString(), "day", { type: "hardware" })) as HardwareChecksResult; - const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult; - - expect(hardware.aggregateData.totalChecks).toBe(1); - expect(hardware.upChecks.totalChecks).toBe(1); - expect(docker.aggregateData.totalChecks).toBe(1); - expect(docker.upChecks.totalChecks).toBe(1); - }); - - it("excludes degraded checks from the docker bucket counts but keeps them in its response-time average", async () => { - const dockerMonitor = new mongoose.Types.ObjectId(); - const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" }; - - await seedCheck({ metadata: dockerMeta }); - await seedDegradedFailure({ metadata: dockerMeta }); - - const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult; - - expect(docker.aggregate).toHaveLength(1); - expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1, avgResponseTime: 200 }); - }); - it("keeps degraded checks in the paginated listing so they can be shown as such", async () => { await seedCheck({ status: false }); await seedDegradedFailure(); @@ -179,4 +166,33 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { expect(checksCount).toBe(2); expect(checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort()); }); + + it("keeps degraded checks in the team listing and under the resolve filter", async () => { + await seedCheck({ status: false, statusCode: NETWORK_ERROR }); + await seedDegradedFailure({ statusCode: NETWORK_ERROR }); + + // The Checks page reads this listing when no single monitor is selected, and a degraded check has the + // same shape the resolve filter looks for, so neither may quietly drop it. + const team = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "down"); + const resolve = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "resolve"); + + expect(team.checksCount).toBe(2); + expect(resolve.checksCount).toBe(2); + expect(resolve.checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort()); + }); + + it("leaves the docker aggregates alone, since a docker check is never attributed to egress", async () => { + const dockerMonitor = new mongoose.Types.ObjectId(); + const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" }; + + // The schema accepts the field on any type, so this pins the scope decision rather than the schema: + // the producer never sets it for docker or hardware, so their aggregations must not filter on it. + await seedCheck({ metadata: dockerMeta }); + await seedDegradedFailure({ metadata: dockerMeta }); + + const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult; + + expect(docker.aggregateData.totalChecks).toBe(2); + expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 2 }); + }); }); diff --git a/server/test/integration/heartbeatJobDegradedEgress.test.ts b/server/test/integration/heartbeatJobDegradedEgress.test.ts index 327adaf0e..df8378790 100644 --- a/server/test/integration/heartbeatJobDegradedEgress.test.ts +++ b/server/test/integration/heartbeatJobDegradedEgress.test.ts @@ -134,6 +134,30 @@ describe("Heartbeat job: degraded egress", () => { expect(h.incidentsRepo.getAll()).toHaveLength(1); }); + it("leaves an open incident open when egress drops during it", async () => { + const monitor = makeMonitor(); + h.monitorsRepo.seed(monitor); + + // The target goes down first and is detected normally + h.setEgressStatus(null); + h.setNextResponse(false, NETWORK_ERROR); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down"); + expect(h.incidentsRepo.getAll()).toHaveLength(1); + + // Egress then drops. The monitor must stay down rather than be resolved by the silence. + h.setEgressStatus("degraded"); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down"); + expect(h.incidentsRepo.getAll()).toHaveLength(1); + expect(h.notificationsService.handleNotifications).toHaveBeenCalledTimes(1); + }); + it("keeps counting real failures taken either side of a degraded spell", async () => { const monitor = makeMonitor(); h.monitorsRepo.seed(monitor);