diff --git a/server/src/domain/checks/check.docker.aggregation.ts b/server/src/domain/checks/check.docker.aggregation.ts index d2db06462..de6870e04 100644 --- a/server/src/domain/checks/check.docker.aggregation.ts +++ b/server/src/domain/checks/check.docker.aggregation.ts @@ -1,5 +1,6 @@ -import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js"; +import CheckModel from "@/domain/checks/check.model.js"; import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js"; +import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js"; import mongoose from "mongoose"; type DateRange = { start: Date; end: Date }; @@ -35,8 +36,9 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt $group: { _id: { $dateToString: { format: dateString, date: "$createdAt" } }, avgResponseTime: { $avg: "$responseTime" }, - upCount: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } }, - totalCount: { $sum: 1 }, + // Counts exclude degraded-egress checks like aggregateData.totalChecks does; the response-time average keeps them. + upCount: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } }, + totalCount: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } }, avgRunning: { $avg: "$containerSummary.running" }, avgTotal: { $avg: "$containerSummary.total" }, avgUnhealthy: { $avg: "$containerSummary.unhealthy" }, diff --git a/server/src/domain/checks/check.hardware.aggregations.ts b/server/src/domain/checks/check.hardware.aggregations.ts index 495ee5278..9a67a8d0a 100644 --- a/server/src/domain/checks/check.hardware.aggregations.ts +++ b/server/src/domain/checks/check.hardware.aggregations.ts @@ -1,6 +1,7 @@ -import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js"; +import { CheckModel } from "@/domain/checks/check.model.js"; import mongoose from "mongoose"; import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js"; +import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js"; type DateRange = { start: Date; end: Date }; type HardwareUpChecks = { totalChecks: number }; diff --git a/server/src/domain/checks/check.model.ts b/server/src/domain/checks/check.model.ts index 094ced6b0..1f92fce3c 100644 --- a/server/src/domain/checks/check.model.ts +++ b/server/src/domain/checks/check.model.ts @@ -1,6 +1,6 @@ import { Schema, model, Types } from "mongoose"; import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js"; -import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js"; +import { EgressStatuses } from "@/domain/egress/egress.type.js"; import type { Check, CheckAudits, @@ -365,10 +365,6 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 }); const CheckModel = model("Check", CheckSchema); -// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they -// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query. -const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const; - export type { CheckDocument, CheckMetadataDocument }; -export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH }; +export { CheckModel, containerSummarySchema }; export default CheckModel; diff --git a/server/src/domain/checks/check.query.ts b/server/src/domain/checks/check.query.ts new file mode 100644 index 000000000..2ebaf59d3 --- /dev/null +++ b/server/src/domain/checks/check.query.ts @@ -0,0 +1,8 @@ +import type { EgressStatus } from "@/domain/egress/egress.type.js"; + +// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they +// are left out of every uptime percentage and up/down count. Response-time series keep them. +// Spread this into the $match / filter of a query, or use the expression form inside a $group accumulator. +export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const; + +export const IS_NOT_DEGRADED_EGRESS_EXPR = { $ne: ["$egressStatus", "degraded" satisfies EgressStatus] } as const; diff --git a/server/src/domain/checks/check.repository.mongo.ts b/server/src/domain/checks/check.repository.mongo.ts index e9ad42ad2..1a9707aa5 100644 --- a/server/src/domain/checks/check.repository.mongo.ts +++ b/server/src/domain/checks/check.repository.mongo.ts @@ -15,7 +15,8 @@ import type { HardwareCheckStats, } from "@/domain/checks/check.type.js"; import type { MonitorType } from "@/domain/monitors/monitor.type.js"; -import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js"; +import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js"; +import { EXCLUDE_DEGRADED_EGRESS_MATCH, IS_NOT_DEGRADED_EGRESS_EXPR } from "@/domain/checks/check.query.js"; import mongoose from "mongoose"; import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js"; import { ILogger } from "@/utils/logger.js"; @@ -179,7 +180,7 @@ class MongoChecksRepository implements IChecksRepository { audits: mapAudits(doc.audits), containers: doc.containers, containerSummary: doc.containerSummary, - ...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }), + egressStatus: doc.egressStatus, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; @@ -395,7 +396,6 @@ class MongoChecksRepository implements IChecksRepository { $match: { "metadata.monitorId": { $in: objectIds }, createdAt: { $gte: windowStart }, - ...EXCLUDE_DEGRADED_EGRESS_MATCH, }, }, { @@ -404,11 +404,14 @@ class MongoChecksRepository implements IChecksRepository { monitorId: "$metadata.monitorId", day: { $dateTrunc: { date: "$createdAt", unit: "day", timezone } }, }, - totalChecks: { $sum: 1 }, - upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } }, + // Counts exclude degraded-egress checks; the response-time average keeps them. + totalChecks: { $sum: { $cond: [IS_NOT_DEGRADED_EGRESS_EXPR, 1, 0] } }, + upChecks: { $sum: { $cond: [{ $and: [{ $eq: ["$status", true] }, IS_NOT_DEGRADED_EGRESS_EXPR] }, 1, 0] } }, avgResponseTime: { $avg: "$responseTime" }, }, }, + // A day with nothing attributable to the target renders as an empty bar; the client divides by totalChecks for the tooltip. + { $match: { totalChecks: { $gt: 0 } } }, { $sort: { "_id.day": 1 } }, { $project: { diff --git a/server/src/domain/checks/check.type.ts b/server/src/domain/checks/check.type.ts index ab304e007..f63cf5c83 100644 --- a/server/src/domain/checks/check.type.ts +++ b/server/src/domain/checks/check.type.ts @@ -1,7 +1,7 @@ import type { MonitorType } from "@/domain/monitors/monitor.type.js"; +import type { EgressStatus } from "@/domain/egress/egress.type.js"; import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js"; import type { Response } from "got"; -import type { EgressStatus } from "@/domain/egress/egress.type.js"; export const CHECK_TTL_SENTINEL = 366; @@ -117,8 +117,9 @@ export interface Check { seo?: number; performance?: number; audits?: CheckAudits; - // Set only on checks produced while the reliability probe ran (a failing check). - // "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target. + // Set on failing checks while the egress check is enabled; absent otherwise. + // "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target. + // "ok": a reliability target was reachable. egressStatus?: EgressStatus; createdAt: string; updatedAt: string; diff --git a/server/src/domain/egress/egress.service.ts b/server/src/domain/egress/egress.service.ts index f336bab12..1088356ff 100644 --- a/server/src/domain/egress/egress.service.ts +++ b/server/src/domain/egress/egress.service.ts @@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const; const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null); // HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code. -const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599; +export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599; // Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs. const PROBE_MONITOR_ID = "egress-probe"; diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index df8c52d9e..f2029bbb1 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -50,6 +50,21 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number]; export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type); +// Types whose transport failure can be the instance's own loss of egress. Hardware (Capture agent) and docker +// (local socket or host) failures are local, so they are never attributed to egress. +export const EgressAttributableTypes = [ + "http", + "ping", + "pagespeed", + "port", + "game", + "grpc", + "websocket", + "dns", +] as const satisfies readonly MonitorType[]; +export type EgressAttributableType = (typeof EgressAttributableTypes)[number]; +export const isEgressAttributable = (type: MonitorType): type is EgressAttributableType => EgressAttributableTypes.some((t) => t === type); + export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const; export type MonitorStatus = (typeof MonitorStatuses)[number]; diff --git a/server/src/worker/worker.check-producer.ts b/server/src/worker/worker.check-producer.ts index 194718a20..3f09e998d 100644 --- a/server/src/worker/worker.check-producer.ts +++ b/server/src/worker/worker.check-producer.ts @@ -11,7 +11,9 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main import { isWindowActive } from "@/utils/maintenanceWindow.js"; import { IProxyResolver } from "@/service/network/ProxyResolver.js"; import { IDockerLogsService } from "@/domain/docker/docker-log.service.js"; -import { IEgressService } from "@/domain/egress/egress.service.js"; +import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js"; +import type { EgressStatus } from "@/domain/egress/egress.type.js"; +import { isEgressAttributable } from "@/domain/monitors/monitor.type.js"; export interface ICheckProducer { produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>; @@ -44,6 +46,26 @@ export class CheckProducer implements ICheckProducer { return dockerTlsKey ?? undefined; } + // Only a failure to reach the target at all can be the instance's own fault. Any HTTP response (4xx, 5xx, + // or a 200 with a content mismatch) proves the target was reached, and hardware/docker failures are local. + private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean { + return status.status === false && isEgressAttributable(monitor.type) && !isHttpStatusCode(status.code); + } + + // The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown. + private async assessEgress(monitorId: string): Promise { + try { + return await this.egressService.assessAfterFailure(); + } catch (error: unknown) { + this.logger.warn({ + message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`, + service: SERVICE_NAME, + method: "assessEgress", + }); + return null; + } + } + produce = async (monitor: Monitor) => { if (!monitor.id) { throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" }); @@ -76,9 +98,9 @@ export class CheckProducer implements ICheckProducer { throw new Error("No network response"); } - // Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target. + // Step 1c: On a transport failure, ask whether the instance itself can reach anything before blaming the target. // Null means the egress check is disabled (or failed internally) and the check is treated as usual. - const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null; + const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null; // **************************** // Step 2: Record diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index 9652b021e..47370df0b 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -14,6 +14,7 @@ import type { Monitor } from "../../src/domain/monitors/monitor.type.ts"; import type { MonitorStatusResponse } from "../../src/types/network.ts"; import type { Check } from "../../src/domain/checks/check.type.ts"; import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts"; +import type { EgressStatus } from "../../src/domain/egress/egress.type.ts"; let checkCounter = 0; @@ -79,9 +80,11 @@ export interface HeartbeatTestHarness { bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock }; maintenanceWindowsRepo: { findByMonitorId: jest.Mock }; messageBuilder: { extractThresholdBreaches: jest.Mock }; + egressService: { assessAfterFailure: jest.Mock }; heartbeatJob: (monitor: Monitor) => Promise; setNextResponse: (status: boolean, code: number) => void; setNextResponseFull: (response: MonitorStatusResponse) => void; + setEgressStatus: (status: EgressStatus | null) => void; } export function createHeartbeatTestHarness(): HeartbeatTestHarness { @@ -129,7 +132,12 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) }; const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) }; const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) }; - const egressService = { assessAfterFailure: jest.fn().mockResolvedValue(null) }; + // Null mirrors the egress check being disabled, so the existing heartbeat suites run unchanged. + let nextEgressStatus: EgressStatus | null = null; + const egressService = { assessAfterFailure: jest.fn().mockImplementation(() => Promise.resolve(nextEgressStatus)) }; + const setEgressStatus = (status: EgressStatus | null) => { + nextEgressStatus = status; + }; const checkProducer = new CheckProducer( monitorsRepo as any, @@ -169,8 +177,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { bufferStub, maintenanceWindowsRepo, messageBuilder, + egressService, heartbeatJob, setNextResponse, setNextResponseFull, + setEgressStatus, }; } diff --git a/server/test/integration/checksRepositoryEgressExclusion.test.ts b/server/test/integration/checksRepositoryEgressExclusion.test.ts index b2f488b66..eea421019 100644 --- a/server/test/integration/checksRepositoryEgressExclusion.test.ts +++ b/server/test/integration/checksRepositoryEgressExclusion.test.ts @@ -78,7 +78,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { // The evaluator runs from this read, so the flag must survive persistence for the short-circuit to fire. expect(checks.map((check) => check.egressStatus)).toEqual(["degraded", "ok", undefined]); - expect(checks[2]).not.toHaveProperty("egressStatus"); + expect(checks[2].egressStatus).toBeUndefined(); }); it("rejects a value outside the EgressStatuses tuple", async () => { @@ -95,8 +95,9 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { // 1 up of 2 attributable checks. Counting the degraded pair would give 25%. expect(result.uptimePercentage).toBe(0.5); - // The graph still shows the degraded checks as failures. - expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4 }); + // The graph still shows the degraded checks as failures, and their response times stay in the averages. + expect(result.avgResponseTime).toBe(200); + expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 4, avgResponseTime: 200 }); expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 3 }); expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 }); }); @@ -120,7 +121,7 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { expect(summary).toEqual({ totalChecks: 2, downChecks: 1 }); }); - it("excludes degraded checks from the daily status buckets and yields no row for a day of only degraded checks", async () => { + it("excludes degraded checks from the daily status bucket counts but keeps them in its response-time average", async () => { const otherMonitor = new mongoose.Types.ObjectId(); await seedCheck(); await seedCheck({ status: false }); @@ -129,8 +130,10 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC"); + // (100 + 100 + 300) / 3 = 167: the degraded check's response time counts, its failure does not. + // The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket. expect(buckets).toEqual([ - { monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 }, + { monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 167 }, ]); }); @@ -154,6 +157,19 @@ describe("MongoChecksRepository degraded-egress exclusion", () => { expect(docker.upChecks.totalChecks).toBe(1); }); + it("excludes degraded checks from the docker bucket counts but keeps them in its response-time average", async () => { + const dockerMonitor = new mongoose.Types.ObjectId(); + const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" }; + + await seedCheck({ metadata: dockerMeta }); + await seedDegradedFailure({ metadata: dockerMeta }); + + const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult; + + expect(docker.aggregate).toHaveLength(1); + expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1, avgResponseTime: 200 }); + }); + it("keeps degraded checks in the paginated listing so they can be shown as such", async () => { await seedCheck({ status: false }); await seedDegradedFailure(); diff --git a/server/test/integration/heartbeatJobDegradedEgress.test.ts b/server/test/integration/heartbeatJobDegradedEgress.test.ts new file mode 100644 index 000000000..327adaf0e --- /dev/null +++ b/server/test/integration/heartbeatJobDegradedEgress.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it, beforeEach } from "@jest/globals"; +import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts"; +import { NETWORK_ERROR } from "../../src/types/network.ts"; + +// Egress is only consulted for transport failures, so these heartbeats fail with the network-error +// sentinel (timeout / connection refused) rather than an HTTP status code. + +describe("Heartbeat job: degraded egress", () => { + let h: HeartbeatTestHarness; + + beforeEach(() => { + h = createHeartbeatTestHarness(); + }); + + it("records failing checks as degraded without changing monitor status, opening an incident or notifying", async () => { + const monitor = makeMonitor(); + h.monitorsRepo.seed(monitor); + + // Enough failures to cross the 60% threshold if they were counted + h.setEgressStatus("degraded"); + h.setNextResponse(false, NETWORK_ERROR); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + + // Every check was buffered, flagged, so the repository can exclude it from uptime + expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3); + expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(3); + for (const [check] of h.bufferStub.addToBuffer.mock.calls) { + expect(check).toMatchObject({ status: false, egressStatus: "degraded" }); + } + + // Nothing downstream moved + const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1"); + expect(storedMonitor.status).toBe("up"); + expect(storedMonitor.statusWindow).toEqual([true, true, true, true, true]); + expect(h.incidentsRepo.getAll()).toHaveLength(0); + expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled(); + }); + + it("does not fire a spurious recovery once egress returns", async () => { + const monitor = makeMonitor(); + h.monitorsRepo.seed(monitor); + + h.setEgressStatus("degraded"); + h.setNextResponse(false, NETWORK_ERROR); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + + // Egress is back and the target is fine + h.setEgressStatus(null); + h.setNextResponse(true, 200); + await h.heartbeatJob(monitor); + + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up"); + expect(h.incidentsRepo.getAll()).toHaveLength(0); + expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled(); + }); + + it("detects down at the normal threshold when the egress check is disabled", async () => { + const monitor = makeMonitor(); + h.monitorsRepo.seed(monitor); + + h.setEgressStatus(null); + h.setNextResponse(false, NETWORK_ERROR); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + + expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3); + for (const [check] of h.bufferStub.addToBuffer.mock.calls) { + expect(check).not.toHaveProperty("egressStatus"); + } + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down"); + expect(h.incidentsRepo.getAll()).toHaveLength(1); + expect(h.notificationsService.handleNotifications).toHaveBeenCalled(); + }); + + it("detects down at the normal threshold when egress is confirmed ok", async () => { + const monitor = makeMonitor(); + h.monitorsRepo.seed(monitor); + + h.setEgressStatus("ok"); + h.setNextResponse(false, NETWORK_ERROR); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + + for (const [check] of h.bufferStub.addToBuffer.mock.calls) { + expect(check).toMatchObject({ status: false, egressStatus: "ok" }); + } + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down"); + expect(h.incidentsRepo.getAll()).toHaveLength(1); + expect(h.notificationsService.handleNotifications).toHaveBeenCalled(); + }); + + it("evaluates HTTP error responses normally while egress is degraded", async () => { + const monitor = makeMonitor(); + h.monitorsRepo.seed(monitor); + + // The target answered, so the failure is its own even though the instance is degraded + h.setEgressStatus("degraded"); + h.setNextResponse(false, 503); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + + expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled(); + for (const [check] of h.bufferStub.addToBuffer.mock.calls) { + expect(check).not.toHaveProperty("egressStatus"); + } + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down"); + expect(h.incidentsRepo.getAll()).toHaveLength(1); + expect(h.incidentsRepo.getAll()[0].statusCode).toBe(503); + expect(h.notificationsService.handleNotifications).toHaveBeenCalled(); + }); + + it("evaluates a failing docker check normally while egress is degraded", async () => { + const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" }); + h.monitorsRepo.seed(monitor); + + h.setEgressStatus("degraded"); + h.setNextResponse(false, NETWORK_ERROR); + for (let i = 0; i < 3; i++) { + await h.heartbeatJob(monitor); + } + + expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled(); + for (const [check] of h.bufferStub.addToBuffer.mock.calls) { + expect(check).not.toHaveProperty("egressStatus"); + } + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down"); + expect(h.incidentsRepo.getAll()).toHaveLength(1); + }); + + it("keeps counting real failures taken either side of a degraded spell", async () => { + const monitor = makeMonitor(); + h.monitorsRepo.seed(monitor); + + // 2 real failures: window [t, t, t, f, f] = 40% < 60% + h.setNextResponse(false, NETWORK_ERROR); + await h.heartbeatJob(monitor); + await h.heartbeatJob(monitor); + + // A degraded spell must not push the window over the threshold + h.setEgressStatus("degraded"); + await h.heartbeatJob(monitor); + await h.heartbeatJob(monitor); + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up"); + expect(h.incidentsRepo.getAll()).toHaveLength(0); + + // 3rd real failure: window [t, t, f, f, f] = 60% >= 60% + h.setEgressStatus(null); + await h.heartbeatJob(monitor); + + expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down"); + expect(h.incidentsRepo.getAll()).toHaveLength(1); + }); +}); diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts index 478262c20..bb2a0b536 100644 --- a/server/test/unit/services/checkProducer.test.ts +++ b/server/test/unit/services/checkProducer.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals"; import { CheckProducer } from "../../../src/worker/worker.check-producer.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; +import { NETWORK_ERROR } from "../../../src/types/network.ts"; const makeMonitor = (overrides?: Partial): Monitor => ({ @@ -215,7 +216,8 @@ describe("CheckProducer", () => { // ── egress self-check ───────────────────────────────────────────────────── - const failingStatus = { monitorId: "m1", status: false, code: 500, message: "Error" }; + // A transport failure: nothing came back from the target, so the code is the network-error sentinel. + const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" }; it("does not consult the egress service for a successful check and leaves the field unset", async () => { const check: Record = { id: "check-1" }; @@ -229,6 +231,57 @@ describe("CheckProducer", () => { expect(check).not.toHaveProperty("egressStatus"); }); + it("does not consult egress when an HTTP response was received", async () => { + for (const status of [ + { monitorId: "m1", status: false, code: 500, message: "Internal Server Error" }, + { monitorId: "m1", status: false, code: 200, message: "Content mismatch" }, + ]) { + const check: Record = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await producer.produce(makeMonitor()); + + expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled(); + expect(check).not.toHaveProperty("egressStatus"); + expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); + } + }); + + it("consults egress for a timeout or network error", async () => { + const check: Record = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await producer.produce(makeMonitor()); + + expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check.egressStatus).toBe("degraded"); + }); + + it("never consults egress for a failing docker or hardware check", async () => { + for (const type of ["docker", "hardware"] as const) { + const check: Record = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type }) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + await producer.produce(makeMonitor({ type })); + + expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled(); + expect(check).not.toHaveProperty("egressStatus"); + expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); + } + }); + it("flags a failing check as degraded when the egress service reports degraded egress", async () => { const check: Record = { id: "check-1" }; const { producer, defaults } = createProducer({ @@ -258,6 +311,24 @@ describe("CheckProducer", () => { expect(check.egressStatus).toBe("ok"); }); + it("buffers the check when the egress assessment rejects", async () => { + const check: Record = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) }, + }); + + const result = await producer.produce(makeMonitor()); + + expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); + expect(check).not.toHaveProperty("egressStatus"); + expect(result).toEqual({ status: failingStatus, check }); + expect(defaults.logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" }) + ); + }); + it("leaves the field unset on a failing check when the egress check is disabled", async () => { const check: Record = { id: "check-1" }; const { producer, defaults } = createProducer({