diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index 802ad4b0c..88b5a18e2 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -129,9 +129,10 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig proxyResolver, bufferService, dockerLogsService, + egressService, logger ); - const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy); + const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy, logger); const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger); // *********************** diff --git a/server/src/domain/checks/check.docker.aggregation.ts b/server/src/domain/checks/check.docker.aggregation.ts index 37d6d8e87..d2db06462 100644 --- a/server/src/domain/checks/check.docker.aggregation.ts +++ b/server/src/domain/checks/check.docker.aggregation.ts @@ -1,4 +1,4 @@ -import CheckModel from "@/domain/checks/check.model.js"; +import CheckModel, { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js"; import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js"; import mongoose from "mongoose"; @@ -9,6 +9,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange): "metadata.monitorId": new mongoose.Types.ObjectId(monitorId), "metadata.type": "docker", createdAt: { $gte: dates.start, $lte: dates.end }, + ...EXCLUDE_DEGRADED_EGRESS_MATCH, }); export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => { diff --git a/server/src/domain/checks/check.hardware.aggregations.ts b/server/src/domain/checks/check.hardware.aggregations.ts index 6c672b630..495ee5278 100644 --- a/server/src/domain/checks/check.hardware.aggregations.ts +++ b/server/src/domain/checks/check.hardware.aggregations.ts @@ -1,4 +1,4 @@ -import { CheckModel } from "@/domain/checks/check.model.js"; +import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.model.js"; import mongoose from "mongoose"; import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js"; @@ -70,6 +70,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange "metadata.monitorId": new mongoose.Types.ObjectId(monitorId), "metadata.type": "hardware", createdAt: { $gte: dates.start, $lte: dates.end }, + ...EXCLUDE_DEGRADED_EGRESS_MATCH, }); }; diff --git a/server/src/domain/checks/check.model.ts b/server/src/domain/checks/check.model.ts index a4319056e..094ced6b0 100644 --- a/server/src/domain/checks/check.model.ts +++ b/server/src/domain/checks/check.model.ts @@ -1,5 +1,6 @@ import { Schema, model, Types } from "mongoose"; import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js"; +import { EgressStatuses, type EgressStatus } from "@/domain/egress/egress.type.js"; import type { Check, CheckAudits, @@ -337,6 +338,11 @@ const CheckSchema = new Schema( type: containerSummarySchema, default: undefined, }, + egressStatus: { + type: String, + enum: EgressStatuses, + default: undefined, + }, }, { timestamps: true, @@ -359,6 +365,10 @@ CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 }); const CheckModel = model("Check", CheckSchema); +// Checks recorded while the instance's own egress was down cannot be attributed to the target, so they +// are left out of every uptime percentage and down-count. Spread this into the $match / filter of any such query. +const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const; + export type { CheckDocument, CheckMetadataDocument }; -export { CheckModel, containerSummarySchema }; +export { CheckModel, containerSummarySchema, EXCLUDE_DEGRADED_EGRESS_MATCH }; export default CheckModel; diff --git a/server/src/domain/checks/check.repository.mongo.ts b/server/src/domain/checks/check.repository.mongo.ts index 82e89d0d9..e9ad42ad2 100644 --- a/server/src/domain/checks/check.repository.mongo.ts +++ b/server/src/domain/checks/check.repository.mongo.ts @@ -15,7 +15,7 @@ import type { HardwareCheckStats, } from "@/domain/checks/check.type.js"; import type { MonitorType } from "@/domain/monitors/monitor.type.js"; -import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js"; +import { CheckModel, EXCLUDE_DEGRADED_EGRESS_MATCH, type CheckDocument } from "@/domain/checks/check.model.js"; import mongoose from "mongoose"; import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js"; import { ILogger } from "@/utils/logger.js"; @@ -179,6 +179,7 @@ class MongoChecksRepository implements IChecksRepository { audits: mapAudits(doc.audits), containers: doc.containers, containerSummary: doc.containerSummary, + ...(doc.egressStatus !== undefined && { egressStatus: doc.egressStatus }), createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; @@ -348,6 +349,7 @@ class MongoChecksRepository implements IChecksRepository { const baseMatch = { "metadata.teamId": new mongoose.Types.ObjectId(teamId), createdAt: { $gte: getDateForRange(dateRange) }, + ...EXCLUDE_DEGRADED_EGRESS_MATCH, }; const [totalResult, downResult] = await Promise.all([ @@ -393,6 +395,7 @@ class MongoChecksRepository implements IChecksRepository { $match: { "metadata.monitorId": { $in: objectIds }, createdAt: { $gte: windowStart }, + ...EXCLUDE_DEGRADED_EGRESS_MATCH, }, }, { @@ -480,6 +483,8 @@ class MongoChecksRepository implements IChecksRepository { { $facet: { uptimePercentage: [ + // Response-time series (groupedUpChecks / groupedDownChecks) deliberately keep degraded checks so they still show as failures on the graph. + { $match: EXCLUDE_DEGRADED_EGRESS_MATCH }, { $group: { _id: null, diff --git a/server/src/domain/checks/check.type.ts b/server/src/domain/checks/check.type.ts index afea301be..ab304e007 100644 --- a/server/src/domain/checks/check.type.ts +++ b/server/src/domain/checks/check.type.ts @@ -1,6 +1,7 @@ import type { MonitorType } from "@/domain/monitors/monitor.type.js"; import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js"; import type { Response } from "got"; +import type { EgressStatus } from "@/domain/egress/egress.type.js"; export const CHECK_TTL_SENTINEL = 366; @@ -116,6 +117,9 @@ export interface Check { seo?: number; performance?: number; audits?: CheckAudits; + // Set only on checks produced while the reliability probe ran (a failing check). + // "degraded" means every reliability target was unreachable, so the failure is not attributable to the monitored target. + egressStatus?: EgressStatus; createdAt: string; updatedAt: string; } diff --git a/server/src/worker/worker.check-evaluator.ts b/server/src/worker/worker.check-evaluator.ts index da7e13dbe..4c4c4a7ac 100644 --- a/server/src/worker/worker.check-evaluator.ts +++ b/server/src/worker/worker.check-evaluator.ts @@ -4,17 +4,52 @@ import { Check } from "@/domain/checks/check.type.js"; import { Monitor } from "@/domain/monitors/monitor.type.js"; import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; import { IStatusService } from "@/service/statusService.js"; +import { MonitorActionDecision } from "@/worker/worker.helper.js"; +import { ILogger } from "@/utils/logger.js"; export interface ICheckEvaluator { evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise; } +const SERVICE_NAME = "CheckEvaluator"; + export class CheckEvaluator implements ICheckEvaluator { constructor( private statusService: IStatusService, - private monitorStatusPolicy: IMonitorStatusPolicy + private monitorStatusPolicy: IMonitorStatusPolicy, + private logger: ILogger ) {} + + // A check taken while the instance had no outbound connectivity says nothing about the target. + // It is left out of the status window, running stats and monitor status entirely, so that no incident + // opens for it and no spurious "resolved"/"up" fires once egress returns. + private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => { + this.logger.debug({ + message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`, + service: SERVICE_NAME, + method: "evaluate", + }); + const decision: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }; + return { + monitor, + status, + check, + statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() }, + decision, + }; + }; + evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => { + if (check.egressStatus === "degraded") { + return this.skipDegradedEgressCheck(status, check, monitor); + } + // **************************** // Step 3: Evaluate and return result to reactors // **************************** diff --git a/server/src/worker/worker.check-producer.ts b/server/src/worker/worker.check-producer.ts index 603eec8b6..194718a20 100644 --- a/server/src/worker/worker.check-producer.ts +++ b/server/src/worker/worker.check-producer.ts @@ -11,6 +11,7 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/main import { isWindowActive } from "@/utils/maintenanceWindow.js"; import { IProxyResolver } from "@/service/network/ProxyResolver.js"; import { IDockerLogsService } from "@/domain/docker/docker-log.service.js"; +import { IEgressService } from "@/domain/egress/egress.service.js"; export interface ICheckProducer { produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>; @@ -27,6 +28,7 @@ export class CheckProducer implements ICheckProducer { private proxyResolver: IProxyResolver, private bufferService: IBufferService, private dockerLogsService: IDockerLogsService, + private egressService: IEgressService, private logger: ILogger ) {} @@ -74,6 +76,10 @@ export class CheckProducer implements ICheckProducer { throw new Error("No network response"); } + // Step 1c: On failure, ask whether the instance itself can reach anything before blaming the target. + // Null means the egress check is disabled (or failed internally) and the check is treated as usual. + const egressStatus = status.status === false ? await this.egressService.assessAfterFailure() : null; + // **************************** // Step 2: Record // **************************** @@ -89,6 +95,9 @@ export class CheckProducer implements ICheckProducer { }); return null; } + if (egressStatus !== null) { + check.egressStatus = egressStatus; + } // Step 2b: Add to buffer this.bufferService.addToBuffer(check); diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index 0661d6445..9652b021e 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -129,6 +129,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) }; const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) }; const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) }; + const egressService = { assessAfterFailure: jest.fn().mockResolvedValue(null) }; const checkProducer = new CheckProducer( monitorsRepo as any, @@ -138,9 +139,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { proxyResolver as any, bufferStub as any, dockerLogsService as any, + egressService as any, logger ); - const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy()); + const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy(), logger); const notificationReactor = new NotificationReactor(notificationsService as any); const incidentReactor = new IncidentReactor(incidentService as any); diff --git a/server/test/unit/services/checkEvaluator.test.ts b/server/test/unit/services/checkEvaluator.test.ts index 22919495a..23d64a0b0 100644 --- a/server/test/unit/services/checkEvaluator.test.ts +++ b/server/test/unit/services/checkEvaluator.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals"; import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts"; import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; const makeMonitor = (overrides?: Partial): Monitor => ({ @@ -19,9 +20,10 @@ const createEvaluator = (overrides?: Record) => { updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }), }, monitorStatusPolicy: new MonitorStatusPolicy(), + logger: createMockLogger(), ...overrides, }; - const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); + const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any); return { evaluator, defaults }; }; @@ -84,4 +86,38 @@ describe("CheckEvaluator", () => { shouldSendNotification: false, }); }); + + // ── degraded egress ─────────────────────────────────────────────────────── + + it("short-circuits a check flagged with degraded egress without touching monitor status", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; + const check = { id: "check-1", egressStatus: "degraded" } as any; + const monitor = makeMonitor({ status: "up" }); + const { evaluator, defaults } = createEvaluator(); + + const result = await evaluator.evaluate(status, check, monitor); + + expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled(); + expect(result.monitor).toBe(monitor); + expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 }); + expect(result.statusChange.timestamp).toEqual(expect.any(Number)); + expect(result.decision).toEqual({ + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + incidentReason: null, + notificationReason: null, + }); + expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") })); + }); + + it("evaluates a check flagged with egress ok as normal", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; + const check = { id: "check-1", egressStatus: "ok" } as any; + const { evaluator, defaults } = createEvaluator(); + + await evaluator.evaluate(status, check, makeMonitor()); + + expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything()); + }); }); diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts index c10cc2f29..478262c20 100644 --- a/server/test/unit/services/checkProducer.test.ts +++ b/server/test/unit/services/checkProducer.test.ts @@ -29,6 +29,7 @@ const createProducer = (overrides?: Record) => { proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) }, buffer: { addToBuffer: jest.fn() }, dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) }, ...overrides, }; const producer = new CheckProducer( @@ -39,6 +40,7 @@ const createProducer = (overrides?: Record) => { defaults.proxyResolver as any, defaults.buffer as any, defaults.dockerLogsService as any, + defaults.egressService as any, defaults.logger as any ); return { producer, defaults }; @@ -210,4 +212,62 @@ describe("CheckProducer", () => { expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled(); }); + + // ── egress self-check ───────────────────────────────────────────────────── + + const failingStatus = { monitorId: "m1", status: false, code: 500, message: "Error" }; + + it("does not consult the egress service for a successful check and leaves the field unset", async () => { + const check: Record = { id: "check-1" }; + const { producer, defaults } = createProducer({ + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + }); + + await producer.produce(makeMonitor()); + + expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled(); + expect(check).not.toHaveProperty("egressStatus"); + }); + + it("flags a failing check as degraded when the egress service reports degraded egress", async () => { + const check: Record = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") }, + }); + + const result = await producer.produce(makeMonitor()); + + expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check.egressStatus).toBe("degraded"); + expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" })); + expect(result?.check.egressStatus).toBe("degraded"); + }); + + it("flags a failing check as ok when the probe found egress fine", async () => { + const check: Record = { id: "check-1" }; + const { producer } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") }, + }); + + await producer.produce(makeMonitor()); + + expect(check.egressStatus).toBe("ok"); + }); + + it("leaves the field unset on a failing check when the egress check is disabled", async () => { + const check: Record = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + }); + + await producer.produce(makeMonitor()); + + expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1); + expect(check).not.toHaveProperty("egressStatus"); + }); });