From 6d4e944a582c4a7ecdc11cfb120da9853c8e429a Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 23 Sep 2026 11:17:01 -0700 Subject: [PATCH] Add tests --- .../providers/network/dockerProvider.test.ts | 53 +++++- .../notifications/emailProvider.test.ts | 12 ++ .../notifications/pagerdutyProvider.test.ts | 12 ++ .../test/unit/services/checkService.test.ts | 30 +++- .../unit/services/incidentService.test.ts | 40 +++++ .../notificationMessageBuilder.test.ts | 140 +++++++++++++++- .../test/unit/services/statusService.test.ts | 158 +++++++++++++++++- .../test/unit/services/workerPipeline.test.ts | 9 +- server/test/unit/utils/dockerAlert.test.ts | 104 ++++++++++++ .../unit/validation/monitorValidation.test.ts | 54 ++++++ 10 files changed, 601 insertions(+), 11 deletions(-) create mode 100644 server/test/unit/utils/dockerAlert.test.ts diff --git a/server/test/unit/providers/network/dockerProvider.test.ts b/server/test/unit/providers/network/dockerProvider.test.ts index a90e2b65f..50e666a1d 100644 --- a/server/test/unit/providers/network/dockerProvider.test.ts +++ b/server/test/unit/providers/network/dockerProvider.test.ts @@ -229,6 +229,18 @@ describe("DockerProvider", () => { }); }); + it("leaves the exit code undefined for Capture containers", async () => { + const response = makeCaptureResponse({ + payload: { data: [makeCaptureContainer({ status: "exited", running: false, stats: undefined })], errors: [] }, + }); + const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) }); + + const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" })); + + expect(result.payload?.containers[0]?.state).toBe("exited"); + expect(result.payload?.containers[0]?.exitCode).toBeUndefined(); + }); + it("keeps state-derived Capture health separate from Docker healthchecks", async () => { const response = makeCaptureResponse({ payload: { @@ -558,6 +570,16 @@ describe("DockerProvider", () => { ); }); + it("reads the exit code from inspect", async () => { + const inspect = jest.fn().mockResolvedValue(makeInspect({ State: { StartedAt: "2026-08-28T10:00:00.000Z", ExitCode: 137 } })); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status: "Exited (137) 2 minutes ago" })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ state: "exited", exitCode: 137, health: "none" })); + }); + it("counts running, stopped, and unhealthy states in the summary", async () => { const containers = [ makeContainer({ Id: "a".repeat(64) }), @@ -682,11 +704,40 @@ describe("DockerProvider", () => { expect(result.status).toBe(true); expect(result.payload?.containers).toHaveLength(1); - expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ health: "none" })); expect(result.payload?.containers[0]?.restartCount).toBeUndefined(); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Failed to inspect") })); }); + it.each([ + ["Up 3 hours (healthy)", "healthy"], + ["Up 3 hours (unhealthy)", "unhealthy"], + ["Up 5 seconds (health: starting)", "starting"], + ["Up 3 hours", "none"], + ["Exited (1) 2 minutes ago", "none"], + ])("reads health from the list status %s when inspect rejects", async (Status, health) => { + const inspect = jest.fn().mockRejectedValue(new Error("inspect failed")); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ Status })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]?.health).toBe(health); + }); + + it.each([ + ["Exited (0) 2 minutes ago", 0], + ["Exited (137) 2 minutes ago", 137], + ["Up 3 hours", undefined], + ])("reads the exit code from the list status %s when inspect rejects", async (Status, exitCode) => { + const inspect = jest.fn().mockRejectedValue(new Error("inspect failed")); + const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status })]); + const { provider } = setup({ inspect, listContainers }); + + const result = await provider.handle(makeMonitor()); + + expect(result.payload?.containers[0]?.exitCode).toBe(exitCode); + }); + it("keeps the container without metrics when stats rejects", async () => { const stats = jest.fn().mockRejectedValue(new Error("stats failed")); const { provider } = setup({ stats }); diff --git a/server/test/unit/providers/notifications/emailProvider.test.ts b/server/test/unit/providers/notifications/emailProvider.test.ts index 7f3b34d4e..3610523a4 100644 --- a/server/test/unit/providers/notifications/emailProvider.test.ts +++ b/server/test/unit/providers/notifications/emailProvider.test.ts @@ -115,6 +115,18 @@ describe("EmailProvider", () => { expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor thresholds resolved", expect.anything()); }); + it("builds correct subject for container_breach", async () => { + const { provider, emailService } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" })); + expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor container alert", expect.anything()); + }); + + it("builds correct subject for container_resolved", async () => { + const { provider, emailService } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" })); + expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor containers recovered", expect.anything()); + }); + it("builds default subject for unknown type", async () => { const { provider, emailService } = createProvider(); await provider.sendMessage(makeNotification() as any, makeMessage({ type: "test" })); diff --git a/server/test/unit/providers/notifications/pagerdutyProvider.test.ts b/server/test/unit/providers/notifications/pagerdutyProvider.test.ts index 8b8d185b4..b17d212ba 100644 --- a/server/test/unit/providers/notifications/pagerdutyProvider.test.ts +++ b/server/test/unit/providers/notifications/pagerdutyProvider.test.ts @@ -89,6 +89,18 @@ describe("PagerDutyProvider", () => { expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve"); }); + it("uses 'resolve' event_action for container_resolved", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" })); + expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve"); + }); + + it("uses 'trigger' event_action for container_breach", async () => { + const { provider } = createProvider(); + await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" })); + expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("trigger"); + }); + it("includes threshold info in summary and custom_details", async () => { const { provider } = createProvider(); await provider.sendMessage(makeNotification() as any, makeMessageWithThresholds()); diff --git a/server/test/unit/services/checkService.test.ts b/server/test/unit/services/checkService.test.ts index 4ced65947..230857b0e 100644 --- a/server/test/unit/services/checkService.test.ts +++ b/server/test/unit/services/checkService.test.ts @@ -3,7 +3,7 @@ import { CheckService } from "../../../src/domain/checks/check.service.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; @@ -357,6 +357,34 @@ describe("CheckService", () => { expect(fromRaw[0].metric).toBe("cpu"); expect(fromRebuilt).toEqual(fromRaw); }); + + it("docker: preserves payload.containers and payload.summary verbatim", () => { + const { service } = createService(); + const payload: DockerStatusPayload = { + containers: [ + { id: "1", name: "db", image: "postgres:16", state: "exited", status: "Exited (1) 2 minutes ago", health: "none", exitCode: 1 }, + { id: "2", name: "api", image: "api:latest", state: "running", status: "Up 3 hours (unhealthy)", health: "unhealthy", restartCount: 2 }, + ], + summary: { total: 2, running: 1, stopped: 1, unhealthy: 1 }, + }; + const status = makeStatusResponse({ type: "docker", status: true, payload } as any); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + const roundtripped = rebuilt.payload as DockerStatusPayload; + expect(roundtripped.containers).toEqual(payload.containers); + expect(roundtripped.summary).toEqual(payload.summary); + expect(roundtripped.containers[0].exitCode).toBe(1); + }); + + it("docker: leaves payload undefined for a failed check with no containers", () => { + const { service } = createService(); + const status = makeStatusResponse({ type: "docker", status: false, code: 5000, message: "connect ECONNREFUSED", payload: null } as any); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + expect(rebuilt.payload).toBeUndefined(); + }); }); // ── createChecks ───────────────────────────────────────────────────────── diff --git a/server/test/unit/services/incidentService.test.ts b/server/test/unit/services/incidentService.test.ts index 73a422cf7..a2e161769 100644 --- a/server/test/unit/services/incidentService.test.ts +++ b/server/test/unit/services/incidentService.test.ts @@ -175,6 +175,46 @@ describe("IncidentService", () => { expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" })); }); + it("builds the container list for docker threshold_breach incidents", async () => { + const created = makeIncident(); + const { service, incidentsRepository, notificationMessageBuilder } = createService(); + (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); + (incidentsRepository.create as jest.Mock).mockResolvedValue(created); + + const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }); + const response = { + monitorId: "mon-1", + payload: { + containers: [ + { id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 }, + { id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" }, + { id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" }, + ], + summary: { total: 3, running: 2, stopped: 1, unhealthy: 1 }, + }, + } as any; + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + await service.handleIncident(monitor, 200, decision, response); + + expect(incidentsRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" }) + ); + expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled(); + }); + + it("uses the container fallback message for docker incidents with no response", async () => { + const created = makeIncident(); + const { service, incidentsRepository } = createService(); + (incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null); + (incidentsRepository.create as jest.Mock).mockResolvedValue(created); + + const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true }); + const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" }); + await service.handleIncident(monitor, 200, decision, undefined); + + expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 9999, message: "Container alert" })); + }); + it("joins multiple threshold breaches with commas", async () => { const created = makeIncident(); const { service, incidentsRepository, notificationMessageBuilder } = createService(); diff --git a/server/test/unit/services/notificationMessageBuilder.test.ts b/server/test/unit/services/notificationMessageBuilder.test.ts index b0b8397e3..afa6519e9 100644 --- a/server/test/unit/services/notificationMessageBuilder.test.ts +++ b/server/test/unit/services/notificationMessageBuilder.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, beforeEach } from "@jest/globals"; import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -51,6 +51,29 @@ const makeHardwarePayload = (overrides?: Partial) }, }) as HardwareStatusPayload; +const makeDockerPayload = (containers: Partial[]): DockerStatusPayload => ({ + containers: containers.map((container, index) => ({ + id: `container-${index}`, + name: `container-${index}`, + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...container, + })), + summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 }, +}); + +const makeDockerMonitor = (overrides?: Partial): Monitor => + makeMonitor({ + type: "docker", + status: "breached", + url: "unix:///var/run/docker.sock", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + ...overrides, + }); + // ── Tests ──────────────────────────────────────────────────────────────────── describe("NotificationMessageBuilder", () => { @@ -123,7 +146,7 @@ describe("NotificationMessageBuilder", () => { it("builds a threshold_resolved message for hardware monitor recovering", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); - const decision = makeDecision({ notificationReason: "status_change" }); + const decision = makeDecision({ notificationReason: "threshold_resolved" }); const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com"); @@ -163,13 +186,46 @@ describe("NotificationMessageBuilder", () => { expect(msg.type).toBe("threshold_breach"); }); - it("returns threshold_resolved for hardware monitor with status_change and up", () => { + it("returns threshold_resolved for hardware monitor with threshold_resolved reason", () => { + const monitor = makeMonitor({ status: "up", type: "hardware" }); + const decision = makeDecision({ notificationReason: "threshold_resolved" }); + + const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("threshold_resolved"); + }); + + it("returns monitor_up for hardware monitor recovering from down with status_change", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); const decision = makeDecision({ notificationReason: "status_change" }); const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, ""); - expect(msg.type).toBe("threshold_resolved"); + expect(msg.type).toBe("monitor_up"); + }); + + it("returns container_breach for docker monitor with threshold_breach reason", () => { + const decision = makeDecision({ notificationReason: "threshold_breach" }); + + const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("container_breach"); + }); + + it("returns container_resolved for docker monitor with threshold_resolved reason", () => { + const decision = makeDecision({ notificationReason: "threshold_resolved" }); + + const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("container_resolved"); + }); + + it("returns monitor_down for docker monitor that is down regardless of reason", () => { + const decision = makeDecision({ notificationReason: "threshold_breach" }); + + const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeStatusResponse(), decision, ""); + + expect(msg.type).toBe("monitor_down"); }); it("returns monitor_up for non-hardware monitor with status up", () => { @@ -223,7 +279,22 @@ describe("NotificationMessageBuilder", () => { const msg = builder.buildMessage( makeMonitor({ status: "up", type: "hardware" }), makeStatusResponse(), - makeDecision({ notificationReason: "status_change" }), + makeDecision({ notificationReason: "threshold_resolved" }), + "" + ); + expect(msg.severity).toBe("success"); + }); + + it("returns warning for container_breach", () => { + const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), makeDecision({ notificationReason: "threshold_breach" }), ""); + expect(msg.severity).toBe("warning"); + }); + + it("returns success for container_resolved", () => { + const msg = builder.buildMessage( + makeDockerMonitor({ status: "up" }), + makeStatusResponse(), + makeDecision({ notificationReason: "threshold_resolved" }), "" ); expect(msg.severity).toBe("success"); @@ -311,7 +382,7 @@ describe("NotificationMessageBuilder", () => { describe("threshold_resolved", () => { it("includes resolved details", () => { const monitor = makeMonitor({ status: "up", type: "hardware" }); - const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), ""); + const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision({ notificationReason: "threshold_resolved" }), ""); expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor"); expect(msg.content.summary).toBe('Monitor "Test Monitor" thresholds have returned to normal.'); @@ -319,6 +390,63 @@ describe("NotificationMessageBuilder", () => { }); }); + describe("container_breach", () => { + it("lists one detail line per breaching container", () => { + const response = makeStatusResponse({ + type: "docker", + status: true, + payload: makeDockerPayload([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]), + } as any); + + const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.title).toBe("Container Alert: Test Monitor"); + expect(msg.content.summary).toBe('2 container(s) on "Test Monitor" need attention.'); + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "db: stopped (exit code 1)", "api: unhealthy"]); + expect(msg.content.thresholds).toBeUndefined(); + }); + + it("only lists containers matching an enabled switch", () => { + const response = makeStatusResponse({ + type: "docker", + status: true, + payload: makeDockerPayload([ + { name: "db", state: "exited", exitCode: 1 }, + { name: "api", health: "unhealthy" }, + ]), + } as any); + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); + + const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "api: unhealthy"]); + }); + + it("lists no containers when the payload is an error string", () => { + const response = makeStatusResponse({ type: "docker", status: false, payload: "connect ECONNREFUSED" } as any); + + const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), ""); + + expect(msg.content.summary).toBe('0 container(s) on "Test Monitor" need attention.'); + expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker"]); + }); + }); + + describe("container_resolved", () => { + it("includes recovered details", () => { + const msg = builder.buildMessage( + makeDockerMonitor({ status: "up" }), + makeStatusResponse(), + makeDecision({ notificationReason: "threshold_resolved" }), + "" + ); + + expect(msg.content.title).toBe("Containers Recovered: Test Monitor"); + expect(msg.content.summary).toBe('All containers on "Test Monitor" are back to normal.'); + expect(msg.content.details).toContain("Status: Up"); + }); + }); + describe("default content", () => { it("builds default content for unhandled notification type", () => { // Force default by creating a scenario where type falls through diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 071465d48..393cb5676 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts"; import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; -import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; +import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; @@ -1086,6 +1086,162 @@ describe("StatusService", () => { // they're private implementation details; exposing them publicly would // widen the API surface just for testability. + describe("docker container alerts", () => { + const makeDockerMonitor = (overrides?: Partial) => + makeMonitor({ + type: "docker", + url: "unix:///var/run/docker.sock", + statusWindow: [true, true, true, true], + statusWindowSize: 5, + status: "up", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + ...overrides, + }); + + const makeContainers = (containers: Partial[]): DockerStatusPayload => ({ + containers: containers.map((container, index) => ({ + id: `container-${index}`, + name: `container-${index}`, + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...container, + })), + summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 }, + }); + + const makeDockerResponse = (payload: DockerStatusPayload | null, overrides?: Partial>) => + makeStatusResponse({ + type: "docker", + status: true, + payload, + ...overrides, + } as any) as MonitorStatusResponse; + + const run = async (monitor: Monitor, response: MonitorStatusResponse, check = makeCheck()) => { + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); + const result = await service.updateMonitorStatus(response, check, monitor); + return { result, monitorsRepository }; + }; + + const stopped = { name: "db", state: "exited" as const, exitCode: 1 }; + const unhealthy = { name: "api", health: "unhealthy" as const }; + + it("goes to breached on the first check with a stopped container", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("breached"); + }); + + it("goes to breached on the first check with an unhealthy container", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([unhealthy]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("breached"); + }); + + it("stays up on a stopped container when both switches are off", async () => { + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped, unhealthy]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays up on a stopped container when only the unhealthy switch is on", async () => { + const monitor = makeDockerMonitor({ dockerAlertOnStopped: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays up on an exited container with code 0", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "job", state: "exited", exitCode: 0 }]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("stays breached without a transition while a container is still down", async () => { + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("breached"); + }); + + it("stays up without a transition while every container is clean", async () => { + const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "web" }]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("recovers from breached to up on the first clean check", async () => { + const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([{ name: "web" }]))); + + expect(result.statusChanged).toBe(true); + expect(result.prevStatus).toBe("breached"); + expect(result.monitor.status).toBe("up"); + }); + + it("recovers from breached to up when the breaching switch is turned off", async () => { + const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("up"); + }); + + it("does not override down with breached when the host is unreachable", async () => { + const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.monitor.status).toBe("down"); + }); + + it("transitions from breached to down when the reachability threshold trips", async () => { + const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("down"); + }); + + it("does not evaluate containers while the status window is short", async () => { + const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 }); + const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped]))); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + }); + + it("does not evaluate a failed check with a null payload", async () => { + const monitor = makeDockerMonitor({ status: "breached" }); + const response = makeDockerResponse(null, { status: false, code: 5000 }); + const { result } = await run(monitor, response, makeCheck({ status: false })); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("breached"); + }); + + it("does not touch the hardware counters or thresholdBreaches", async () => { + const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped]))); + + expect(result.thresholdBreaches).toBeUndefined(); + const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; + expect(patch.cpuAlertCounter).toBeUndefined(); + expect(patch.status).toBe("breached"); + }); + }); + describe("computeReachability (pure)", () => { const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => { const { service } = createService(); diff --git a/server/test/unit/services/workerPipeline.test.ts b/server/test/unit/services/workerPipeline.test.ts index e7c117152..0b168ef24 100644 --- a/server/test/unit/services/workerPipeline.test.ts +++ b/server/test/unit/services/workerPipeline.test.ts @@ -861,9 +861,14 @@ describe("WorkerPipeline", () => { }); }); - it("resolves incident when monitor recovers from breached", async () => { + it("resolves incident with threshold_resolved when monitor recovers from breached", async () => { const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" })); - expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); + expect(decision).toMatchObject({ + shouldResolveIncident: true, + shouldSendNotification: true, + shouldCreateIncident: false, + notificationReason: "threshold_resolved", + }); }); it("does not create or resolve for unhandled status transitions", async () => { diff --git a/server/test/unit/utils/dockerAlert.test.ts b/server/test/unit/utils/dockerAlert.test.ts new file mode 100644 index 000000000..a9d360511 --- /dev/null +++ b/server/test/unit/utils/dockerAlert.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "@jest/globals"; +import { describeContainerBreach, findContainerBreaches, isContainerStopped, isContainerUnhealthy } from "../../../src/domain/docker/docker-alert.ts"; +import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; + +const makeContainer = (overrides?: Partial): DockerContainerInfo => ({ + id: "abc123", + name: "web", + image: "nginx:latest", + state: "running", + status: "Up 3 hours", + health: "none", + ...overrides, +}); + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "mon-1", + type: "docker", + dockerAlertOnStopped: false, + dockerAlertOnUnhealthy: false, + ...overrides, + }) as Monitor; + +describe("isContainerStopped", () => { + it("treats dead as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "dead" }))).toBe(true); + }); + + it.each([1, 137, 143])("treats exited with code %s as stopped", (exitCode) => { + expect(isContainerStopped(makeContainer({ state: "exited", exitCode }))).toBe(true); + }); + + it("does not treat exited with code 0 as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "exited", exitCode: 0 }))).toBe(false); + }); + + it("treats exited with an unknown exit code as stopped", () => { + expect(isContainerStopped(makeContainer({ state: "exited" }))).toBe(true); + }); + + it.each(["running", "restarting", "paused", "created", "removing"] as const)("does not treat %s as stopped", (state) => { + expect(isContainerStopped(makeContainer({ state, exitCode: 1 }))).toBe(false); + }); +}); + +describe("isContainerUnhealthy", () => { + it("treats unhealthy as unhealthy", () => { + expect(isContainerUnhealthy(makeContainer({ health: "unhealthy" }))).toBe(true); + }); + + it.each(["healthy", "starting", "none"] as const)("does not treat %s as unhealthy", (health) => { + expect(isContainerUnhealthy(makeContainer({ health }))).toBe(false); + }); +}); + +describe("findContainerBreaches", () => { + const stopped = makeContainer({ name: "db", state: "exited", exitCode: 1 }); + const unhealthy = makeContainer({ name: "api", health: "unhealthy" }); + + it("returns nothing when both switches are off", () => { + expect(findContainerBreaches(makeMonitor(), [stopped, unhealthy])).toEqual([]); + }); + + it("reports only stopped containers when only the stopped switch is on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true }), [stopped, unhealthy]); + expect(breaches).toEqual([{ name: "db", reason: "stopped", exitCode: 1 }]); + }); + + it("reports only unhealthy containers when only the unhealthy switch is on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnUnhealthy: true }), [stopped, unhealthy]); + expect(breaches).toEqual([{ name: "api", reason: "unhealthy" }]); + }); + + it("reports both conditions when both switches are on", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [stopped, unhealthy]); + expect(breaches.map((b) => b.reason)).toEqual(["stopped", "unhealthy"]); + }); + + it("reports a container that is both stopped and unhealthy once, as stopped", () => { + const both = makeContainer({ name: "worker", state: "dead", health: "unhealthy" }); + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [both]); + expect(breaches).toEqual([{ name: "worker", reason: "stopped", exitCode: undefined }]); + }); + + it("ignores healthy running containers", () => { + const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [makeContainer()]); + expect(breaches).toEqual([]); + }); +}); + +describe("describeContainerBreach", () => { + it("describes an unhealthy container", () => { + expect(describeContainerBreach({ name: "api", reason: "unhealthy" })).toBe("api: unhealthy"); + }); + + it("describes a stopped container with its exit code", () => { + expect(describeContainerBreach({ name: "db", reason: "stopped", exitCode: 137 })).toBe("db: stopped (exit code 137)"); + }); + + it("describes a stopped container without an exit code", () => { + expect(describeContainerBreach({ name: "db", reason: "stopped" })).toBe("db: stopped"); + }); +}); diff --git a/server/test/unit/validation/monitorValidation.test.ts b/server/test/unit/validation/monitorValidation.test.ts index 218a0942d..63f3572a8 100644 --- a/server/test/unit/validation/monitorValidation.test.ts +++ b/server/test/unit/validation/monitorValidation.test.ts @@ -247,6 +247,20 @@ describe("monitorValidation — strategy gating", () => { expect(parsed.monitors[0].dockerLogsEnabled).toBe(false); }); + it("defaults the docker alert switches to false on imported docker monitors", () => { + const parsed = importMonitorsBodyValidation.parse({ + monitors: [ + { + name: "Imported Docker", + type: "docker", + url: "unix:///var/run/docker.sock", + }, + ], + }); + expect(parsed.monitors[0].dockerAlertOnStopped).toBe(false); + expect(parsed.monitors[0].dockerAlertOnUnhealthy).toBe(false); + }); + it("rejects strategy on imported non-pagespeed monitors", () => { expect(() => importMonitorsBodyValidation.parse({ @@ -818,3 +832,43 @@ describe("monitorValidation — Docker TLS credentials", () => { }); }); }); + +describe("monitorValidation — docker alert switches", () => { + describe("createMonitorBodyValidation", () => { + it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on a docker monitor", () => { + const parsed = createMonitorBodyValidation.parse({ + name: "Docker host", + type: "docker", + url: "unix:///var/run/docker.sock", + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: true, + }); + + expect(parsed.dockerAlertOnStopped).toBe(true); + expect(parsed.dockerAlertOnUnhealthy).toBe(true); + }); + + it("leaves the switches undefined when omitted", () => { + const parsed = createMonitorBodyValidation.parse({ + name: "Docker host", + type: "docker", + url: "unix:///var/run/docker.sock", + }); + + expect(parsed.dockerAlertOnStopped).toBeUndefined(); + expect(parsed.dockerAlertOnUnhealthy).toBeUndefined(); + }); + }); + + describe("editMonitorBodyValidation", () => { + it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on edits", () => { + const parsed = editMonitorBodyValidation.parse({ + dockerAlertOnStopped: true, + dockerAlertOnUnhealthy: false, + }); + + expect(parsed.dockerAlertOnStopped).toBe(true); + expect(parsed.dockerAlertOnUnhealthy).toBe(false); + }); + }); +});