Add tests
This commit is contained in:
1 parent
cfe7d2a359
commit
6d4e944a58
10 files changed
+601
-11
No files matched your search
@@ -229,6 +229,18 @@ describe("DockerProvider", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves the exit code undefined for Capture containers", async () => {
|
||||
const response = makeCaptureResponse({
|
||||
payload: { data: [makeCaptureContainer({ status: "exited", running: false, stats: undefined })], errors: [] },
|
||||
});
|
||||
const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) });
|
||||
|
||||
const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" }));
|
||||
|
||||
expect(result.payload?.containers[0]?.state).toBe("exited");
|
||||
expect(result.payload?.containers[0]?.exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps state-derived Capture health separate from Docker healthchecks", async () => {
|
||||
const response = makeCaptureResponse({
|
||||
payload: {
|
||||
@@ -558,6 +570,16 @@ describe("DockerProvider", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("reads the exit code from inspect", async () => {
|
||||
const inspect = jest.fn().mockResolvedValue(makeInspect({ State: { StartedAt: "2026-08-28T10:00:00.000Z", ExitCode: 137 } }));
|
||||
const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status: "Exited (137) 2 minutes ago" })]);
|
||||
const { provider } = setup({ inspect, listContainers });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ state: "exited", exitCode: 137, health: "none" }));
|
||||
});
|
||||
|
||||
it("counts running, stopped, and unhealthy states in the summary", async () => {
|
||||
const containers = [
|
||||
makeContainer({ Id: "a".repeat(64) }),
|
||||
@@ -682,11 +704,40 @@ describe("DockerProvider", () => {
|
||||
|
||||
expect(result.status).toBe(true);
|
||||
expect(result.payload?.containers).toHaveLength(1);
|
||||
expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ health: "none" }));
|
||||
expect(result.payload?.containers[0]?.restartCount).toBeUndefined();
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Failed to inspect") }));
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Up 3 hours (healthy)", "healthy"],
|
||||
["Up 3 hours (unhealthy)", "unhealthy"],
|
||||
["Up 5 seconds (health: starting)", "starting"],
|
||||
["Up 3 hours", "none"],
|
||||
["Exited (1) 2 minutes ago", "none"],
|
||||
])("reads health from the list status %s when inspect rejects", async (Status, health) => {
|
||||
const inspect = jest.fn().mockRejectedValue(new Error("inspect failed"));
|
||||
const listContainers = jest.fn().mockResolvedValue([makeContainer({ Status })]);
|
||||
const { provider } = setup({ inspect, listContainers });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.payload?.containers[0]?.health).toBe(health);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Exited (0) 2 minutes ago", 0],
|
||||
["Exited (137) 2 minutes ago", 137],
|
||||
["Up 3 hours", undefined],
|
||||
])("reads the exit code from the list status %s when inspect rejects", async (Status, exitCode) => {
|
||||
const inspect = jest.fn().mockRejectedValue(new Error("inspect failed"));
|
||||
const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status })]);
|
||||
const { provider } = setup({ inspect, listContainers });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.payload?.containers[0]?.exitCode).toBe(exitCode);
|
||||
});
|
||||
|
||||
it("keeps the container without metrics when stats rejects", async () => {
|
||||
const stats = jest.fn().mockRejectedValue(new Error("stats failed"));
|
||||
const { provider } = setup({ stats });
|
||||
|
||||
@@ -115,6 +115,18 @@ describe("EmailProvider", () => {
|
||||
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor thresholds resolved", expect.anything());
|
||||
});
|
||||
|
||||
it("builds correct subject for container_breach", async () => {
|
||||
const { provider, emailService } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" }));
|
||||
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor container alert", expect.anything());
|
||||
});
|
||||
|
||||
it("builds correct subject for container_resolved", async () => {
|
||||
const { provider, emailService } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" }));
|
||||
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor containers recovered", expect.anything());
|
||||
});
|
||||
|
||||
it("builds default subject for unknown type", async () => {
|
||||
const { provider, emailService } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "test" }));
|
||||
|
||||
@@ -89,6 +89,18 @@ describe("PagerDutyProvider", () => {
|
||||
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve");
|
||||
});
|
||||
|
||||
it("uses 'resolve' event_action for container_resolved", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" }));
|
||||
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve");
|
||||
});
|
||||
|
||||
it("uses 'trigger' event_action for container_breach", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" }));
|
||||
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("trigger");
|
||||
});
|
||||
|
||||
it("includes threshold info in summary and custom_details", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithThresholds());
|
||||
|
||||
@@ -3,7 +3,7 @@ import { CheckService } from "../../../src/domain/checks/check.service.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts";
|
||||
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
|
||||
@@ -357,6 +357,34 @@ describe("CheckService", () => {
|
||||
expect(fromRaw[0].metric).toBe("cpu");
|
||||
expect(fromRebuilt).toEqual(fromRaw);
|
||||
});
|
||||
|
||||
it("docker: preserves payload.containers and payload.summary verbatim", () => {
|
||||
const { service } = createService();
|
||||
const payload: DockerStatusPayload = {
|
||||
containers: [
|
||||
{ id: "1", name: "db", image: "postgres:16", state: "exited", status: "Exited (1) 2 minutes ago", health: "none", exitCode: 1 },
|
||||
{ id: "2", name: "api", image: "api:latest", state: "running", status: "Up 3 hours (unhealthy)", health: "unhealthy", restartCount: 2 },
|
||||
],
|
||||
summary: { total: 2, running: 1, stopped: 1, unhealthy: 1 },
|
||||
};
|
||||
const status = makeStatusResponse({ type: "docker", status: true, payload } as any);
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
const roundtripped = rebuilt.payload as DockerStatusPayload;
|
||||
expect(roundtripped.containers).toEqual(payload.containers);
|
||||
expect(roundtripped.summary).toEqual(payload.summary);
|
||||
expect(roundtripped.containers[0].exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it("docker: leaves payload undefined for a failed check with no containers", () => {
|
||||
const { service } = createService();
|
||||
const status = makeStatusResponse({ type: "docker", status: false, code: 5000, message: "connect ECONNREFUSED", payload: null } as any);
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
expect(rebuilt.payload).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── createChecks ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -175,6 +175,46 @@ describe("IncidentService", () => {
|
||||
expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" }));
|
||||
});
|
||||
|
||||
it("builds the container list for docker threshold_breach incidents", async () => {
|
||||
const created = makeIncident();
|
||||
const { service, incidentsRepository, notificationMessageBuilder } = createService();
|
||||
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
|
||||
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
|
||||
|
||||
const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true });
|
||||
const response = {
|
||||
monitorId: "mon-1",
|
||||
payload: {
|
||||
containers: [
|
||||
{ id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 },
|
||||
{ id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" },
|
||||
{ id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" },
|
||||
],
|
||||
summary: { total: 3, running: 2, stopped: 1, unhealthy: 1 },
|
||||
},
|
||||
} as any;
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
|
||||
await service.handleIncident(monitor, 200, decision, response);
|
||||
|
||||
expect(incidentsRepository.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" })
|
||||
);
|
||||
expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the container fallback message for docker incidents with no response", async () => {
|
||||
const created = makeIncident();
|
||||
const { service, incidentsRepository } = createService();
|
||||
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
|
||||
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
|
||||
|
||||
const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true });
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
|
||||
await service.handleIncident(monitor, 200, decision, undefined);
|
||||
|
||||
expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 9999, message: "Container alert" }));
|
||||
});
|
||||
|
||||
it("joins multiple threshold breaches with commas", async () => {
|
||||
const created = makeIncident();
|
||||
const { service, incidentsRepository, notificationMessageBuilder } = createService();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
@@ -51,6 +51,29 @@ const makeHardwarePayload = (overrides?: Partial<HardwareStatusPayload["data"]>)
|
||||
},
|
||||
}) as HardwareStatusPayload;
|
||||
|
||||
const makeDockerPayload = (containers: Partial<DockerStatusPayload["containers"][number]>[]): DockerStatusPayload => ({
|
||||
containers: containers.map((container, index) => ({
|
||||
id: `container-${index}`,
|
||||
name: `container-${index}`,
|
||||
image: "nginx:latest",
|
||||
state: "running",
|
||||
status: "Up 3 hours",
|
||||
health: "none",
|
||||
...container,
|
||||
})),
|
||||
summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 },
|
||||
});
|
||||
|
||||
const makeDockerMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
makeMonitor({
|
||||
type: "docker",
|
||||
status: "breached",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: true,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("NotificationMessageBuilder", () => {
|
||||
@@ -123,7 +146,7 @@ describe("NotificationMessageBuilder", () => {
|
||||
|
||||
it("builds a threshold_resolved message for hardware monitor recovering", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const decision = makeDecision({ notificationReason: "status_change" });
|
||||
const decision = makeDecision({ notificationReason: "threshold_resolved" });
|
||||
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com");
|
||||
|
||||
@@ -163,13 +186,46 @@ describe("NotificationMessageBuilder", () => {
|
||||
expect(msg.type).toBe("threshold_breach");
|
||||
});
|
||||
|
||||
it("returns threshold_resolved for hardware monitor with status_change and up", () => {
|
||||
it("returns threshold_resolved for hardware monitor with threshold_resolved reason", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const decision = makeDecision({ notificationReason: "threshold_resolved" });
|
||||
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("threshold_resolved");
|
||||
});
|
||||
|
||||
it("returns monitor_up for hardware monitor recovering from down with status_change", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const decision = makeDecision({ notificationReason: "status_change" });
|
||||
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("threshold_resolved");
|
||||
expect(msg.type).toBe("monitor_up");
|
||||
});
|
||||
|
||||
it("returns container_breach for docker monitor with threshold_breach reason", () => {
|
||||
const decision = makeDecision({ notificationReason: "threshold_breach" });
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("container_breach");
|
||||
});
|
||||
|
||||
it("returns container_resolved for docker monitor with threshold_resolved reason", () => {
|
||||
const decision = makeDecision({ notificationReason: "threshold_resolved" });
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("container_resolved");
|
||||
});
|
||||
|
||||
it("returns monitor_down for docker monitor that is down regardless of reason", () => {
|
||||
const decision = makeDecision({ notificationReason: "threshold_breach" });
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("monitor_down");
|
||||
});
|
||||
|
||||
it("returns monitor_up for non-hardware monitor with status up", () => {
|
||||
@@ -223,7 +279,22 @@ describe("NotificationMessageBuilder", () => {
|
||||
const msg = builder.buildMessage(
|
||||
makeMonitor({ status: "up", type: "hardware" }),
|
||||
makeStatusResponse(),
|
||||
makeDecision({ notificationReason: "status_change" }),
|
||||
makeDecision({ notificationReason: "threshold_resolved" }),
|
||||
""
|
||||
);
|
||||
expect(msg.severity).toBe("success");
|
||||
});
|
||||
|
||||
it("returns warning for container_breach", () => {
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
expect(msg.severity).toBe("warning");
|
||||
});
|
||||
|
||||
it("returns success for container_resolved", () => {
|
||||
const msg = builder.buildMessage(
|
||||
makeDockerMonitor({ status: "up" }),
|
||||
makeStatusResponse(),
|
||||
makeDecision({ notificationReason: "threshold_resolved" }),
|
||||
""
|
||||
);
|
||||
expect(msg.severity).toBe("success");
|
||||
@@ -311,7 +382,7 @@ describe("NotificationMessageBuilder", () => {
|
||||
describe("threshold_resolved", () => {
|
||||
it("includes resolved details", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision({ notificationReason: "threshold_resolved" }), "");
|
||||
|
||||
expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor");
|
||||
expect(msg.content.summary).toBe('Monitor "Test Monitor" thresholds have returned to normal.');
|
||||
@@ -319,6 +390,63 @@ describe("NotificationMessageBuilder", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("container_breach", () => {
|
||||
it("lists one detail line per breaching container", () => {
|
||||
const response = makeStatusResponse({
|
||||
type: "docker",
|
||||
status: true,
|
||||
payload: makeDockerPayload([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]),
|
||||
} as any);
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
|
||||
expect(msg.content.title).toBe("Container Alert: Test Monitor");
|
||||
expect(msg.content.summary).toBe('2 container(s) on "Test Monitor" need attention.');
|
||||
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "db: stopped (exit code 1)", "api: unhealthy"]);
|
||||
expect(msg.content.thresholds).toBeUndefined();
|
||||
});
|
||||
|
||||
it("only lists containers matching an enabled switch", () => {
|
||||
const response = makeStatusResponse({
|
||||
type: "docker",
|
||||
status: true,
|
||||
payload: makeDockerPayload([
|
||||
{ name: "db", state: "exited", exitCode: 1 },
|
||||
{ name: "api", health: "unhealthy" },
|
||||
]),
|
||||
} as any);
|
||||
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false });
|
||||
|
||||
const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
|
||||
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "api: unhealthy"]);
|
||||
});
|
||||
|
||||
it("lists no containers when the payload is an error string", () => {
|
||||
const response = makeStatusResponse({ type: "docker", status: false, payload: "connect ECONNREFUSED" } as any);
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
|
||||
expect(msg.content.summary).toBe('0 container(s) on "Test Monitor" need attention.');
|
||||
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("container_resolved", () => {
|
||||
it("includes recovered details", () => {
|
||||
const msg = builder.buildMessage(
|
||||
makeDockerMonitor({ status: "up" }),
|
||||
makeStatusResponse(),
|
||||
makeDecision({ notificationReason: "threshold_resolved" }),
|
||||
""
|
||||
);
|
||||
|
||||
expect(msg.content.title).toBe("Containers Recovered: Test Monitor");
|
||||
expect(msg.content.summary).toBe('All containers on "Test Monitor" are back to normal.');
|
||||
expect(msg.content.details).toContain("Status: Up");
|
||||
});
|
||||
});
|
||||
|
||||
describe("default content", () => {
|
||||
it("builds default content for unhandled notification type", () => {
|
||||
// Force default by creating a scenario where type falls through
|
||||
|
||||
@@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { Check } from "../../../src/domain/checks/check.type.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts";
|
||||
|
||||
@@ -1086,6 +1086,162 @@ describe("StatusService", () => {
|
||||
// they're private implementation details; exposing them publicly would
|
||||
// widen the API surface just for testability.
|
||||
|
||||
describe("docker container alerts", () => {
|
||||
const makeDockerMonitor = (overrides?: Partial<Monitor>) =>
|
||||
makeMonitor({
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
statusWindow: [true, true, true, true],
|
||||
statusWindowSize: 5,
|
||||
status: "up",
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: true,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const makeContainers = (containers: Partial<DockerStatusPayload["containers"][number]>[]): DockerStatusPayload => ({
|
||||
containers: containers.map((container, index) => ({
|
||||
id: `container-${index}`,
|
||||
name: `container-${index}`,
|
||||
image: "nginx:latest",
|
||||
state: "running",
|
||||
status: "Up 3 hours",
|
||||
health: "none",
|
||||
...container,
|
||||
})),
|
||||
summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 },
|
||||
});
|
||||
|
||||
const makeDockerResponse = (payload: DockerStatusPayload | null, overrides?: Partial<MonitorStatusResponse<DockerStatusPayload>>) =>
|
||||
makeStatusResponse({
|
||||
type: "docker",
|
||||
status: true,
|
||||
payload,
|
||||
...overrides,
|
||||
} as any) as MonitorStatusResponse<DockerStatusPayload>;
|
||||
|
||||
const run = async (monitor: Monitor, response: MonitorStatusResponse<DockerStatusPayload>, check = makeCheck()) => {
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
const result = await service.updateMonitorStatus(response, check, monitor);
|
||||
return { result, monitorsRepository };
|
||||
};
|
||||
|
||||
const stopped = { name: "db", state: "exited" as const, exitCode: 1 };
|
||||
const unhealthy = { name: "api", health: "unhealthy" as const };
|
||||
|
||||
it("goes to breached on the first check with a stopped container", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("goes to breached on the first check with an unhealthy container", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([unhealthy])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("stays up on a stopped container when both switches are off", async () => {
|
||||
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped, unhealthy])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("stays up on a stopped container when only the unhealthy switch is on", async () => {
|
||||
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("stays up on an exited container with code 0", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "job", state: "exited", exitCode: 0 }])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("stays breached without a transition while a container is still down", async () => {
|
||||
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("stays up without a transition while every container is clean", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "web" }])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("recovers from breached to up on the first clean check", async () => {
|
||||
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([{ name: "web" }])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.prevStatus).toBe("breached");
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("recovers from breached to up when the breaching switch is turned off", async () => {
|
||||
const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("does not override down with breached when the host is unreachable", async () => {
|
||||
const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
|
||||
const response = makeDockerResponse(null, { status: false, code: 5000 });
|
||||
const { result } = await run(monitor, response, makeCheck({ status: false }));
|
||||
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
|
||||
it("transitions from breached to down when the reachability threshold trips", async () => {
|
||||
const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
|
||||
const response = makeDockerResponse(null, { status: false, code: 5000 });
|
||||
const { result } = await run(monitor, response, makeCheck({ status: false }));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
|
||||
it("does not evaluate containers while the status window is short", async () => {
|
||||
const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("does not evaluate a failed check with a null payload", async () => {
|
||||
const monitor = makeDockerMonitor({ status: "breached" });
|
||||
const response = makeDockerResponse(null, { status: false, code: 5000 });
|
||||
const { result } = await run(monitor, response, makeCheck({ status: false }));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("does not touch the hardware counters or thresholdBreaches", async () => {
|
||||
const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.thresholdBreaches).toBeUndefined();
|
||||
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
||||
expect(patch.cpuAlertCounter).toBeUndefined();
|
||||
expect(patch.status).toBe("breached");
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeReachability (pure)", () => {
|
||||
const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => {
|
||||
const { service } = createService();
|
||||
|
||||
@@ -861,9 +861,14 @@ describe("WorkerPipeline", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves incident when monitor recovers from breached", async () => {
|
||||
it("resolves incident with threshold_resolved when monitor recovers from breached", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" }));
|
||||
expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
|
||||
expect(decision).toMatchObject({
|
||||
shouldResolveIncident: true,
|
||||
shouldSendNotification: true,
|
||||
shouldCreateIncident: false,
|
||||
notificationReason: "threshold_resolved",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not create or resolve for unhandled status transitions", async () => {
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { describeContainerBreach, findContainerBreaches, isContainerStopped, isContainerUnhealthy } from "../../../src/domain/docker/docker-alert.ts";
|
||||
import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
|
||||
const makeContainer = (overrides?: Partial<DockerContainerInfo>): DockerContainerInfo => ({
|
||||
id: "abc123",
|
||||
name: "web",
|
||||
image: "nginx:latest",
|
||||
state: "running",
|
||||
status: "Up 3 hours",
|
||||
health: "none",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "mon-1",
|
||||
type: "docker",
|
||||
dockerAlertOnStopped: false,
|
||||
dockerAlertOnUnhealthy: false,
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
describe("isContainerStopped", () => {
|
||||
it("treats dead as stopped", () => {
|
||||
expect(isContainerStopped(makeContainer({ state: "dead" }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each([1, 137, 143])("treats exited with code %s as stopped", (exitCode) => {
|
||||
expect(isContainerStopped(makeContainer({ state: "exited", exitCode }))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat exited with code 0 as stopped", () => {
|
||||
expect(isContainerStopped(makeContainer({ state: "exited", exitCode: 0 }))).toBe(false);
|
||||
});
|
||||
|
||||
it("treats exited with an unknown exit code as stopped", () => {
|
||||
expect(isContainerStopped(makeContainer({ state: "exited" }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["running", "restarting", "paused", "created", "removing"] as const)("does not treat %s as stopped", (state) => {
|
||||
expect(isContainerStopped(makeContainer({ state, exitCode: 1 }))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isContainerUnhealthy", () => {
|
||||
it("treats unhealthy as unhealthy", () => {
|
||||
expect(isContainerUnhealthy(makeContainer({ health: "unhealthy" }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["healthy", "starting", "none"] as const)("does not treat %s as unhealthy", (health) => {
|
||||
expect(isContainerUnhealthy(makeContainer({ health }))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findContainerBreaches", () => {
|
||||
const stopped = makeContainer({ name: "db", state: "exited", exitCode: 1 });
|
||||
const unhealthy = makeContainer({ name: "api", health: "unhealthy" });
|
||||
|
||||
it("returns nothing when both switches are off", () => {
|
||||
expect(findContainerBreaches(makeMonitor(), [stopped, unhealthy])).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports only stopped containers when only the stopped switch is on", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true }), [stopped, unhealthy]);
|
||||
expect(breaches).toEqual([{ name: "db", reason: "stopped", exitCode: 1 }]);
|
||||
});
|
||||
|
||||
it("reports only unhealthy containers when only the unhealthy switch is on", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnUnhealthy: true }), [stopped, unhealthy]);
|
||||
expect(breaches).toEqual([{ name: "api", reason: "unhealthy" }]);
|
||||
});
|
||||
|
||||
it("reports both conditions when both switches are on", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [stopped, unhealthy]);
|
||||
expect(breaches.map((b) => b.reason)).toEqual(["stopped", "unhealthy"]);
|
||||
});
|
||||
|
||||
it("reports a container that is both stopped and unhealthy once, as stopped", () => {
|
||||
const both = makeContainer({ name: "worker", state: "dead", health: "unhealthy" });
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [both]);
|
||||
expect(breaches).toEqual([{ name: "worker", reason: "stopped", exitCode: undefined }]);
|
||||
});
|
||||
|
||||
it("ignores healthy running containers", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [makeContainer()]);
|
||||
expect(breaches).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("describeContainerBreach", () => {
|
||||
it("describes an unhealthy container", () => {
|
||||
expect(describeContainerBreach({ name: "api", reason: "unhealthy" })).toBe("api: unhealthy");
|
||||
});
|
||||
|
||||
it("describes a stopped container with its exit code", () => {
|
||||
expect(describeContainerBreach({ name: "db", reason: "stopped", exitCode: 137 })).toBe("db: stopped (exit code 137)");
|
||||
});
|
||||
|
||||
it("describes a stopped container without an exit code", () => {
|
||||
expect(describeContainerBreach({ name: "db", reason: "stopped" })).toBe("db: stopped");
|
||||
});
|
||||
});
|
||||
@@ -247,6 +247,20 @@ describe("monitorValidation — strategy gating", () => {
|
||||
expect(parsed.monitors[0].dockerLogsEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("defaults the docker alert switches to false on imported docker monitors", () => {
|
||||
const parsed = importMonitorsBodyValidation.parse({
|
||||
monitors: [
|
||||
{
|
||||
name: "Imported Docker",
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(parsed.monitors[0].dockerAlertOnStopped).toBe(false);
|
||||
expect(parsed.monitors[0].dockerAlertOnUnhealthy).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects strategy on imported non-pagespeed monitors", () => {
|
||||
expect(() =>
|
||||
importMonitorsBodyValidation.parse({
|
||||
@@ -818,3 +832,43 @@ describe("monitorValidation — Docker TLS credentials", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("monitorValidation — docker alert switches", () => {
|
||||
describe("createMonitorBodyValidation", () => {
|
||||
it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on a docker monitor", () => {
|
||||
const parsed = createMonitorBodyValidation.parse({
|
||||
name: "Docker host",
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: true,
|
||||
});
|
||||
|
||||
expect(parsed.dockerAlertOnStopped).toBe(true);
|
||||
expect(parsed.dockerAlertOnUnhealthy).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves the switches undefined when omitted", () => {
|
||||
const parsed = createMonitorBodyValidation.parse({
|
||||
name: "Docker host",
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
});
|
||||
|
||||
expect(parsed.dockerAlertOnStopped).toBeUndefined();
|
||||
expect(parsed.dockerAlertOnUnhealthy).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("editMonitorBodyValidation", () => {
|
||||
it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on edits", () => {
|
||||
const parsed = editMonitorBodyValidation.parse({
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: false,
|
||||
});
|
||||
|
||||
expect(parsed.dockerAlertOnStopped).toBe(true);
|
||||
expect(parsed.dockerAlertOnUnhealthy).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user