add heartbeat tests
This commit is contained in:
1 parent
45d4779b45
commit
233966b73d
10 files changed
+1148
-1
No files matched your search
@@ -0,0 +1,76 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
|
||||
describe("Heartbeat job: down detection", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("transitions monitor to down and creates incident after threshold failures", async () => {
|
||||
// Monitor starts with full passing window: [true, true, true, true, true]
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// 2 failures: window = [true, true, true, false, false] = 40% < 60%
|
||||
h.setNextResponse(false, 503);
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
let storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
|
||||
// 3rd failure: window = [true, true, false, false, false] = 60% >= 60%
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("down");
|
||||
|
||||
// Incident should have been created
|
||||
const incidents = h.incidentsRepo.getAll();
|
||||
expect(incidents).toHaveLength(1);
|
||||
expect(incidents[0].monitorId).toBe("mon-1");
|
||||
expect(incidents[0].status).toBe(true);
|
||||
expect(incidents[0].statusCode).toBe(503);
|
||||
|
||||
// Notification should have been triggered
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not create duplicate incidents on continued failures", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setNextResponse(false, 500);
|
||||
|
||||
// 3 failures trigger down, 4 more continue failing
|
||||
for (let i = 0; i < 7; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Only one incident despite 7 heartbeats
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("stays up when failures are below threshold", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// 2 failures: window = [true, true, true, false, false] = 40% < 60%
|
||||
h.setNextResponse(false, 500);
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// 3 passes: window = [false, false, true, true, true] = 40% < 60%
|
||||
h.setNextResponse(true, 200);
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,129 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
import type { MaintenanceWindow } from "../../src/types/maintenanceWindow.ts";
|
||||
|
||||
const makeMaintenanceWindow = (overrides?: Partial<MaintenanceWindow>): MaintenanceWindow => {
|
||||
const now = new Date();
|
||||
const start = new Date(now.getTime() - 60 * 60 * 1000); // 1 hour ago
|
||||
const end = new Date(now.getTime() + 60 * 60 * 1000); // 1 hour from now
|
||||
return {
|
||||
id: "mw-1",
|
||||
monitorId: "mon-1",
|
||||
teamId: "team-1",
|
||||
active: true,
|
||||
name: "Test Maintenance",
|
||||
duration: 2,
|
||||
durationUnit: "hours",
|
||||
repeat: 0,
|
||||
start: start.toISOString(),
|
||||
end: end.toISOString(),
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
};
|
||||
};
|
||||
|
||||
describe("Heartbeat job: maintenance windows", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("skips checks and sets monitor to maintenance when window is active", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Configure an active maintenance window covering the current time
|
||||
h.maintenanceWindowsRepo.findByMonitorId.mockResolvedValue([makeMaintenanceWindow()]);
|
||||
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// Monitor status should be set to maintenance
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("maintenance");
|
||||
|
||||
// No network request should have been made
|
||||
expect(h.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
|
||||
// No check should have been buffered
|
||||
expect(h.bufferStub.addToBuffer).not.toHaveBeenCalled();
|
||||
|
||||
// No incident should have been created
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
|
||||
// No notification should have been sent
|
||||
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not update status if monitor is already in maintenance", async () => {
|
||||
const monitor = makeMonitor({ status: "maintenance" });
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.maintenanceWindowsRepo.findByMonitorId.mockResolvedValue([makeMaintenanceWindow()]);
|
||||
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// findById reads the stored monitor; updateById should not have been called
|
||||
// since status was already "maintenance"
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("maintenance");
|
||||
expect(h.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resumes normal checks when maintenance window expires", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// First heartbeat: inside maintenance window
|
||||
h.maintenanceWindowsRepo.findByMonitorId.mockResolvedValue([makeMaintenanceWindow()]);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
let storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("maintenance");
|
||||
|
||||
// Second heartbeat: maintenance window expired (no windows returned)
|
||||
h.maintenanceWindowsRepo.findByMonitorId.mockResolvedValue([]);
|
||||
h.setNextResponse(true, 200);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// Network request should have been made this time
|
||||
expect(h.networkService.requestStatus).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Check should have been buffered
|
||||
expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("skips checks when window is inactive (active: false)", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Inactive maintenance window — should be ignored
|
||||
h.maintenanceWindowsRepo.findByMonitorId.mockResolvedValue([makeMaintenanceWindow({ active: false })]);
|
||||
|
||||
h.setNextResponse(true, 200);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// Normal check should proceed since window is inactive
|
||||
expect(h.networkService.requestStatus).toHaveBeenCalledTimes(1);
|
||||
expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("skips checks when window is in the past (expired)", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Window ended an hour ago
|
||||
const pastEnd = new Date(Date.now() - 60 * 60 * 1000);
|
||||
const pastStart = new Date(pastEnd.getTime() - 2 * 60 * 60 * 1000);
|
||||
h.maintenanceWindowsRepo.findByMonitorId.mockResolvedValue([
|
||||
makeMaintenanceWindow({ start: pastStart.toISOString(), end: pastEnd.toISOString() }),
|
||||
]);
|
||||
|
||||
h.setNextResponse(true, 200);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// Window is in the past — normal check should proceed
|
||||
expect(h.networkService.requestStatus).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
|
||||
describe("Heartbeat job: notification failure isolation", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("creates incident even when notification dispatch throws", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Make notifications throw
|
||||
h.notificationsService.handleNotifications.mockRejectedValue(new Error("SMTP connection refused"));
|
||||
|
||||
// Drive monitor down
|
||||
h.setNextResponse(false, 503);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Incident should still have been created
|
||||
const incidents = h.incidentsRepo.getAll();
|
||||
expect(incidents).toHaveLength(1);
|
||||
expect(incidents[0].status).toBe(true);
|
||||
expect(incidents[0].statusCode).toBe(503);
|
||||
|
||||
// Monitor should still be marked down
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("down");
|
||||
});
|
||||
|
||||
it("auto-resolves incident even when recovery notification throws", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Drive monitor down (notifications succeed here)
|
||||
h.setNextResponse(false, 503);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
expect(h.incidentsRepo.getAll()[0].status).toBe(true);
|
||||
|
||||
// Now make notifications throw for the recovery
|
||||
h.notificationsService.handleNotifications.mockRejectedValue(new Error("Slack API timeout"));
|
||||
|
||||
// Recover
|
||||
h.setNextResponse(true, 200);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Incident should still be resolved despite notification failure
|
||||
const incidents = h.incidentsRepo.getAll();
|
||||
expect(incidents).toHaveLength(1);
|
||||
expect(incidents[0].status).toBe(false);
|
||||
expect(incidents[0].resolutionType).toBe("automatic");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
|
||||
describe("Heartbeat job: recovery", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("recovers monitor and auto-resolves incident when checks pass", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// 3 failures to trigger down (60% threshold)
|
||||
h.setNextResponse(false, 503);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.incidentsRepo.getAll()[0].status).toBe(true);
|
||||
|
||||
// Send passing checks to push failure rate below threshold
|
||||
// Window after 3 fails: [false, false, false, false, false] (prior trues shifted out)
|
||||
// Need enough passes to drop below 60%
|
||||
h.setNextResponse(true, 200);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Monitor should be back up
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
|
||||
// Incident should be auto-resolved
|
||||
const incidents = h.incidentsRepo.getAll();
|
||||
expect(incidents).toHaveLength(1);
|
||||
expect(incidents[0].status).toBe(false);
|
||||
expect(incidents[0].resolutionType).toBe("automatic");
|
||||
expect(incidents[0].endTime).not.toBeNull();
|
||||
});
|
||||
|
||||
it("creates a new incident after recovery if monitor goes down again", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// First outage: 3 failures to trigger (window shifts from all-true)
|
||||
h.setNextResponse(false, 503);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Recovery: enough passes to drop below threshold
|
||||
h.setNextResponse(true, 200);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Second outage: 3 more failures to trigger again
|
||||
h.setNextResponse(false, 502);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
const incidents = h.incidentsRepo.getAll();
|
||||
expect(incidents).toHaveLength(2);
|
||||
|
||||
// First incident resolved
|
||||
expect(incidents[0].status).toBe(false);
|
||||
expect(incidents[0].resolutionType).toBe("automatic");
|
||||
expect(incidents[0].statusCode).toBe(503);
|
||||
|
||||
// Second incident active
|
||||
expect(incidents[1].status).toBe(true);
|
||||
expect(incidents[1].statusCode).toBe(502);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,153 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||
import type { HardwareStatusPayload } from "../../src/types/network.ts";
|
||||
|
||||
const makeHardwareResponse = (cpuUsage: number): MonitorStatusResponse<HardwareStatusPayload> => ({
|
||||
monitorId: "mon-1",
|
||||
teamId: "team-1",
|
||||
type: "hardware",
|
||||
status: true,
|
||||
code: 200,
|
||||
message: "OK",
|
||||
responseTime: 50,
|
||||
payload: {
|
||||
data: {
|
||||
cpu: { usage_percent: cpuUsage, temperature: [] },
|
||||
memory: { usage_percent: 0.3 },
|
||||
disk: [{ usage_percent: 0.2 }],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
describe("Heartbeat job: hardware threshold breach", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("transitions to breached and creates incident after alert counter hits zero", async () => {
|
||||
// Hardware monitor with CPU threshold at 80% (stored as 80, compared as 80/100 = 0.8)
|
||||
// Alert counters start at 5, decrement each breach, trigger at 0
|
||||
const monitor = makeMonitor({
|
||||
type: "hardware",
|
||||
status: "up",
|
||||
cpuAlertThreshold: 80,
|
||||
cpuAlertCounter: 5,
|
||||
memoryAlertThreshold: 100,
|
||||
memoryAlertCounter: 5,
|
||||
diskAlertThreshold: 100,
|
||||
diskAlertCounter: 5,
|
||||
tempAlertThreshold: 100,
|
||||
tempAlertCounter: 5,
|
||||
selectedDisks: [],
|
||||
});
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// CPU at 90% (0.9) exceeds threshold of 80% (0.8)
|
||||
h.setNextResponseFull(makeHardwareResponse(0.9));
|
||||
|
||||
// Configure message builder for when incident is created
|
||||
h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]);
|
||||
|
||||
// 4 heartbeats: counter goes 5→4→3→2→1, status still "up"
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
let storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
expect(storedMonitor.cpuAlertCounter).toBe(1);
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
|
||||
// 5th heartbeat: counter hits 0, status transitions to "breached"
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("breached");
|
||||
expect(storedMonitor.cpuAlertCounter).toBe(0);
|
||||
|
||||
// Incident created with statusCode 9999 and threshold message
|
||||
const incidents = h.incidentsRepo.getAll();
|
||||
expect(incidents).toHaveLength(1);
|
||||
expect(incidents[0].statusCode).toBe(9999);
|
||||
expect(incidents[0].message).toBe("CPU: 90% (threshold: 80%)");
|
||||
expect(incidents[0].status).toBe(true);
|
||||
|
||||
// Notification should have been triggered
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("recovers from breached to up when metrics return to normal", async () => {
|
||||
// Start with a monitor already in breached state with counter at 0
|
||||
const monitor = makeMonitor({
|
||||
type: "hardware",
|
||||
status: "breached",
|
||||
cpuAlertThreshold: 80,
|
||||
cpuAlertCounter: 0,
|
||||
memoryAlertThreshold: 100,
|
||||
memoryAlertCounter: 5,
|
||||
diskAlertThreshold: 100,
|
||||
diskAlertCounter: 5,
|
||||
tempAlertThreshold: 100,
|
||||
tempAlertCounter: 5,
|
||||
selectedDisks: [],
|
||||
});
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Also seed an active incident for this monitor
|
||||
await h.incidentsRepo.create({
|
||||
monitorId: "mon-1",
|
||||
teamId: "team-1",
|
||||
status: true,
|
||||
statusCode: 9999,
|
||||
message: "CPU: 90% (threshold: 80%)",
|
||||
});
|
||||
|
||||
// CPU at 50% (0.5) — below threshold of 80% (0.8)
|
||||
h.setNextResponseFull(makeHardwareResponse(0.5));
|
||||
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
// Monitor should recover to "up"
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
// Counter resets to 5 when not breaching
|
||||
expect(storedMonitor.cpuAlertCounter).toBe(5);
|
||||
|
||||
// Incident should be auto-resolved
|
||||
const incidents = h.incidentsRepo.getAll();
|
||||
expect(incidents).toHaveLength(1);
|
||||
expect(incidents[0].status).toBe(false);
|
||||
expect(incidents[0].resolutionType).toBe("automatic");
|
||||
});
|
||||
|
||||
it("does not create duplicate incidents while remaining breached", async () => {
|
||||
const monitor = makeMonitor({
|
||||
type: "hardware",
|
||||
status: "up",
|
||||
cpuAlertThreshold: 80,
|
||||
cpuAlertCounter: 5,
|
||||
memoryAlertThreshold: 100,
|
||||
memoryAlertCounter: 5,
|
||||
diskAlertThreshold: 100,
|
||||
diskAlertCounter: 5,
|
||||
tempAlertThreshold: 100,
|
||||
tempAlertCounter: 5,
|
||||
selectedDisks: [],
|
||||
});
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setNextResponseFull(makeHardwareResponse(0.9));
|
||||
h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]);
|
||||
|
||||
// 5 heartbeats to trigger breach, then 5 more while still breaching
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Only one incident despite 10 heartbeats
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,216 @@
|
||||
import { describe, expect, it, jest, beforeEach } from "@jest/globals";
|
||||
import { IncidentService } from "../../src/service/business/incidentService.ts";
|
||||
import { InMemoryIncidentsRepository } from "../helpers/InMemoryIncidentsRepository.ts";
|
||||
import { createMockLogger } from "../helpers/createMockLogger.ts";
|
||||
import type { IMonitorsRepository, IUsersRepository } from "../../src/repositories/index.ts";
|
||||
import type { INotificationMessageBuilder } from "../../src/service/infrastructure/notificationMessageBuilder.ts";
|
||||
import type { Monitor } from "../../src/types/monitor.ts";
|
||||
import type { MonitorActionDecision } from "../../src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "mon-1",
|
||||
teamId: "team-1",
|
||||
name: "Test Monitor",
|
||||
type: "http",
|
||||
url: "https://example.com",
|
||||
status: "down",
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
const makeDecision = (overrides?: Partial<MonitorActionDecision>): MonitorActionDecision => ({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// ── Test suite ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("Incident lifecycle (integration)", () => {
|
||||
let repo: InMemoryIncidentsRepository;
|
||||
let service: IncidentService;
|
||||
let monitorsRepo: jest.Mocked<IMonitorsRepository>;
|
||||
let usersRepo: jest.Mocked<IUsersRepository>;
|
||||
let messageBuilder: jest.Mocked<INotificationMessageBuilder>;
|
||||
|
||||
beforeEach(() => {
|
||||
repo = new InMemoryIncidentsRepository();
|
||||
monitorsRepo = { findById: jest.fn() } as unknown as jest.Mocked<IMonitorsRepository>;
|
||||
usersRepo = { findById: jest.fn() } as unknown as jest.Mocked<IUsersRepository>;
|
||||
messageBuilder = { extractThresholdBreaches: jest.fn() } as unknown as jest.Mocked<INotificationMessageBuilder>;
|
||||
service = new IncidentService(createMockLogger() as any, repo, monitorsRepo, usersRepo, messageBuilder);
|
||||
});
|
||||
|
||||
// ── Creation ─────────────────────────────────────────────────────────────
|
||||
|
||||
it("creates an incident when monitor goes down", async () => {
|
||||
const monitor = makeMonitor({ status: "down" });
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
|
||||
|
||||
const incident = await service.handleIncident(monitor, 503, decision);
|
||||
|
||||
expect(incident).not.toBeNull();
|
||||
expect(incident!.monitorId).toBe("mon-1");
|
||||
expect(incident!.teamId).toBe("team-1");
|
||||
expect(incident!.status).toBe(true);
|
||||
expect(incident!.statusCode).toBe(503);
|
||||
expect(incident!.endTime).toBeNull();
|
||||
expect(incident!.resolutionType).toBeNull();
|
||||
|
||||
const stored = repo.getAll();
|
||||
expect(stored).toHaveLength(1);
|
||||
expect(stored[0].id).toBe(incident!.id);
|
||||
});
|
||||
|
||||
// ── Idempotency ──────────────────────────────────────────────────────────
|
||||
|
||||
it("does not create a duplicate incident for the same monitor", async () => {
|
||||
const monitor = makeMonitor({ status: "down" });
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
|
||||
|
||||
const first = await service.handleIncident(monitor, 503, decision);
|
||||
const second = await service.handleIncident(monitor, 503, decision);
|
||||
|
||||
expect(second!.id).toBe(first!.id);
|
||||
expect(repo.getAll()).toHaveLength(1);
|
||||
});
|
||||
|
||||
// ── Auto-resolve ─────────────────────────────────────────────────────────
|
||||
|
||||
it("auto-resolves an incident when monitor recovers", async () => {
|
||||
const monitor = makeMonitor({ status: "down" });
|
||||
const createDecision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
|
||||
const created = await service.handleIncident(monitor, 503, createDecision);
|
||||
|
||||
const resolveDecision = makeDecision({ shouldResolveIncident: true });
|
||||
const resolved = await service.handleIncident(monitor, 200, resolveDecision);
|
||||
|
||||
expect(resolved).not.toBeNull();
|
||||
expect(resolved!.id).toBe(created!.id);
|
||||
expect(resolved!.status).toBe(false);
|
||||
expect(resolved!.endTime).toBeDefined();
|
||||
expect(resolved!.resolutionType).toBe("automatic");
|
||||
});
|
||||
|
||||
// ── Resolve with nothing active ──────────────────────────────────────────
|
||||
|
||||
it("returns null when resolving with no active incident", async () => {
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
const decision = makeDecision({ shouldResolveIncident: true });
|
||||
|
||||
const result = await service.handleIncident(monitor, 200, decision);
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
// ── Manual resolution ────────────────────────────────────────────────────
|
||||
|
||||
it("manually resolves an active incident with comment", async () => {
|
||||
const monitor = makeMonitor({ status: "down" });
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
|
||||
const created = await service.handleIncident(monitor, 500, decision);
|
||||
|
||||
const resolved = await service.resolveIncident(created!.id, "user-1", "team-1", "Root cause identified", "user@test.com");
|
||||
|
||||
expect(resolved.status).toBe(false);
|
||||
expect(resolved.resolutionType).toBe("manual");
|
||||
expect(resolved.resolvedBy).toBe("user-1");
|
||||
expect(resolved.resolvedByEmail).toBe("user@test.com");
|
||||
expect(resolved.comment).toBe("Root cause identified");
|
||||
expect(resolved.endTime).toBeDefined();
|
||||
});
|
||||
|
||||
it("throws when manually resolving an already-resolved incident", async () => {
|
||||
const monitor = makeMonitor({ status: "down" });
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
|
||||
const created = await service.handleIncident(monitor, 500, decision);
|
||||
|
||||
await service.resolveIncident(created!.id, "user-1", "team-1");
|
||||
|
||||
await expect(service.resolveIncident(created!.id, "user-1", "team-1")).rejects.toThrow("Incident not found");
|
||||
});
|
||||
|
||||
// ── Threshold breach ─────────────────────────────────────────────────────
|
||||
|
||||
it("creates a threshold breach incident with statusCode 9999 and descriptive message", async () => {
|
||||
const monitor = makeMonitor({ status: "breached", type: "hardware" });
|
||||
(messageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([
|
||||
{ metric: "cpu", formattedValue: "92%", threshold: 80, unit: "%" },
|
||||
{ metric: "memory", formattedValue: "88%", threshold: 85, unit: "%" },
|
||||
]);
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
|
||||
|
||||
const incident = await service.handleIncident(monitor, 200, decision, { monitorId: "mon-1" } as any);
|
||||
|
||||
expect(incident!.statusCode).toBe(9999);
|
||||
expect(incident!.message).toBe("CPU: 92% (threshold: 80%), MEMORY: 88% (threshold: 85%)");
|
||||
});
|
||||
|
||||
// ── No action ────────────────────────────────────────────────────────────
|
||||
|
||||
it("returns null when no action is needed", async () => {
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
const decision = makeDecision();
|
||||
|
||||
const result = await service.handleIncident(monitor, 200, decision);
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(repo.getAll()).toHaveLength(0);
|
||||
});
|
||||
|
||||
// ── Full lifecycle ───────────────────────────────────────────────────────
|
||||
|
||||
it("handles a complete lifecycle: create -> manual resolve -> new incident -> auto-resolve", async () => {
|
||||
const monitor = makeMonitor({ status: "down" });
|
||||
|
||||
// First outage
|
||||
const first = await service.handleIncident(monitor, 503, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }));
|
||||
expect(first!.status).toBe(true);
|
||||
|
||||
// Manually resolved
|
||||
await service.resolveIncident(first!.id, "user-1", "team-1", "Restarted server");
|
||||
|
||||
// Second outage — new incident since previous was resolved
|
||||
const second = await service.handleIncident(monitor, 502, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }));
|
||||
expect(second!.id).not.toBe(first!.id);
|
||||
expect(second!.statusCode).toBe(502);
|
||||
|
||||
// Auto-resolve
|
||||
const resolved = await service.handleIncident(monitor, 200, makeDecision({ shouldResolveIncident: true }));
|
||||
expect(resolved!.id).toBe(second!.id);
|
||||
expect(resolved!.status).toBe(false);
|
||||
expect(resolved!.resolutionType).toBe("automatic");
|
||||
|
||||
// Both incidents stored, both resolved
|
||||
const all = repo.getAll();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all.every((i) => i.status === false)).toBe(true);
|
||||
});
|
||||
|
||||
// ── Cross-monitor isolation ──────────────────────────────────────────────
|
||||
|
||||
it("incidents for different monitors do not interfere", async () => {
|
||||
const monitorA = makeMonitor({ id: "mon-a", status: "down" });
|
||||
const monitorB = makeMonitor({ id: "mon-b", status: "down" });
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
|
||||
|
||||
const incidentA = await service.handleIncident(monitorA, 500, decision);
|
||||
const incidentB = await service.handleIncident(monitorB, 502, decision);
|
||||
|
||||
expect(incidentA!.id).not.toBe(incidentB!.id);
|
||||
expect(repo.getAll()).toHaveLength(2);
|
||||
|
||||
// Resolving monitor A does not affect monitor B
|
||||
const resolved = await service.handleIncident(monitorA, 200, makeDecision({ shouldResolveIncident: true }));
|
||||
expect(resolved!.id).toBe(incidentA!.id);
|
||||
|
||||
const bStillActive = await repo.findActiveByMonitorId("mon-b", "team-1");
|
||||
expect(bStillActive).not.toBeNull();
|
||||
expect(bStillActive!.status).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user