1459 lines
65 KiB
TypeScript
1459 lines
65 KiB
TypeScript
import { describe, expect, it, jest, beforeEach } from "@jest/globals";
|
|
import { StatusService } from "../../../src/service/statusService.ts";
|
|
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
|
import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts";
|
|
import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts";
|
|
import type { Check } from "../../../src/domain/checks/check.type.ts";
|
|
import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts";
|
|
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
|
import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts";
|
|
|
|
// ── Helpers ──────────────────────────────────────────────────────────────────
|
|
|
|
const createMonitorsRepo = () => {
|
|
const findById = jest.fn();
|
|
const updateStatusWindowAndChecks = jest
|
|
.fn()
|
|
.mockImplementation(
|
|
async (
|
|
_id: unknown,
|
|
_tid: unknown,
|
|
status: boolean,
|
|
checkSnapshot: unknown,
|
|
windowSize: number,
|
|
maxRecentChecks: number,
|
|
statusPatch?: Partial<Monitor>
|
|
) => {
|
|
// updateMonitorStatus no longer reads the monitor itself — tests seed it via
|
|
// findById.mockResolvedValue, so the mock resolves it here as the post-write monitor.
|
|
const m: any = await findById();
|
|
m.statusWindow = m.statusWindow || [];
|
|
m.statusWindow.push(status);
|
|
while (m.statusWindow.length > windowSize) {
|
|
m.statusWindow.shift();
|
|
}
|
|
m.recentChecks = m.recentChecks || [];
|
|
m.recentChecks.push(checkSnapshot);
|
|
while (m.recentChecks.length > maxRecentChecks) {
|
|
m.recentChecks.shift();
|
|
}
|
|
if (statusPatch) {
|
|
Object.assign(m, statusPatch);
|
|
}
|
|
return { ...m };
|
|
}
|
|
);
|
|
return {
|
|
findById,
|
|
updateById: jest.fn(),
|
|
updateStatusWindowAndChecks,
|
|
} as unknown as jest.Mocked<IMonitorsRepository>;
|
|
};
|
|
|
|
const createMonitorStatsRepo = () =>
|
|
({
|
|
findByMonitorId: jest.fn(),
|
|
create: jest.fn(),
|
|
updateByMonitorId: jest.fn(),
|
|
}) as unknown as jest.Mocked<IMonitorStatsRepository>;
|
|
|
|
const createService = (overrides?: {
|
|
logger?: ReturnType<typeof createMockLogger>;
|
|
monitorsRepository?: ReturnType<typeof createMonitorsRepo>;
|
|
monitorStatsRepository?: ReturnType<typeof createMonitorStatsRepo>;
|
|
}) => {
|
|
const logger = overrides?.logger ?? createMockLogger();
|
|
const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo();
|
|
const monitorStatsRepository = overrides?.monitorStatsRepository ?? createMonitorStatsRepo();
|
|
|
|
const service = new StatusService(logger as any, monitorsRepository, monitorStatsRepository);
|
|
return { service, logger, monitorsRepository, monitorStatsRepository };
|
|
};
|
|
|
|
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
|
({
|
|
id: "mon-1",
|
|
userId: "user-1",
|
|
teamId: "team-1",
|
|
name: "Test Monitor",
|
|
type: "http",
|
|
url: "https://example.com",
|
|
isActive: true,
|
|
interval: 60000,
|
|
status: "up",
|
|
statusWindow: [],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
recentChecks: [],
|
|
cpuAlertThreshold: 80,
|
|
memoryAlertThreshold: 80,
|
|
diskAlertThreshold: 80,
|
|
tempAlertThreshold: 80,
|
|
cpuAlertCounter: 5,
|
|
memoryAlertCounter: 5,
|
|
diskAlertCounter: 5,
|
|
tempAlertCounter: 5,
|
|
createdAt: "2026-01-01T00:00:00Z",
|
|
updatedAt: "2026-01-01T00:00:00Z",
|
|
...overrides,
|
|
}) as Monitor;
|
|
|
|
const makeCheck = (overrides?: Partial<Check>): Check =>
|
|
({
|
|
id: "check-1",
|
|
metadata: { monitorId: "mon-1", teamId: "team-1" },
|
|
status: true,
|
|
responseTime: 100,
|
|
statusCode: 200,
|
|
message: "OK",
|
|
createdAt: "2026-01-01T00:00:00Z",
|
|
updatedAt: "2026-01-01T00:00:00Z",
|
|
...overrides,
|
|
}) as Check;
|
|
|
|
const makeExistingStats = (overrides?: Record<string, unknown>) => ({
|
|
id: "stats-1",
|
|
monitorId: "mon-1",
|
|
avgResponseTime: 100,
|
|
maxResponseTime: 200,
|
|
totalChecks: 10,
|
|
totalUpChecks: 9,
|
|
totalDownChecks: 1,
|
|
uptimePercentage: 0.9,
|
|
lastResponseTime: 90,
|
|
lastCheckTimestamp: 1000,
|
|
timeOfLastFailure: 500,
|
|
createdAt: "2026-01-01T00:00:00Z",
|
|
updatedAt: "2026-01-01T00:00:00Z",
|
|
...overrides,
|
|
});
|
|
|
|
// ── Tests ────────────────────────────────────────────────────────────────────
|
|
|
|
describe("StatusService", () => {
|
|
// ── running stats (via updateMonitorStatus) ──────────────────────────────
|
|
|
|
describe("running stats", () => {
|
|
// The service delegates the running-stats computation to the repository's atomic `updateByMonitorId`,
|
|
// so these cases only verify what is forwarded and that a failure never blocks the status update.
|
|
// The math (running average, uptimePercentage, timeOfLastFailure) is exercised by the repository tests.
|
|
|
|
const run = async (check: Check) => {
|
|
const monitor = makeMonitor();
|
|
const deps = createService();
|
|
(deps.monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(deps.monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({});
|
|
const result = await deps.service.updateMonitorStatus(check, monitor);
|
|
return { ...deps, result };
|
|
};
|
|
|
|
it("forwards a successful check to updateByMonitorId", async () => {
|
|
const { monitorStatsRepository } = await run(makeCheck({ responseTime: 50 }));
|
|
|
|
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith(
|
|
"mon-1",
|
|
expect.objectContaining({ status: true, responseTime: 50, now: expect.any(Number) })
|
|
);
|
|
});
|
|
|
|
it("forwards a failed check to updateByMonitorId", async () => {
|
|
const { monitorStatsRepository } = await run(makeCheck({ status: false, responseTime: 100 }));
|
|
|
|
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false, responseTime: 100 }));
|
|
});
|
|
|
|
it("passes 0 when responseTime is undefined", async () => {
|
|
const { monitorStatsRepository } = await run(makeCheck({ responseTime: undefined as any }));
|
|
|
|
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 }));
|
|
});
|
|
|
|
it("forwards responseTime of 0 as 0 (falsy but defined)", async () => {
|
|
const { monitorStatsRepository } = await run(makeCheck({ responseTime: 0 }));
|
|
|
|
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 }));
|
|
});
|
|
|
|
it("coerces a non-true status to false (e.g. undefined)", async () => {
|
|
const { monitorStatsRepository } = await run(makeCheck({ status: undefined as any }));
|
|
|
|
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false }));
|
|
});
|
|
|
|
it("warns once and still updates the status when updateByMonitorId throws", async () => {
|
|
const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" });
|
|
const { service, logger, monitorsRepository, monitorStatsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down"));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.monitor.status).toBe("up");
|
|
expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledTimes(1);
|
|
expect(logger.warn).toHaveBeenCalledTimes(1);
|
|
expect(logger.warn).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
service: "StatusService",
|
|
method: "tryUpdateRunningStats",
|
|
message: "stats db down",
|
|
stack: expect.any(String),
|
|
})
|
|
);
|
|
expect(logger.error).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("warns with 'Unknown error' for non-Error rejections", async () => {
|
|
const monitor = makeMonitor();
|
|
const { service, logger, monitorsRepository, monitorStatsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue("string error");
|
|
|
|
await service.updateMonitorStatus(makeCheck(), monitor);
|
|
|
|
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
|
|
});
|
|
});
|
|
|
|
// ── updateMonitorStatus ──────────────────────────────────────────────────
|
|
|
|
describe("updateMonitorStatus", () => {
|
|
it("returns early with no status change when statusWindow is not full", async () => {
|
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.prevStatus).toBe("up");
|
|
});
|
|
|
|
it("uses the prefetched monitor and skips findById when one is passed", async () => {
|
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor);
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(monitorsRepository.findById).not.toHaveBeenCalled();
|
|
expect(result.prevStatus).toBe("up");
|
|
expect(result.monitor).toBe(monitor);
|
|
});
|
|
|
|
it("pushes to statusWindow and trims to statusWindowSize", async () => {
|
|
const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
// Atomic push should have been called with the correct status and window size
|
|
expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith(
|
|
"mon-1",
|
|
"team-1",
|
|
false,
|
|
expect.any(Object),
|
|
5,
|
|
MAX_RECENT_CHECKS,
|
|
expect.objectContaining({ status: expect.any(String) })
|
|
);
|
|
});
|
|
|
|
it("pushes check snapshot to recentChecks and trims to MAX_RECENT_CHECKS", async () => {
|
|
const existingChecks = Array.from({ length: MAX_RECENT_CHECKS }, (_, i) => ({ id: `old-${i}` }));
|
|
const monitor = makeMonitor({ recentChecks: existingChecks as any, statusWindow: [], statusWindowSize: 5 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
await service.updateMonitorStatus(makeCheck({ id: "new-check" }), monitor);
|
|
|
|
expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS);
|
|
expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" }));
|
|
});
|
|
|
|
it("marks status as down when failure threshold is met", async () => {
|
|
// 5/5 failures = 100% >= 80% threshold
|
|
const monitor = makeMonitor({
|
|
statusWindow: [false, false, false, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "up",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("recovers to up when failure rate drops below threshold", async () => {
|
|
// 1/5 failures = 20% < 80% threshold, and monitor was down
|
|
const monitor = makeMonitor({
|
|
statusWindow: [true, true, true, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "down",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("does not change status when already up and below threshold", async () => {
|
|
const monitor = makeMonitor({
|
|
statusWindow: [true, true, true, true],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "up",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("keeps status 'up' on a sub-threshold failed check (regression: #3438)", async () => {
|
|
// Monitor is up, window has one failure already, incoming check fails.
|
|
// 2/5 = 40% < 80% threshold — must NOT silently write status='down',
|
|
// otherwise the next successful check fires a spurious 'Recovered' notification.
|
|
const monitor = makeMonitor({
|
|
statusWindow: [true, true, true, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "up",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("does not change status when already down and still above threshold", async () => {
|
|
const monitor = makeMonitor({
|
|
statusWindow: [false, false, false, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "down",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("initializes statusWindow when undefined", async () => {
|
|
const monitor = makeMonitor({ statusWindow: undefined as any, statusWindowSize: 5 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
await service.updateMonitorStatus(makeCheck(), monitor);
|
|
|
|
expect(monitor.statusWindow).toEqual([true]);
|
|
});
|
|
|
|
it("initializes recentChecks when undefined", async () => {
|
|
const monitor = makeMonitor({ recentChecks: undefined as any, statusWindowSize: 5 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
await service.updateMonitorStatus(makeCheck(), monitor);
|
|
|
|
expect(monitor.recentChecks).toHaveLength(1);
|
|
});
|
|
|
|
it("throws AppError when repository throws", async () => {
|
|
const monitor = makeMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error"));
|
|
|
|
await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Failed to update monitor");
|
|
});
|
|
|
|
it("throws AppError with 'Unknown error' for non-Error exceptions", async () => {
|
|
const monitor = makeMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error");
|
|
|
|
await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Unknown error");
|
|
});
|
|
|
|
it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => {
|
|
// Monitor is down, window is [false,false,false,false]. A single successful check makes it
|
|
// [false,false,false,false,true] → 4/5 = 80%, still at threshold, must NOT silently recover.
|
|
// Pre-fix, newStatus was seeded "up" from the raw check and fell through to be persisted,
|
|
// causing the next failing check to fire a spurious second 'Down' notification.
|
|
const monitor = makeMonitor({
|
|
statusWindow: [false, false, false, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "down",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("triggers 'down' transition exactly at the failure threshold boundary", async () => {
|
|
// 4/5 = 80%, threshold is 80, comparison is >= so this must trip.
|
|
const monitor = makeMonitor({
|
|
statusWindow: [false, false, false, true],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "up",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("resumes from 'initializing' to 'up' on a passing check with a full window (regression: pause/resume)", async () => {
|
|
// When a monitor is paused and resumed, togglePauseById sets status='initializing'
|
|
// but does NOT reset statusWindow. On the next check the window is already full,
|
|
// so the warmup early-return is skipped. Pre-fix, newStatus was seeded from
|
|
// monitor.status ('initializing') and computeReachability never transitioned
|
|
// out of 'initializing' on a passing window, so the monitor was stuck forever.
|
|
const monitor = makeMonitor({
|
|
statusWindow: [true, true, true, true, true],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "initializing",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("resumes from 'initializing' to 'down' on a failing check with a full window (regression: pause/resume mirror)", async () => {
|
|
// Mirror of the pause/resume regression with a failing check: should flip to 'down'
|
|
// immediately rather than stay stuck in 'initializing'.
|
|
const monitor = makeMonitor({
|
|
statusWindow: [false, false, false, false, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "initializing",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("resolves a legacy boolean status ('true') to 'up' on a passing check (regression: pre-upgrade monitors stuck forever)", async () => {
|
|
// Old Checkmate versions stored monitor.status as a boolean. Pre-fix, a monitor
|
|
// carrying status='true' never matched the initializing override and
|
|
// computeReachability only transitions to 'up' from 'down', so the legacy value
|
|
// was written back unchanged on every evaluation — stuck forever while the UI
|
|
// rendered it as 'initializing'.
|
|
const monitor = makeMonitor({
|
|
statusWindow: [true, true, true, true, true],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "true" as MonitorStatus,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.monitor.status).toBe("up");
|
|
expect(result.statusChanged).toBe(false); // was effectively up already — no notification storm
|
|
});
|
|
|
|
it("resolves a legacy boolean status ('false') to 'down' on a failing check (regression mirror)", async () => {
|
|
const monitor = makeMonitor({
|
|
statusWindow: [false, false, false, false, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "false" as MonitorStatus,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.monitor.status).toBe("down");
|
|
expect(result.statusChanged).toBe(true); // down must surface so an incident is opened
|
|
});
|
|
|
|
it("during warmup, sub-threshold raw checks no longer flip stored status (regression: down-during-init incident not resolving)", async () => {
|
|
// Once the monitor has left 'initializing', the warmup branch must NOT silently
|
|
// flip status from raw checks. Otherwise a down check during warmup writes
|
|
// status='down' without statusChanged, and the next up check leaks status='up'
|
|
// the same way — by the time the window fills, monitor.status is already 'up'
|
|
// and computeReachability cannot fire down→up to resolve the incident.
|
|
const monitor = makeMonitor({ statusWindow: [true, true], statusWindowSize: 5, status: "up" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
expect(monitor.statusWindow).toEqual([true, true, false]);
|
|
});
|
|
|
|
it("creates incident on initial down check when statusWindowSize is 1 (regression: alert on initial down skips warmup branch)", async () => {
|
|
// statusWindowSize=1 means the very first check skips the warmup branch entirely
|
|
// (1 < 1 is false) and lands in the full-window branch. The 'initializing' override
|
|
// must still surface statusChanged=true on a down result, otherwise computeReachability
|
|
// sees currentStatus='down' and refuses to transition, and no incident is opened.
|
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 1, statusWindowThreshold: 60, status: "initializing" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("down");
|
|
expect(result.prevStatus).toBe("initializing");
|
|
});
|
|
|
|
it("creates incident on initial down check for a brand-new monitor (regression: alert on initial down)", async () => {
|
|
// Brand-new monitor: status='initializing', empty statusWindow. First check is down.
|
|
// Must surface statusChanged=true with monitor.status='down' so the helper opens an incident.
|
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "initializing" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("down");
|
|
expect(result.prevStatus).toBe("initializing");
|
|
});
|
|
|
|
it("leaves 'maintenance' for 'up' on a passing check after the window closes (empty statusWindow)", async () => {
|
|
// On entry to a maintenance window the producer clears statusWindow, so the first
|
|
// check after the window closes sees an empty window and lands in the warmup
|
|
// early-return branch. A passing check must move the monitor out of 'maintenance'
|
|
// to 'up' without flagging a status change (no incident).
|
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
expect(result.prevStatus).toBe("maintenance");
|
|
});
|
|
|
|
it("leaves 'maintenance' for 'down' on a failing check after the window closes (empty statusWindow)", async () => {
|
|
// Mirror of the recovery case: a failing first check after the window closes must
|
|
// flip the monitor straight to 'down' and surface statusChanged so an incident opens,
|
|
// rather than leave it stuck in 'maintenance'.
|
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("down");
|
|
expect(result.prevStatus).toBe("maintenance");
|
|
});
|
|
|
|
it("unsticks from 'maintenance' to 'up' on a passing check even with a full window (regression: stuck in maintenance)", async () => {
|
|
// Guards the up-front override directly: even if a full window survives (e.g. the
|
|
// entry-clear is bypassed), computeReachability only transitions out to 'up' from
|
|
// 'down', so without the override the monitor would stay stuck in 'maintenance'.
|
|
const monitor = makeMonitor({
|
|
statusWindow: [true, true, true, true, true],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "maintenance",
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("does not flip status on subsequent checks during warmup once 'initializing' has been left (regression: down→up incident resolves once window fills)", async () => {
|
|
// After a 'initializing'→'down' transition on check #1, monitor.status is now 'down'
|
|
// and statusWindow has one false. Subsequent up checks during warmup must leave
|
|
// monitor.status='down' so the sliding window can detect the down→up transition
|
|
// once the window fills.
|
|
const monitor = makeMonitor({ statusWindow: [false], statusWindowSize: 5, status: "down" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
// ── Hardware threshold breach tests ──────────────────────────────────
|
|
|
|
describe("hardware threshold breaches", () => {
|
|
const makeHardwareMonitor = (overrides?: Partial<Monitor>) =>
|
|
makeMonitor({
|
|
type: "hardware",
|
|
statusWindow: [true, true, true, true],
|
|
statusWindowSize: 5,
|
|
status: "up",
|
|
cpuAlertThreshold: 80,
|
|
memoryAlertThreshold: 80,
|
|
diskAlertThreshold: 80,
|
|
tempAlertThreshold: 80,
|
|
cpuAlertCounter: 5,
|
|
memoryAlertCounter: 5,
|
|
diskAlertCounter: 5,
|
|
tempAlertCounter: 5,
|
|
...overrides,
|
|
});
|
|
|
|
const makeHardwareCheck = (metrics: Pick<Check, "cpu" | "memory" | "disk" | "host">, overrides?: Partial<Check>) =>
|
|
makeCheck({
|
|
status: true,
|
|
...metrics,
|
|
...overrides,
|
|
});
|
|
|
|
it("detects CPU threshold breach and decrements counter", async () => {
|
|
const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.cpu).toBe(true);
|
|
expect(result.thresholdBreaches?.memory).toBe(false);
|
|
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
|
expect(patch.cpuAlertCounter).toBe(0);
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("detects memory threshold breach", async () => {
|
|
const monitor = makeHardwareMonitor({ memoryAlertCounter: 1 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.memory).toBe(true);
|
|
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
|
expect(patch.memoryAlertCounter).toBe(0);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("detects disk threshold breach", async () => {
|
|
const monitor = makeHardwareMonitor({ diskAlertCounter: 1 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.disk).toBe(true);
|
|
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
|
expect(patch.diskAlertCounter).toBe(0);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("detects temperature threshold breach", async () => {
|
|
const monitor = makeHardwareMonitor({ tempAlertCounter: 1 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.temp).toBe(true);
|
|
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
|
expect(patch.tempAlertCounter).toBe(0);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("resets counters to 5 when thresholds are not breached", async () => {
|
|
const monitor = makeHardwareMonitor({
|
|
cpuAlertCounter: 2,
|
|
memoryAlertCounter: 2,
|
|
diskAlertCounter: 2,
|
|
tempAlertCounter: 2,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({
|
|
cpu: { usage_percent: 0.1, temperature: [30] },
|
|
memory: { usage_percent: 0.1 },
|
|
disk: [{ usage_percent: 0.1 }],
|
|
host: {},
|
|
});
|
|
await service.updateMonitorStatus(check, monitor);
|
|
|
|
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
|
expect(patch.cpuAlertCounter).toBe(5);
|
|
expect(patch.memoryAlertCounter).toBe(5);
|
|
expect(patch.diskAlertCounter).toBe(5);
|
|
expect(patch.tempAlertCounter).toBe(5);
|
|
});
|
|
|
|
it("stays breached without statusChanged when already breached and counter still at 0", async () => {
|
|
const monitor = makeHardwareMonitor({
|
|
status: "breached",
|
|
cpuAlertCounter: 1,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.monitor.status).toBe("breached");
|
|
expect(result.statusChanged).toBe(false);
|
|
});
|
|
|
|
it("recovers from breached to up when all thresholds return to normal", async () => {
|
|
const monitor = makeHardwareMonitor({ status: "breached" });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({
|
|
cpu: { usage_percent: 0.1, temperature: [30] },
|
|
memory: { usage_percent: 0.1 },
|
|
disk: [{ usage_percent: 0.1 }],
|
|
host: {},
|
|
});
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("does not override down status with breached", async () => {
|
|
// Monitor is down due to failure threshold, hardware breaches should not override
|
|
const monitor = makeHardwareMonitor({
|
|
statusWindow: [false, false, false, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
status: "up",
|
|
cpuAlertCounter: 1,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("handles missing cpu usage_percent (returns -1)", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.cpu).toBe(false);
|
|
});
|
|
|
|
it("handles missing memory usage_percent", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.memory).toBe(false);
|
|
});
|
|
|
|
it("handles empty temperature array", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.temp).toBe(false);
|
|
});
|
|
|
|
it("handles undefined temperature", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.temp).toBe(false);
|
|
});
|
|
|
|
it("reports no breaches when the check carries no hardware metrics", async () => {
|
|
// A stored hardware check with no metrics (cpu/memory/disk/host all absent) must not
|
|
// throw and must not report any breach.
|
|
const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck(), monitor);
|
|
|
|
expect(result.thresholdBreaches).toEqual({ cpu: false, memory: false, disk: false, temp: false });
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("does not set thresholdBreaches for non-hardware monitors", async () => {
|
|
const monitor = makeMonitor({
|
|
type: "http",
|
|
statusWindow: [true, true, true, true],
|
|
statusWindowSize: 5,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const result = await service.updateMonitorStatus(makeCheck(), monitor);
|
|
|
|
expect(result.thresholdBreaches).toBeUndefined();
|
|
});
|
|
|
|
it("handles null entry in disk array", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.disk).toBe(false);
|
|
});
|
|
|
|
it("handles disk with undefined usage_percent entries", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.disk).toBe(false);
|
|
});
|
|
|
|
it("handles nullish cpu in metrics", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.cpu).toBe(false);
|
|
expect(result.thresholdBreaches?.temp).toBe(false);
|
|
});
|
|
|
|
it("handles nullish memory in metrics", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.memory).toBe(false);
|
|
});
|
|
|
|
it("handles nullish disk in metrics", async () => {
|
|
const monitor = makeHardwareMonitor();
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.thresholdBreaches?.disk).toBe(false);
|
|
});
|
|
|
|
it("decrements breach counter without changing status while counter > 0", async () => {
|
|
// Counter at 3 → decrements to 2, status must stay 'up' and statusChanged must be false.
|
|
// Only counter→0 may flip to 'breached'.
|
|
const monitor = makeHardwareMonitor({ cpuAlertCounter: 3 });
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
|
expect(patch.cpuAlertCounter).toBe(2);
|
|
expect(result.thresholdBreaches?.cpu).toBe(true);
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("transitions from 'breached' to 'down' when the reachability threshold trips", async () => {
|
|
// A hardware monitor currently in 'breached' state starts failing its reachability
|
|
// checks enough to cross the statusWindow threshold — 'down' must take precedence.
|
|
const monitor = makeHardwareMonitor({
|
|
status: "breached",
|
|
statusWindow: [false, false, false, false],
|
|
statusWindowSize: 5,
|
|
statusWindowThreshold: 80,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false });
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("counters do not go below 0", async () => {
|
|
const monitor = makeHardwareMonitor({
|
|
cpuAlertCounter: 0,
|
|
memoryAlertCounter: 0,
|
|
diskAlertCounter: 0,
|
|
tempAlertCounter: 0,
|
|
});
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
|
|
const check = makeHardwareCheck({
|
|
cpu: { usage_percent: 0.9, temperature: [90] },
|
|
memory: { usage_percent: 0.9 },
|
|
disk: [{ usage_percent: 0.95 }],
|
|
host: {},
|
|
});
|
|
await service.updateMonitorStatus(check, monitor);
|
|
|
|
expect(monitor.cpuAlertCounter).toBe(0);
|
|
expect(monitor.memoryAlertCounter).toBe(0);
|
|
expect(monitor.diskAlertCounter).toBe(0);
|
|
expect(monitor.tempAlertCounter).toBe(0);
|
|
});
|
|
});
|
|
});
|
|
|
|
// ── Pure helpers ─────────────────────────────────────────────────────────
|
|
// These test the private state-machine helpers directly, without repository
|
|
// or logger mocks. The helpers are accessed via `service as any` because
|
|
// they're private implementation details; exposing them publicly would
|
|
// widen the API surface just for testability.
|
|
|
|
describe("docker container alerts", () => {
|
|
const makeDockerMonitor = (overrides?: Partial<Monitor>) =>
|
|
makeMonitor({
|
|
type: "docker",
|
|
url: "unix:///var/run/docker.sock",
|
|
statusWindow: [true, true, true, true],
|
|
statusWindowSize: 5,
|
|
status: "up",
|
|
dockerAlertOnStopped: true,
|
|
dockerAlertOnUnhealthy: true,
|
|
...overrides,
|
|
});
|
|
|
|
const makeContainers = (containers: Partial<DockerContainerInfo>[]): DockerContainerInfo[] =>
|
|
containers.map((container, index) => ({
|
|
id: `container-${index}`,
|
|
name: `container-${index}`,
|
|
image: "nginx:latest",
|
|
state: "running",
|
|
status: "Up 3 hours",
|
|
health: "none",
|
|
...container,
|
|
}));
|
|
|
|
const makeDockerCheck = (containers: DockerContainerInfo[] | null, overrides?: Partial<Check>) =>
|
|
makeCheck({
|
|
status: true,
|
|
...(containers ? { containers } : {}),
|
|
...overrides,
|
|
});
|
|
|
|
const run = async (monitor: Monitor, check: Check) => {
|
|
const { service, monitorsRepository } = createService();
|
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
|
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
|
const result = await service.updateMonitorStatus(check, monitor);
|
|
return { result, monitorsRepository };
|
|
};
|
|
|
|
const stopped = { name: "db", state: "exited" as const, exitCode: 1 };
|
|
const unhealthy = { name: "api", health: "unhealthy" as const };
|
|
|
|
it("goes to breached on the first check with a stopped container", async () => {
|
|
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped])));
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("goes to breached on the first check with an unhealthy container", async () => {
|
|
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([unhealthy])));
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("stays up on a stopped container when both switches are off", async () => {
|
|
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false });
|
|
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped, unhealthy])));
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("stays up on a stopped container when only the unhealthy switch is on", async () => {
|
|
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false });
|
|
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped])));
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("stays up on an exited container with code 0", async () => {
|
|
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "job", state: "exited", exitCode: 0 }])));
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("stays breached without a transition while a container is still down", async () => {
|
|
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([stopped])));
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("stays up without a transition while every container is clean", async () => {
|
|
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "web" }])));
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("recovers from breached to up on the first clean check", async () => {
|
|
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([{ name: "web" }])));
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.prevStatus).toBe("breached");
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("recovers from breached to up when the breaching switch is turned off", async () => {
|
|
const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false });
|
|
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped])));
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("does not override down with breached when the host is unreachable", async () => {
|
|
const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
|
|
const check = makeDockerCheck(null, { status: false, statusCode: 5000 });
|
|
const { result } = await run(monitor, check);
|
|
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("transitions from breached to down when the reachability threshold trips", async () => {
|
|
const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
|
|
const check = makeDockerCheck(null, { status: false, statusCode: 5000 });
|
|
const { result } = await run(monitor, check);
|
|
|
|
expect(result.statusChanged).toBe(true);
|
|
expect(result.monitor.status).toBe("down");
|
|
});
|
|
|
|
it("does not evaluate containers while the status window is short", async () => {
|
|
const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 });
|
|
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped])));
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("up");
|
|
});
|
|
|
|
it("does not evaluate a failed check with a null payload", async () => {
|
|
const monitor = makeDockerMonitor({ status: "breached" });
|
|
const check = makeDockerCheck(null, { status: false, statusCode: 5000 });
|
|
const { result } = await run(monitor, check);
|
|
|
|
expect(result.statusChanged).toBe(false);
|
|
expect(result.monitor.status).toBe("breached");
|
|
});
|
|
|
|
it("does not touch the hardware counters or thresholdBreaches", async () => {
|
|
const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped])));
|
|
|
|
expect(result.thresholdBreaches).toBeUndefined();
|
|
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
|
expect(patch.cpuAlertCounter).toBeUndefined();
|
|
expect(patch.status).toBe("breached");
|
|
});
|
|
});
|
|
|
|
describe("computeReachability (pure)", () => {
|
|
const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => {
|
|
const { service } = createService();
|
|
return (service as any).computeReachability(currentStatus, window, threshold) as {
|
|
nextStatus: "up" | "down";
|
|
transitioned: boolean;
|
|
};
|
|
};
|
|
|
|
// ── Monitor is "up" ──
|
|
it("up + window below threshold → no transition", () => {
|
|
expect(reach("up", [true, true, true, true, true])).toEqual({ nextStatus: "up", transitioned: false });
|
|
});
|
|
|
|
it("up + window exactly at threshold → transitions to down (>= boundary)", () => {
|
|
// 4/5 = 80% and threshold is 80, >= so this must trip
|
|
expect(reach("up", [false, false, false, false, true])).toEqual({ nextStatus: "down", transitioned: true });
|
|
});
|
|
|
|
it("up + window over threshold → transitions to down", () => {
|
|
expect(reach("up", [false, false, false, false, false])).toEqual({ nextStatus: "down", transitioned: true });
|
|
});
|
|
|
|
// ── Monitor is "down" ──
|
|
it("down + window below threshold → transitions to up (recovery)", () => {
|
|
// 0/5 failures, fully healthy window
|
|
expect(reach("down", [true, true, true, true, true])).toEqual({ nextStatus: "up", transitioned: true });
|
|
});
|
|
|
|
it("down + window exactly at threshold → no transition (strict < for recovery)", () => {
|
|
// 4/5 = 80%; recovery requires failureRate < threshold, not <=, so stays down
|
|
const result = reach("down", [false, false, false, false, true]);
|
|
expect(result.transitioned).toBe(false);
|
|
// Orchestrator ignores nextStatus when !transitioned, so the returned value
|
|
// is irrelevant to observable behavior; documenting here for clarity.
|
|
});
|
|
|
|
it("down + window over threshold → no transition", () => {
|
|
const result = reach("down", [false, false, false, false, false]);
|
|
expect(result.transitioned).toBe(false);
|
|
});
|
|
|
|
// ── Monitor is "breached" (hardware state, passes through reachability) ──
|
|
it("breached + healthy window → no transition (reachability does not touch breached)", () => {
|
|
// Critical: reachability must NOT claim to recover a breached monitor. That
|
|
// transition is owned by the hardware block (all thresholds back to normal).
|
|
const result = reach("breached", [true, true, true, true, true]);
|
|
expect(result.transitioned).toBe(false);
|
|
});
|
|
|
|
it("breached + window at threshold → transitions to down (down beats breached)", () => {
|
|
// A breached monitor that also loses reachability must flip to "down".
|
|
// Current status is "breached" which satisfies `!== "down"`, so the branch fires.
|
|
expect(reach("breached", [false, false, false, false, true])).toEqual({ nextStatus: "down", transitioned: true });
|
|
});
|
|
|
|
it("breached + window over threshold → transitions to down", () => {
|
|
expect(reach("breached", [false, false, false, false, false])).toEqual({ nextStatus: "down", transitioned: true });
|
|
});
|
|
|
|
// ── Threshold variations ──
|
|
it("non-default threshold (50%) trips at 3/5 failures", () => {
|
|
expect(reach("up", [false, false, false, true, true], 50)).toEqual({ nextStatus: "down", transitioned: true });
|
|
});
|
|
|
|
it("non-default threshold (50%) does not trip at 2/5 failures", () => {
|
|
expect(reach("up", [false, false, true, true, true], 50).transitioned).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("computeHardwareStatus (pure)", () => {
|
|
type Params = {
|
|
currentStatus: MonitorStatus;
|
|
reachabilityDown: boolean;
|
|
metrics: any;
|
|
thresholds: { cpu: number; memory: number; disk: number; temp: number };
|
|
counters: { cpu: number; memory: number; disk: number; temp: number };
|
|
};
|
|
|
|
const healthyMetrics = () => ({
|
|
cpu: { usage_percent: 0.1, temperature: [30] },
|
|
memory: { usage_percent: 0.1 },
|
|
disk: [{ usage_percent: 0.1 }],
|
|
host: {},
|
|
});
|
|
|
|
const defaults = (): Params => ({
|
|
currentStatus: "up",
|
|
reachabilityDown: false,
|
|
metrics: healthyMetrics(),
|
|
thresholds: { cpu: 80, memory: 80, disk: 80, temp: 80 },
|
|
counters: { cpu: 5, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
|
|
const compute = (overrides: Partial<Params> = {}) => {
|
|
const { service } = createService();
|
|
return (service as any).computeHardwareStatus({ ...defaults(), ...overrides });
|
|
};
|
|
|
|
// ── Breach detection ──
|
|
it("detects CPU breach when usage > threshold", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } } });
|
|
expect(r.breaches.cpu).toBe(true);
|
|
});
|
|
|
|
it("does not flag CPU at exactly the threshold (strict >, not >=)", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.8, temperature: [30] } } });
|
|
expect(r.breaches.cpu).toBe(false);
|
|
});
|
|
|
|
it("does not flag CPU when usage_percent is missing", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), cpu: { temperature: [30] } } });
|
|
expect(r.breaches.cpu).toBe(false);
|
|
});
|
|
|
|
it("detects memory breach", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), memory: { usage_percent: 0.9 } } });
|
|
expect(r.breaches.memory).toBe(true);
|
|
});
|
|
|
|
it("detects disk breach when any disk exceeds threshold", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), disk: [{ usage_percent: 0.1 }, { usage_percent: 0.95 }] } });
|
|
expect(r.breaches.disk).toBe(true);
|
|
});
|
|
|
|
it("skips null disk entries", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), disk: [null, { usage_percent: 0.1 }] } });
|
|
expect(r.breaches.disk).toBe(false);
|
|
});
|
|
|
|
it("skips disk entries missing usage_percent", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), disk: [{ device: "/dev/sda" }] } });
|
|
expect(r.breaches.disk).toBe(false);
|
|
});
|
|
|
|
it("detects temperature breach when any core exceeds threshold", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [30, 50, 90] } } });
|
|
expect(r.breaches.temp).toBe(true);
|
|
});
|
|
|
|
it("empty temperature array → no temp breach", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [] } } });
|
|
expect(r.breaches.temp).toBe(false);
|
|
});
|
|
|
|
it("missing temperature → no temp breach", () => {
|
|
const r = compute({ metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1 } } });
|
|
expect(r.breaches.temp).toBe(false);
|
|
});
|
|
|
|
it("detects multiple simultaneous breaches", () => {
|
|
const r = compute({
|
|
metrics: {
|
|
cpu: { usage_percent: 0.9, temperature: [90] },
|
|
memory: { usage_percent: 0.9 },
|
|
disk: [{ usage_percent: 0.95 }],
|
|
host: {},
|
|
},
|
|
});
|
|
expect(r.breaches).toEqual({ cpu: true, memory: true, disk: true, temp: true });
|
|
});
|
|
|
|
// ── Counter mechanics ──
|
|
it("decrements counter on breach", () => {
|
|
const r = compute({
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 3, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.nextCounters.cpu).toBe(2);
|
|
});
|
|
|
|
it("resets counter to start value when metric returns to normal", () => {
|
|
const r = compute({ counters: { cpu: 2, memory: 2, disk: 2, temp: 2 } });
|
|
expect(r.nextCounters).toEqual({ cpu: 5, memory: 5, disk: 5, temp: 5 });
|
|
});
|
|
|
|
it("counter floors at zero (does not go negative)", () => {
|
|
const r = compute({
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 0, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.nextCounters.cpu).toBe(0);
|
|
});
|
|
|
|
it("counters are independent per metric", () => {
|
|
// CPU breaching, memory normal: CPU decrements, memory resets.
|
|
const r = compute({
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 3, memory: 2, disk: 2, temp: 2 },
|
|
});
|
|
expect(r.nextCounters).toEqual({ cpu: 2, memory: 5, disk: 5, temp: 5 });
|
|
});
|
|
|
|
// ── Status transitions ──
|
|
it("up → breached when counter hits zero on breach", () => {
|
|
const r = compute({
|
|
currentStatus: "up",
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 1, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.nextStatus).toBe("breached");
|
|
expect(r.transitioned).toBe(true);
|
|
});
|
|
|
|
it("up → up (no transition) while counter is still decrementing", () => {
|
|
const r = compute({
|
|
currentStatus: "up",
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 3, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.nextStatus).toBe("up");
|
|
expect(r.transitioned).toBe(false);
|
|
});
|
|
|
|
it("breached → breached (no transition) when already breached and still breaching", () => {
|
|
const r = compute({
|
|
currentStatus: "breached",
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 0, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.transitioned).toBe(false);
|
|
// nextStatus stays at currentStatus ("breached") since no transition fires
|
|
expect(r.nextStatus).toBe("breached");
|
|
});
|
|
|
|
it("breached → up when all metrics return to normal", () => {
|
|
const r = compute({ currentStatus: "breached" });
|
|
expect(r.nextStatus).toBe("up");
|
|
expect(r.transitioned).toBe(true);
|
|
});
|
|
|
|
it("any one counter at zero is enough to trigger initial breach", () => {
|
|
// memory counter at 1, memory currently breaching, cpu fine
|
|
const r = compute({
|
|
currentStatus: "up",
|
|
metrics: { ...healthyMetrics(), memory: { usage_percent: 0.9 } },
|
|
counters: { cpu: 5, memory: 1, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.nextStatus).toBe("breached");
|
|
expect(r.transitioned).toBe(true);
|
|
});
|
|
|
|
// ── Reachability-down gate ──
|
|
it("reachabilityDown blocks status transition even when counter hits zero", () => {
|
|
const r = compute({
|
|
currentStatus: "up",
|
|
reachabilityDown: true,
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 1, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.transitioned).toBe(false);
|
|
expect(r.nextStatus).toBe("up"); // stays at currentStatus
|
|
});
|
|
|
|
it("reachabilityDown still updates counters (the gate is only around status, not metrics math)", () => {
|
|
const r = compute({
|
|
reachabilityDown: true,
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.9, temperature: [30] } },
|
|
counters: { cpu: 3, memory: 5, disk: 5, temp: 5 },
|
|
});
|
|
expect(r.nextCounters.cpu).toBe(2);
|
|
expect(r.breaches.cpu).toBe(true);
|
|
});
|
|
|
|
it("reachabilityDown blocks recovery from breached", () => {
|
|
// Even with all metrics normal, if reachability says down, we can't recover.
|
|
const r = compute({ currentStatus: "breached", reachabilityDown: true });
|
|
expect(r.transitioned).toBe(false);
|
|
expect(r.nextStatus).toBe("breached");
|
|
});
|
|
|
|
// ── Percentage vs raw-degree thresholds ──
|
|
it("CPU/memory/disk thresholds are percentages (compared against usage * 100)", () => {
|
|
// threshold 50 means 50%, usage 0.6 = 60% → breach
|
|
const r = compute({
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.6, temperature: [30] } },
|
|
thresholds: { cpu: 50, memory: 80, disk: 80, temp: 80 },
|
|
});
|
|
expect(r.breaches.cpu).toBe(true);
|
|
});
|
|
|
|
it("temperature threshold is raw degrees (no /100 scaling)", () => {
|
|
// threshold 75, temp 80 → breach
|
|
const r = compute({
|
|
metrics: { ...healthyMetrics(), cpu: { usage_percent: 0.1, temperature: [80] } },
|
|
thresholds: { cpu: 80, memory: 80, disk: 80, temp: 75 },
|
|
});
|
|
expect(r.breaches.temp).toBe(true);
|
|
});
|
|
});
|
|
});
|