TRX/Checkmate
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3993 from bluewave-labs/fix/extract-formatting-from-incident-service

Browse Source 
fix: extract formatting from incident service
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-09-24 15:08:20 -07:00
2 parents 1ef540679b 5d77b8804f
commit 32a5ca8799
8 files changed
+83 -97
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/domain/incidents/incident.service.ts
+2 -19
View File
@@ -1,4 +1,3 @@
const SERVICE_NAME = "incidentService";
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { Check } from "@/domain/checks/check.type.js";
import { AppError } from "@/utils/AppError.js";
@@ -12,8 +11,8 @@ import type { MonitorActionDecision } from "@/worker/worker.interface.js";
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
import type { ILogger } from "@/utils/logger.js";
import { DateRange } from "@/types/query.js";
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
const SERVICE_NAME = "incidentService";
export interface IIncidentService {
handleIncident(monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise<Incident | null>;
resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise<Incident>;
@@ -71,7 +70,7 @@ export class IncidentService implements IIncidentService {
// For threshold breaches, use 9999 status code and build descriptive message
if (decision.incidentReason === "threshold_breach") {
statusCode = 9999;
message = this.buildThresholdBreachMessage(monitor, check, decision);
message = this.notificationMessageBuilder.buildThresholdBreachMessage(monitor, check, decision.thresholdBreaches);
}
const incident = {
@@ -96,22 +95,6 @@ export class IncidentService implements IIncidentService {
return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
};
private buildThresholdBreachMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision): string {
if (monitor.type === "docker") {
const containers = check.containers ?? [];
const breaches = findContainerBreaches(monitor, containers);
return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert";
}
const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check, decision.thresholdBreaches);
if (breaches.length === 0) {
return "Threshold breach detected";
}
return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", ");
}
resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => {
try {
if (!incidentId) {
server/src/domain/notifications/notification.message-builder.ts
+22 -6
View File
@@ -12,7 +12,7 @@ import type { Check } from "@/domain/checks/check.type.js";
export interface INotificationMessageBuilder {
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage;
extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[];
buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string;
}
const SERVICE_NAME = "NotificationMessageBuilder";
@@ -167,11 +167,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
}
private buildContainerBreachContent(monitor: Monitor, check: Check): NotificationContent {
const containers = check.containers ?? [];
const breaches = findContainerBreaches(monitor, containers);
const lines = this.describeContainerBreaches(monitor, check);
const title = `Container Alert: ${monitor.name}`;
const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`;
const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)];
const summary = `${lines.length} container(s) on "${monitor.name}" need attention.`;
const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...lines];
return { title, summary, details, timestamp: new Date() };
}
@@ -212,7 +211,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
}
}
public extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] {
private extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] {
const breaches: ThresholdBreach[] = [];
// Check if this is a hardware monitor with threshold data
@@ -227,4 +226,21 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
return breaches;
}
private describeContainerBreaches(monitor: Monitor, check: Check): string[] {
return findContainerBreaches(monitor, check.containers ?? []).map(describeContainerBreach);
}
public buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string {
if (monitor.type === "docker") {
const lines = this.describeContainerBreaches(monitor, check);
return lines.length > 0 ? lines.join(", ") : "Container alert";
}
const breaches = this.extractThresholdBreaches(monitor, check, thresholdBreaches);
if (breaches.length === 0) {
return "Threshold breach detected";
}
return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", ");
}
}
server/test/helpers/heartbeatTestHarness.ts
+2 -2
View File
@@ -63,7 +63,7 @@ export interface HeartbeatTestHarness {
networkService: { requestStatus: jest.Mock };
bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock };
maintenanceWindowsRepo: { findByMonitorId: jest.Mock };
messageBuilder: { extractThresholdBreaches: jest.Mock };
messageBuilder: { buildThresholdBreachMessage: jest.Mock };
egressService: { assessAfterFailure: jest.Mock };
heartbeatJob: (monitor: Monitor) => Promise<void>;
setNextResponse: (status: boolean, code: number) => void;
@@ -79,7 +79,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
const statusService = new StatusService(logger, monitorsRepo as any, createStubMonitorStatsRepo() as any);
const messageBuilder = { extractThresholdBreaches: jest.fn().mockReturnValue([]) };
const messageBuilder = { buildThresholdBreachMessage: jest.fn().mockReturnValue("") };
const incidentService = new IncidentService(logger, incidentsRepo, monitorsRepo as any, { findById: jest.fn() } as any, messageBuilder as any);
const notificationsService = { handleNotifications: jest.fn().mockResolvedValue(true) };
server/test/integration/heartbeatJobThresholdBreach.test.ts
+2 -2
View File
@@ -49,7 +49,7 @@ describe("Heartbeat job: hardware threshold breach", () => {
h.setNextResponseFull(makeHardwareResponse(0.9));
// Configure message builder for when incident is created
h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]);
h.messageBuilder.buildThresholdBreachMessage.mockReturnValue("CPU: 90% (threshold: 80%)");
// 4 heartbeats: counter goes 5→4→3→2→1, status still "up"
for (let i = 0; i < 4; i++) {
@@ -191,7 +191,7 @@ describe("Heartbeat job: hardware threshold breach", () => {
h.monitorsRepo.seed(monitor);
h.setNextResponseFull(makeHardwareResponse(0.9));
h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]);
h.messageBuilder.buildThresholdBreachMessage.mockReturnValue("CPU: 90% (threshold: 80%)");
// 5 heartbeats to trigger breach, then 5 more while still breaching
for (let i = 0; i < 10; i++) {
server/test/integration/incidentLifecycle.test.ts
+2 -5
View File
@@ -55,7 +55,7 @@ describe("Incident lifecycle (integration)", () => {
repo = new InMemoryIncidentsRepository();
monitorsRepo = { findById: jest.fn() } as unknown as jest.Mocked<IMonitorsRepository>;
usersRepo = { findById: jest.fn() } as unknown as jest.Mocked<IUsersRepository>;
messageBuilder = { extractThresholdBreaches: jest.fn() } as unknown as jest.Mocked<INotificationMessageBuilder>;
messageBuilder = { buildThresholdBreachMessage: jest.fn() } as unknown as jest.Mocked<INotificationMessageBuilder>;
service = new IncidentService(createMockLogger() as any, repo, monitorsRepo, usersRepo, messageBuilder);
});
@@ -152,10 +152,7 @@ describe("Incident lifecycle (integration)", () => {
it("creates a threshold breach incident with statusCode 9999 and descriptive message", async () => {
const monitor = makeMonitor({ status: "breached", type: "hardware" });
(messageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([
{ metric: "cpu", formattedValue: "92%", threshold: 80, unit: "%" },
{ metric: "memory", formattedValue: "88%", threshold: 85, unit: "%" },
]);
(messageBuilder.buildThresholdBreachMessage as jest.Mock).mockReturnValue("CPU: 92% (threshold: 80%), MEMORY: 88% (threshold: 85%)");
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
const incident = await service.handleIncident(monitor, decision, makeCheck(200));
server/test/unit/services/incidentService.test.ts
+5 -62
View File
@@ -36,7 +36,7 @@ const createUsersRepo = () =>
const createMessageBuilder = () =>
({
extractThresholdBreaches: jest.fn(),
buildThresholdBreachMessage: jest.fn(),
}) as unknown as jest.Mocked<INotificationMessageBuilder>;
const createService = (overrides?: {
@@ -131,7 +131,7 @@ describe("IncidentService", () => {
it("creates a new incident when shouldCreateIncident and no active incident exists", async () => {
const created = makeIncident();
const { service, incidentsRepository } = createService();
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
@@ -150,6 +150,7 @@ describe("IncidentService", () => {
statusCode: 500,
})
);
expect(notificationMessageBuilder.buildThresholdBreachMessage).not.toHaveBeenCalled();
});
it("uses status code 9999 and builds message for threshold_breach incidents", async () => {
@@ -157,9 +158,7 @@ describe("IncidentService", () => {
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
(notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([
{ metric: "cpu", formattedValue: "95%", threshold: 80, unit: "%" },
]);
(notificationMessageBuilder.buildThresholdBreachMessage as jest.Mock).mockReturnValue("CPU: 95% (threshold: 80%)");
const monitor = makeMonitor({ type: "hardware" });
const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } });
@@ -167,7 +166,7 @@ describe("IncidentService", () => {
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach", thresholdBreaches });
await service.handleIncident(monitor, decision, check);
expect(notificationMessageBuilder.extractThresholdBreaches).toHaveBeenCalledWith(monitor, check, thresholdBreaches);
expect(notificationMessageBuilder.buildThresholdBreachMessage).toHaveBeenCalledWith(monitor, check, thresholdBreaches);
expect(incidentsRepository.create).toHaveBeenCalledWith(
expect.objectContaining({
statusCode: 9999,
@@ -176,62 +175,6 @@ describe("IncidentService", () => {
);
});
it("uses fallback message when extractThresholdBreaches returns empty array", async () => {
const created = makeIncident();
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
(notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([]);
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
await service.handleIncident(makeMonitor(), decision, makeCheck());
expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" }));
});
it("builds the container list for docker threshold_breach incidents", async () => {
const created = makeIncident();
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true });
const check = makeCheck({
metadata: { monitorId: "mon-1", teamId: "team-1", type: "docker" },
containers: [
{ id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 },
{ id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" },
{ id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" },
],
containerSummary: { total: 3, running: 2, stopped: 1, unhealthy: 1 },
});
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
await service.handleIncident(monitor, decision, check);
expect(incidentsRepository.create).toHaveBeenCalledWith(
expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" })
);
expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled();
});
it("joins multiple threshold breaches with commas", async () => {
const created = makeIncident();
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
(notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([
{ metric: "cpu", formattedValue: "95%", threshold: 80, unit: "%" },
{ metric: "memory", formattedValue: "90%", threshold: 80, unit: "%" },
]);
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
await service.handleIncident(makeMonitor(), decision, makeCheck());
expect(incidentsRepository.create).toHaveBeenCalledWith(
expect.objectContaining({ message: "CPU: 95% (threshold: 80%), MEMORY: 90% (threshold: 80%)" })
);
});
it("resolves active incident when shouldResolveIncident", async () => {
const active = makeIncident();
const resolved = makeIncident({ status: false, endTime: "123", resolutionType: "automatic" });
server/test/unit/services/notificationMessageBuilder.test.ts
+47
View File
@@ -627,4 +627,51 @@ describe("NotificationMessageBuilder", () => {
});
});
});
// ── buildThresholdBreachMessage ─────────────────────────────────────────
describe("buildThresholdBreachMessage", () => {
const none: HardwareBreaches = { cpu: false, memory: false, disk: false, temp: false };
const hardwareMonitor = () => makeMonitor({ type: "hardware", cpuAlertThreshold: 80, memoryAlertThreshold: 80 });
it("describes a single flagged metric", () => {
const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] } });
const message = builder.buildThresholdBreachMessage(hardwareMonitor(), check, { ...none, cpu: true });
expect(message).toBe("CPU: 95.0% (threshold: 80%)");
});
it("joins several flagged metrics with a comma", () => {
const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] }, memory: { usage_percent: 0.9 } });
const message = builder.buildThresholdBreachMessage(hardwareMonitor(), check, { ...none, cpu: true, memory: true });
expect(message).toBe("CPU: 95.0% (threshold: 80%), MEMORY: 90.0% (threshold: 80%)");
});
it("falls back when no metric is flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] } });
expect(builder.buildThresholdBreachMessage(hardwareMonitor(), check, none)).toBe("Threshold breach detected");
expect(builder.buildThresholdBreachMessage(hardwareMonitor(), check, undefined)).toBe("Threshold breach detected");
});
it("lists the breaching containers for a docker monitor", () => {
const check = makeCheck({
status: true,
containers: makeContainers([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]),
});
const message = builder.buildThresholdBreachMessage(makeDockerMonitor(), check, undefined);
expect(message).toBe("db: stopped (exit code 1), api: unhealthy");
});
it("falls back when no container is breaching", () => {
const check = makeCheck({ status: true, containers: makeContainers([{ name: "web" }]) });
expect(builder.buildThresholdBreachMessage(makeDockerMonitor(), check, undefined)).toBe("Container alert");
});
});
});
server/test/unit/services/notificationsService.test.ts
+1 -1
View File
@@ -33,7 +33,7 @@ const createSettingsService = (clientHost = "https://app.example.com") => ({
const createMessageBuilder = () => ({
buildMessage: jest.fn().mockReturnValue({ type: "monitor_down", content: { title: "Down" } }),
extractThresholdBreaches: jest.fn(),
buildThresholdBreachMessage: jest.fn(),
});
const createService = (overrides?: Record<string, unknown>) => {