refactor handleIncidents

This commit is contained in:
Alex Holliday committed 2026-02-15 18:20:42 +00:00
1 parent 7061eb8eb0
commit 8020bfde21
3 files changed
+46 -101

No files matched your search

@@ -27,15 +27,22 @@ export const buildHardwareAlerts = (
const cpuThreshold = monitor.cpuAlertThreshold !== undefined ? monitor.cpuAlertThreshold / 100 : -1;
const memoryThreshold = monitor.memoryAlertThreshold !== undefined ? monitor.memoryAlertThreshold / 100 : -1;
const diskThreshold = monitor.diskAlertThreshold !== undefined ? monitor.diskAlertThreshold / 100 : -1;
const tempThreshold = monitor.tempAlertThreshold !== undefined ? monitor.tempAlertThreshold : -1;
const payload = networkResponse?.payload as HardwareStatusPayload;
const metrics = payload.data || {};
const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics;
const { cpu = {}, memory = {}, disk = [] } = metrics;
const cpuUsage = cpu.usage_percent ?? -1;
const memoryUsage = memory.usage_percent ?? -1;
// Get max temperature from CPU temperature sensors array
const temps = cpu.temperature ?? [];
const maxTemp = temps.length > 0 ? Math.max(...temps) : -1;
const alerts: Record<string, boolean> = {
cpu: cpuThreshold !== -1 && cpuUsage > cpuThreshold ? true : false,
memory: memoryThreshold !== -1 && memoryUsage > memoryThreshold ? true : false,
disk: disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold) ?? false,
temp: tempThreshold !== -1 && maxTemp > tempThreshold ? true : false,
};
const alertsToSend = [];
@@ -45,6 +52,7 @@ export const buildHardwareAlerts = (
{ name: "URL", value: monitor.url, inline: false },
];
const goToIncidentField = { name: `Go to incident`, value: `${clientHost}/infrastructure/${monitor.id}` };
const formatDiscordAlert = {
cpu: () => ({
title: "CPU alert",
@@ -77,38 +85,33 @@ export const buildHardwareAlerts = (
goToIncidentField,
],
}),
temp: () => ({
title: "Temperature alert",
description: `Your current temperature (${maxTemp.toFixed(0)}°C) is above your threshold (${tempThreshold.toFixed(0)}°C)`,
color: 15548997,
fields: [...monitorInfoFields, goToIncidentField],
footer: { text: "Checkmate" },
}),
};
const alertTypes = ["cpu", "memory", "disk"] as const;
const alertThresholdKeyMap: Record<(typeof alertTypes)[number], "cpuAlertThreshold" | "memoryAlertThreshold" | "diskAlertThreshold"> = {
cpu: "cpuAlertThreshold",
memory: "memoryAlertThreshold",
disk: "diskAlertThreshold",
};
const alertTypes = ["cpu", "memory", "disk", "temp"] as const;
for (const type of alertTypes) {
const thresholdKey = alertThresholdKeyMap[type];
// Iterate over each alert type to see if any need to be decremented
if (alerts[type] === true) {
const nextValue = monitor[thresholdKey] - 1;
monitor[thresholdKey] = nextValue; // Decrement threshold if an alert is triggered
if (monitor[thresholdKey] <= 0) {
// If threshold drops below 0, reset and send notification
monitor[thresholdKey] = 100;
const formatAlert = {
cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
memory: () =>
`Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
disk: () =>
`Your current disk usage: ${disk
.map((d, idx) => `(Disk${idx}: ${(d?.usage_percent ?? 0 * 100).toFixed(0)}%)`)
.join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
};
alertsToSend.push(formatAlert[type]());
discordEmbeds.push(formatDiscordAlert[type]());
}
const formatAlert = {
cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
memory: () =>
`Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
disk: () =>
`Your current disk usage: ${disk.map((d, idx) => `(Disk${idx}: ${(d?.usage_percent ?? 0 * 100).toFixed(0)}%)`).join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
temp: () => `Your current temperature (${maxTemp.toFixed(0)}°C) is above your threshold (${tempThreshold.toFixed(0)}°C)`,
};
alertsToSend.push(formatAlert[type]());
discordEmbeds.push(formatDiscordAlert[type]());
}
}
const discordPayload = discordEmbeds.length ? { embeds: discordEmbeds } : null;
return { alertsToSend, discordPayload };
};
@@ -125,34 +128,6 @@ export const buildHardwareWebhookBody = (alerts: string[], monitor: Monitor): st
return content;
};
export const shouldSendHardwareAlert = (monitor: Monitor, networkResponse: MonitorStatusResponse): boolean => {
// Thresholds are stored as percentages (0-100), convert to decimal (0-1) for comparison
const cpuThreshold = monitor.cpuAlertThreshold !== undefined ? monitor.cpuAlertThreshold / 100 : -1;
const memoryThreshold = monitor.memoryAlertThreshold !== undefined ? monitor.memoryAlertThreshold / 100 : -1;
const diskThreshold = monitor.diskAlertThreshold !== undefined ? monitor.diskAlertThreshold / 100 : -1;
const payload = networkResponse?.payload as HardwareStatusPayload;
const metrics = payload.data || {};
const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics;
const cpuBreach = cpuThreshold !== -1 && cpuUsage > cpuThreshold;
if (cpuBreach && monitor.cpuAlertThreshold - 1 <= 0) {
return true;
}
const memoryBreach = memoryThreshold !== -1 && memoryUsage > memoryThreshold;
if (memoryBreach && monitor.memoryAlertThreshold - 1 <= 0) {
return true;
}
const diskBreach = disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold);
if (diskBreach && monitor.diskAlertThreshold - 1 <= 0) {
return true;
}
return false;
};
export const buildWebhookBody = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
const { status, code } = monitorStatusResponse;
const { localTimeZone, localTime, utcTime } = getTime();