Compare commits

..
Author SHA1 Message Date
Alex Holliday 5d77b8804f update tests 2026-09-24 14:22:31 -07:00
Alex Holliday 544502c6e5 move formatting 2026-09-24 14:19:13 -07:00
Alexander Holliday 1ef540679b Merge pull request #3992 from bluewave-labs/chore/refactor-cleanup
chore: refactor cleanup
2026-09-24 14:10:19 -07:00
Alex Holliday 18f38818e2 update tests 2026-09-24 14:03:34 -07:00
Alex Holliday 02e6333c3c narrow notificaitonReason type 2026-09-24 14:00:54 -07:00
Alex Holliday cb47439e9f remove double logging 2026-09-24 13:55:17 -07:00
Alex Holliday 4bbfe2e889 remove extra write on short window 2026-09-24 13:49:54 -07:00
Alex Holliday 187a0fc3d4 remove extra guard 2026-09-24 13:43:40 -07:00
Alex Holliday 4e4d3c3971 remove code and timestamp from stautschangeresult, no longer used 2026-09-24 13:41:58 -07:00
Alexander Holliday 6267e08a51 Merge pull request #3991 from bluewave-labs/fix/threshold-source-of-truth
fix: threshold source of truth
2026-09-24 13:33:12 -07:00
Alex Holliday f2c23636fe type only imports, add empty guard for threshold breaches do Docker monitors don't get this 2026-09-24 13:20:03 -07:00
Alex Holliday 11c89d150a update tests 2026-09-24 12:58:28 -07:00
Alex Holliday 6b75f9bf7c remove statusChange from decision, no longer needed 2026-09-24 12:41:33 -07:00
Alex Holliday 91de9e7fb7 pass threshold breaches on rather than reconsructing 2026-09-24 12:25:22 -07:00
Alex Holliday 21fb320f3a add hardware metric key types 2026-09-24 11:46:04 -07:00
Alexander Holliday 3586ac9437 Merge pull request #3984 from bluewave-labs/feat/dashboard-add-new-cards-options
feat(dashboard): add currently down and lowest uptime dashboard cards
2026-09-24 09:38:44 -07:00
Malena Aguiar b0ed75382e feat(dashboard): add code refactors 2026-09-23 17:57:28 -06:00
Alexander Holliday b1646cd83b Merge pull request #3988 from bluewave-labs/chore/evaluator-refactor
chore: refactor out rebuilding of status response
2026-09-23 13:56:56 -07:00
Alex Holliday 7abc74ac98 udpate tests 2026-09-23 13:54:55 -07:00
Alex Holliday a1b10c7469 remove unused param 2026-09-23 13:51:30 -07:00
Alex Holliday 0d75629dcb refactor out rebuilding of status response 2026-09-23 13:38:02 -07:00
Alexander Holliday 849ea26cf9 Merge pull request #3987 from bluewave-labs/fix/email-error-handling
fix: improve email error handling
2026-09-23 12:19:39 -07:00
Alex Holliday 4a66cf6e11 improve email error handling 2026-09-23 12:13:27 -07:00
Alexander Holliday 41c7cfbdaa Merge pull request #3986 from bluewave-labs/feat/docker-container-alerts
feat: docker container alerts
2026-09-23 11:50:04 -07:00
Alex Holliday 7fb57d8090 fix incident links with helper 2026-09-23 11:46:58 -07:00
Alex Holliday 618258d282 fix docker path 2026-09-23 11:39:16 -07:00
Alex Holliday e50568dba3 add summary props to docker monitor list page, add localisation strings 2026-09-23 11:31:37 -07:00
Alex Holliday 5fa894cb79 Add client side types, validation, add swithces to monitor form 2026-09-23 11:17:19 -07:00
Alex Holliday 6d4e944a58 Add tests 2026-09-23 11:17:01 -07:00
Alex Holliday cfe7d2a359 add missing braces, fix type error 2026-09-23 11:06:25 -07:00
Alex Holliday 5fb26367ed reorder docker case 2026-09-23 11:03:33 -07:00
Alex Holliday cd65488cbd add docker branch to incident service, update providers 2026-09-23 10:57:49 -07:00
Alex Holliday 4ea6539a7d build notifications for container breaches 2026-09-23 10:54:35 -07:00
Alex Holliday a75bb5fae6 add docker container alert fields to monitors,checks 2026-09-23 10:16:49 -07:00
Malena Aguiar bdd4634a3b feat(dashboard): add currently down and lowest uptime dashboard cards 2026-09-22 16:18:51 -06:00
77 changed files with 1955 additions and 1448 deletions

No files matched your search

+11 -1
View File
@@ -1,4 +1,8 @@
import { dashboardCardKeys, type DashboardCardKey } from "@/Types/Dashboard";
import {
dashboardCardKeys,
type DashboardCardKey,
type DashboardSortOrder,
} from "@/Types/Dashboard";
import { createSlice, type PayloadAction } from "@reduxjs/toolkit";
type ThemeMode = "light" | "dark";
@@ -38,6 +42,7 @@ interface UIState {
starPromptOpen: boolean;
chartType: ChartType;
dashboardVisibleCards: DashboardCardKey[];
dashboardUptimeSortOrder: DashboardSortOrder;
}
const initialMode: ThemeMode = window?.matchMedia?.("(prefers-color-scheme: dark)")
@@ -78,6 +83,7 @@ const initialState: UIState = {
starPromptOpen: true,
chartType: "histogram",
dashboardVisibleCards: [...dashboardCardKeys],
dashboardUptimeSortOrder: "ascending",
};
const uiSlice = createSlice({
@@ -124,6 +130,9 @@ const uiSlice = createSlice({
setDashboardVisibleCards: (state, action: PayloadAction<DashboardCardKey[]>) => {
state.dashboardVisibleCards = action.payload;
},
setDashboardUptimeSortOrder: (state, action: PayloadAction<DashboardSortOrder>) => {
state.dashboardUptimeSortOrder = action.payload;
},
},
});
@@ -141,4 +150,5 @@ export const {
setStarPromptOpen,
setChartType,
setDashboardVisibleCards,
setDashboardUptimeSortOrder,
} = uiSlice.actions;
+2
View File
@@ -85,6 +85,8 @@ export const getMonitorDefaults = (
dockerTlsKey: "",
dockerTlsKeySet: data?.dockerTlsKeySet ?? false,
dockerLogsEnabled: data?.dockerLogsEnabled ?? false,
dockerAlertOnStopped: data?.dockerAlertOnStopped ?? false,
dockerAlertOnUnhealthy: data?.dockerAlertOnUnhealthy ?? false,
};
break;
case "game":
+22 -12
View File
@@ -617,7 +617,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(0) && (
<ConfigBox
title={t("pages.createMonitor.form.general.title")}
@@ -758,7 +757,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(0) && watchedType === "docker" && isDockerTlsUrl(watchedUrl) && (
<ConfigBox
title={t("pages.createMonitor.form.dockerTls.title")}
@@ -816,7 +814,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(1) && (
<ConfigBox
title={t("pages.createMonitor.form.frequency.title")}
@@ -837,7 +834,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{/* Alert Thresholds - only for hardware type */}
{watchedType === "hardware" && showStep(1) && (
<ConfigBox
@@ -893,7 +889,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(1) && (
<ConfigBox
title={t("pages.createMonitor.form.incidents.title")}
@@ -920,7 +915,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(1) && (
<ConfigBox
title={t("pages.createMonitor.form.notifications.title")}
@@ -933,7 +927,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(1) && (
<ConfigBox
title={t("pages.createMonitor.form.tags.title")}
@@ -960,7 +953,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(1) && watchedType === "docker" && !isCaptureDocker && (
<ConfigBox
title={t("pages.createMonitor.form.dockerLogs.title")}
@@ -973,7 +965,28 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(1) && watchedType === "docker" && (
<ConfigBox
title={t("pages.createMonitor.form.dockerAlerts.title")}
subtitle={t("pages.createMonitor.form.dockerAlerts.description")}
rightContent={
<Stack gap={theme.spacing(LAYOUT.MD)}>
<FormSwitchField
name="dockerAlertOnStopped"
label={t(
"pages.createMonitor.form.dockerAlerts.option.onStopped.label"
)}
/>
<FormSwitchField
name="dockerAlertOnUnhealthy"
label={t(
"pages.createMonitor.form.dockerAlerts.option.onUnhealthy.label"
)}
/>
</Stack>
}
/>
)}
{showStep(2) &&
(watchedType === "http" ||
watchedType === "grpc" ||
@@ -989,7 +1002,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(2) && watchedType === "http" && (
<ConfigBox
title={t("pages.createMonitor.form.advanced.title")}
@@ -1089,7 +1101,6 @@ const CreateMonitorPage = () => {
}
/>
)}
{showStep(2) && supportsGeoCheck(watchedType) && (
<ConfigBox
title={t("pages.createMonitor.form.geoChecks.title")}
@@ -1122,7 +1133,6 @@ const CreateMonitorPage = () => {
}
/>
)}
<Stack
direction="row"
justifyContent={!isEditMode ? "space-between" : "flex-end"}
@@ -1,8 +1,11 @@
import { useState } from "react";
import { useMemo, useState } from "react";
import Stack from "@mui/material/Stack";
import FormControlLabel from "@mui/material/FormControlLabel";
import Typography from "@mui/material/Typography";
import { useTheme } from "@mui/material/styles";
import { useTranslation } from "react-i18next";
import { Checkbox, Dialog } from "@/Components/inputs";
import { LAYOUT } from "@/Utils/Theme/constants";
import { type DashboardCardKey, dashboardCardKeys } from "@/Types/Dashboard";
interface EditCardsModalProps {
@@ -18,16 +21,19 @@ export const EditCardsModal = ({
onClose,
onSave,
}: EditCardsModalProps) => {
const theme = useTheme();
const { t } = useTranslation();
const [draft, setDraft] = useState<Set<DashboardCardKey>>(new Set(visibleCards));
const [cardsSelection, setCardsSelection] = useState<Set<DashboardCardKey>>(
new Set(visibleCards)
);
const handleClose = () => {
setDraft(new Set(visibleCards));
setCardsSelection(new Set(visibleCards));
onClose();
};
const handleToggle = (key: DashboardCardKey) => {
setDraft((prev) => {
setCardsSelection((prev) => {
const next = new Set(prev);
if (next.has(key)) {
next.delete(key);
@@ -38,26 +44,40 @@ export const EditCardsModal = ({
});
};
const sortedCardKeys = useMemo(
() =>
[...dashboardCardKeys].sort((a, b) =>
t(`pages.dashboard.cards.${a}`).localeCompare(t(`pages.dashboard.cards.${b}`))
),
[t]
);
return (
<Dialog
open={open}
title={t("pages.dashboard.editCardsModal.title")}
content={t("pages.dashboard.editCardsModal.description")}
onCancel={handleClose}
onConfirm={() => onSave(draft)}
onConfirm={() => onSave(cardsSelection)}
confirmText={t("pages.dashboard.editCardsModal.done")}
maxWidth="xs"
fullWidth
>
<Stack>
{dashboardCardKeys.map((key: DashboardCardKey) => (
<Stack gap={theme.spacing(LAYOUT.XXS)}>
<Typography
variant="eyebrow"
color={theme.palette.text.disabled}
>
{t("pages.dashboard.editCardsModal.groups.uptime")}
</Typography>
{sortedCardKeys.map((key: DashboardCardKey) => (
<FormControlLabel
key={key}
label={t(`pages.dashboard.cards.${key}`)}
control={
<Checkbox
checked={draft.has(key)}
disabled={draft.has(key) && draft.size === 1}
checked={cardsSelection.has(key)}
disabled={cardsSelection.has(key) && cardsSelection.size === 1}
onChange={() => handleToggle(key)}
/>
}
@@ -0,0 +1,58 @@
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { useTheme } from "@mui/material/styles";
import { useTranslation } from "react-i18next";
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
import { Dot } from "@/Components/design-elements";
import { typographyLevels } from "@/Utils/Theme/Palette";
import { LAYOUT } from "@/Utils/Theme/constants";
import type { Monitor } from "@/Types/Monitor";
interface CurrentlyDownCardProps {
monitors: Monitor[];
}
export const CurrentlyDownCard = ({ monitors }: CurrentlyDownCardProps) => {
const theme = useTheme();
const { t } = useTranslation();
const downMonitors = monitors.filter((monitor) => monitor.status === "down");
return (
<DashboardCard
title={t("pages.dashboard.cards.currentlyDown")}
topRight={downMonitors.length > 0 ? downMonitors.length : undefined}
>
{downMonitors.length === 0 ? (
<Typography
fontSize={typographyLevels.m}
color={theme.palette.text.disabled}
>
{t("pages.dashboard.currentlyDown.empty")}
</Typography>
) : (
<Stack gap={theme.spacing(LAYOUT.SM)}>
{downMonitors.map((monitor) => (
<Stack
key={monitor.id}
direction="row"
alignItems="center"
gap={theme.spacing(LAYOUT.XS)}
>
<Dot color={theme.palette.error.main} />
<Typography
fontSize={typographyLevels.m}
color={theme.palette.text.primary}
overflow="hidden"
textOverflow="ellipsis"
whiteSpace="nowrap"
>
{monitor.name}
</Typography>
</Stack>
))}
</Stack>
)}
</DashboardCard>
);
};
@@ -0,0 +1,90 @@
import IconButton from "@mui/material/IconButton";
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { useTheme } from "@mui/material/styles";
import { useTranslation } from "react-i18next";
import { useDispatch, useSelector } from "react-redux";
import { ArrowDown, ArrowUp } from "lucide-react";
import { Icon } from "@/Components/design-elements";
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
import { BarRow } from "@/Pages/Dashboard/components/cards/BarRow";
import { getUptimePercentageColor } from "@/Utils/MonitorUtils";
import { formatPercentage } from "@/Utils/FormatUtils";
import { typographyLevels } from "@/Utils/Theme/Palette";
import { LAYOUT } from "@/Utils/Theme/constants";
import { setDashboardUptimeSortOrder } from "@/Features/UI/uiSlice";
import type { Monitor } from "@/Types/Monitor";
import type { AppDispatch, RootState } from "@/store";
const TOP_N = 5;
interface UptimeCardProps {
monitors: Monitor[];
}
export const UptimeCard = ({ monitors }: UptimeCardProps) => {
const theme = useTheme();
const { t } = useTranslation();
const dispatch = useDispatch<AppDispatch>();
const sortOrder = useSelector((state: RootState) => state.ui.dashboardUptimeSortOrder);
const direction = sortOrder === "ascending" ? 1 : -1;
const ranked = monitors
.filter((monitor) => monitor.status !== "paused")
.filter(
(monitor): monitor is Monitor & { uptimePercentage: number } =>
typeof monitor.uptimePercentage === "number"
)
.sort((a, b) => direction * (a.uptimePercentage - b.uptimePercentage))
.slice(0, TOP_N);
return (
<DashboardCard
title={t("pages.dashboard.cards.uptime")}
topRight={
<IconButton
size="small"
aria-label={t("pages.dashboard.uptimeCard.toggleSortOrder")}
onClick={() =>
dispatch(
setDashboardUptimeSortOrder(
sortOrder === "ascending" ? "descending" : "ascending"
)
)
}
>
<Icon
icon={sortOrder === "ascending" ? ArrowUp : ArrowDown}
size={16}
color={theme.palette.text.secondary}
/>
</IconButton>
}
>
{ranked.length === 0 ? (
<Typography
fontSize={typographyLevels.m}
color={theme.palette.text.disabled}
>
{t("pages.dashboard.noData")}
</Typography>
) : (
<Stack gap={theme.spacing(LAYOUT.SM)}>
{ranked.map((monitor) => {
const paletteKey = getUptimePercentageColor(monitor.uptimePercentage);
const color = theme.palette[paletteKey].main;
return (
<BarRow
key={monitor.id}
label={monitor.name}
value={formatPercentage(monitor.uptimePercentage)}
fillRatio={monitor.uptimePercentage}
valueColor={color}
/>
);
})}
</Stack>
)}
</DashboardCard>
);
};
+4
View File
@@ -14,6 +14,8 @@ import { setDashboardVisibleCards } from "@/Features/UI/uiSlice";
import type { RootState, AppDispatch } from "@/store";
import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard";
import { CurrentlyDownCard } from "@/Pages/Dashboard/components/cards/CurrentlyDownCard";
import { UptimeCard } from "@/Pages/Dashboard/components/cards/UptimeCard";
import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard";
import { EditCardsModal } from "@/Pages/Dashboard/components/EditCardsModal";
import { type DashboardCardKey, dashboardCardKeys } from "@/Types/Dashboard";
@@ -48,6 +50,8 @@ const Dashboard = () => {
const cardComponents: Record<DashboardCardKey, ReactNode> = {
monitorStatus: <MonitorStatusCard summary={summary} />,
currentlyDown: <CurrentlyDownCard monitors={monitors ?? []} />,
uptime: <UptimeCard monitors={monitors ?? []} />,
monitorsByType: <MonitorsByTypeCard monitorsByType={monitorsByType} />,
};
@@ -19,6 +19,7 @@ const DockerMonitorsPage = () => {
controller={c}
bulkActions
showTypeFilter={false}
summaryProps={{ showBreached: true }}
>
<DockerMonitorsTable
monitors={c.monitors || []}
+9 -1
View File
@@ -1,2 +1,10 @@
export const dashboardCardKeys = ["monitorStatus", "monitorsByType"] as const;
export const dashboardCardKeys = [
"monitorStatus",
"currentlyDown",
"uptime",
"monitorsByType",
] as const;
export type DashboardCardKey = (typeof dashboardCardKeys)[number];
export const dashboardSortOrders = ["ascending", "descending"] as const;
export type DashboardSortOrder = (typeof dashboardSortOrders)[number];
+2
View File
@@ -169,6 +169,8 @@ export interface Monitor {
geoCheckLocations?: GeoContinent[];
geoCheckInterval?: number;
dockerLogsEnabled?: boolean;
dockerAlertOnStopped?: boolean;
dockerAlertOnUnhealthy?: boolean;
dockerTlsCa?: string;
dockerTlsCert?: string;
dockerTlsKeySet?: boolean;
+1 -1
View File
@@ -20,7 +20,7 @@ export const getMonitorPath = (type: MonitorType): string => {
websocket: "uptime",
dns: "uptime",
unknown: "uptime",
docker: "docker",
docker: "docker/host",
hardware: "infrastructure",
pagespeed: "pagespeed",
};
+2
View File
@@ -152,6 +152,8 @@ const dockerSchema = baseSchema.extend({
// Read only mirror of the server flag, stripped on submit
dockerTlsKeySet: z.boolean(),
dockerLogsEnabled: z.boolean().register(monitorStepRegistry, { step: 1 }),
dockerAlertOnStopped: z.boolean().register(monitorStepRegistry, { step: 1 }),
dockerAlertOnUnhealthy: z.boolean().register(monitorStepRegistry, { step: 1 }),
});
// Game server monitor schema
+29 -6
View File
@@ -557,7 +557,7 @@
},
"status": {
"down": "down",
"breached": "threshold exceeded",
"breached": "alert",
"initializing": "initializing",
"maintenance": "maintenance",
"paused": "paused",
@@ -827,6 +827,18 @@
}
}
},
"dockerAlerts": {
"title": "Container alerts",
"description": "Mark this host as breached while any container on it is stopped or failing its health check, and notify on the change. Containers that exit with code 0 do not count as stopped. Capture does not report exit codes, so on Capture hosts every stopped container counts.",
"option": {
"onStopped": {
"label": "Alert when a container is stopped"
},
"onUnhealthy": {
"label": "Alert when a container is unhealthy"
}
}
},
"stats": {
"title": "Monitor history",
"description": "Clear all monitoring history and statistics for your team. This action is irreversible.",
@@ -1685,7 +1697,7 @@
"status": {
"up": "Operational",
"down": "Down",
"breached": "Threshold exceeded",
"breached": "Alert",
"maintenance": "Maintenance",
"paused": "Paused",
"initializing": "Initializing"
@@ -1725,13 +1737,13 @@
"monitoringSummary": "Monitoring {{count}} service · Updates every 10 seconds",
"monitoringSummary_other": "Monitoring {{count}} services · Updates every 10 seconds",
"allUp": "All systems operational",
"allBreached": "All systems have exceeded thresholds",
"allBreached": "All systems have active alerts",
"allMaintenance": "All systems are under maintenance",
"allDown": "All systems are down",
"allPaused": "All systems are paused",
"allInitializing": "All systems are initializing",
"breachedAndDown": "Some systems have exceeded thresholds and are down",
"breached": "Some systems have exceeded thresholds",
"breachedAndDown": "Some systems have active alerts and some are down",
"breached": "Some systems have active alerts",
"degraded": "Some systems are experiencing issues",
"maintenanceAndDown": "Some systems are down during maintenance",
"maintenance": "Some systems are under maintenance",
@@ -1924,16 +1936,27 @@
},
"cards": {
"monitorStatus": "Monitor status",
"currentlyDown": "Currently down",
"uptime": "Monitor uptime",
"monitorsByType": "Monitors by type"
},
"uptimeCard": {
"toggleSortOrder": "Toggle sort order"
},
"currentlyDown": {
"empty": "No monitors are down at this moment."
},
"header": {
"title": "Dashboard",
"description": "See every monitor type in one place and spot what needs attention."
},
"editCards": "Edit cards",
"editCardsModal": {
"title": "Edit cards",
"title": "Dashboard cards",
"description": "Select the cards to display on your dashboard.",
"groups": {
"uptime": "Uptime"
},
"done": "Done"
}
}
+52
View File
@@ -466,6 +466,12 @@
"dockerLogsEnabled": {
"type": "boolean"
},
"dockerAlertOnStopped": {
"type": "boolean"
},
"dockerAlertOnUnhealthy": {
"type": "boolean"
},
"dockerTlsCa": {
"type": "string"
},
@@ -523,6 +529,8 @@
"geoCheckLocations",
"geoCheckInterval",
"dockerLogsEnabled",
"dockerAlertOnStopped",
"dockerAlertOnUnhealthy",
"teamId",
"userId",
"createdAt",
@@ -694,6 +702,12 @@
"dockerLogsEnabled": {
"type": "boolean"
},
"dockerAlertOnStopped": {
"type": "boolean"
},
"dockerAlertOnUnhealthy": {
"type": "boolean"
},
"dockerTlsCa": {
"type": "string"
},
@@ -753,6 +767,8 @@
"geoCheckLocations",
"geoCheckInterval",
"dockerLogsEnabled",
"dockerAlertOnStopped",
"dockerAlertOnUnhealthy",
"teamId",
"userId",
"createdAt",
@@ -7186,6 +7202,12 @@
"dockerLogsEnabled": {
"type": "boolean"
},
"dockerAlertOnStopped": {
"type": "boolean"
},
"dockerAlertOnUnhealthy": {
"type": "boolean"
},
"dockerTlsCa": {
"type": "string"
},
@@ -7245,6 +7267,8 @@
"geoCheckLocations",
"geoCheckInterval",
"dockerLogsEnabled",
"dockerAlertOnStopped",
"dockerAlertOnUnhealthy",
"teamId",
"userId",
"createdAt",
@@ -7761,6 +7785,12 @@
"dockerLogsEnabled": {
"type": "boolean"
},
"dockerAlertOnStopped": {
"type": "boolean"
},
"dockerAlertOnUnhealthy": {
"type": "boolean"
},
"dockerTlsCa": {
"type": "string"
},
@@ -7820,6 +7850,8 @@
"geoCheckLocations",
"geoCheckInterval",
"dockerLogsEnabled",
"dockerAlertOnStopped",
"dockerAlertOnUnhealthy",
"teamId",
"userId",
"createdAt",
@@ -9308,6 +9340,12 @@
"dockerLogsEnabled": {
"type": "boolean"
},
"dockerAlertOnStopped": {
"type": "boolean"
},
"dockerAlertOnUnhealthy": {
"type": "boolean"
},
"dockerTlsCa": {
"anyOf": [
{
@@ -10033,6 +10071,14 @@
"type": "boolean",
"default": false
},
"dockerAlertOnStopped": {
"type": "boolean",
"default": false
},
"dockerAlertOnUnhealthy": {
"type": "boolean",
"default": false
},
"dnsServer": {
"type": "string",
"minLength": 1
@@ -10600,6 +10646,12 @@
"dockerLogsEnabled": {
"type": "boolean"
},
"dockerAlertOnStopped": {
"type": "boolean"
},
"dockerAlertOnUnhealthy": {
"type": "boolean"
},
"dockerTlsCa": {
"anyOf": [
{
@@ -10,10 +10,7 @@ type WhoisModule = typeof whoiser;
export const fetchMonitorCertificate = async (checker: SSLCheckerType, monitor: Monitor): Promise<SSLDetails> => {
const monitorUrl = new URL(monitor.url);
const hostname = monitorUrl.hostname;
// An explicit port in the URL is the most visible to the user, so it takes precedence over monitor.port.
const urlPort = monitorUrl.port ? Number(monitorUrl.port) : undefined;
const port = urlPort ?? monitor.port;
const cert = await checker(hostname, port !== undefined ? { port } : undefined);
const cert = await checker(hostname);
if (cert?.validTo === null || cert?.validTo === undefined) {
throw new Error("Certificate not found");
}
@@ -35,11 +35,18 @@ class NotificationController implements INotificationController {
testNotification = catchAsync(async (req: Request, res: Response) => {
const notification = testNotificationBodyValidation.parse(req.body);
const success = await this.notificationsService.sendTestNotification(notification);
let success = false;
let reason: string | undefined;
try {
success = await this.notificationsService.sendTestNotification(notification);
} catch (error: unknown) {
if (!(error instanceof AppError)) throw error;
reason = error.message;
}
return res.status(200).json({
success,
msg: success ? "Notification sent successfully" : "Notification could not be sent — check the destination details.",
msg: success ? "Notification sent successfully" : (reason ?? "Notification could not be sent — check the destination details."),
details: { service: SERVICE_NAME },
});
});
@@ -9,6 +9,8 @@ import { IProxiesService } from "@/domain/proxies/proxy.service.js";
import { IEgressStateService } from "@/domain/egress/egress-state.service.js";
import { Settings } from "@/domain/app-settings/app-settings.type.js";
const SERVICE_NAME = "SettingsController";
export interface ISettingsController {
getAppSettings: RequestHandler;
updateAppSettings: RequestHandler;
@@ -119,7 +121,7 @@ class SettingsController implements ISettingsController {
const html = await this.emailService.buildEmail("testEmailTemplate", context);
if (!html) {
throw new AppError({ message: "Failed to build email template.", status: 500 });
throw new AppError({ message: "Failed to build email template.", status: 500, service: SERVICE_NAME, method: "sendTestEmail" });
}
let messageId: string;
try {
@@ -142,8 +144,11 @@ class SettingsController implements ISettingsController {
// Surface the underlying SMTP failure: diagnosing the settings is the whole
// point of the test email endpoint.
throw new AppError({
message: error instanceof Error ? `Failed to send test email. ${error.message}` : "Failed to send test email.",
status: 500,
message: error instanceof AppError ? error.message : "Failed to send test email.",
status: error instanceof AppError ? error.status : 500,
service: SERVICE_NAME,
method: "sendTestEmail",
details: error instanceof AppError ? error.details : undefined,
});
}
@@ -220,6 +220,8 @@ export const createMonitorBodyValidation = z
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
geoCheckInterval: z.number().min(300000).optional(),
dockerLogsEnabled: z.boolean().optional(),
dockerAlertOnStopped: z.boolean().optional(),
dockerAlertOnUnhealthy: z.boolean().optional(),
dockerTlsCa: z.union([z.string(), z.literal("")]).optional(),
dockerTlsCert: z.union([z.string(), z.literal("")]).optional(),
dockerTlsKey: z.union([z.string(), z.literal("")]).optional(),
@@ -270,6 +272,8 @@ export const editMonitorBodyValidation = z
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
geoCheckInterval: z.number().min(300000).optional(),
dockerLogsEnabled: z.boolean().optional(),
dockerAlertOnStopped: z.boolean().optional(),
dockerAlertOnUnhealthy: z.boolean().optional(),
dockerTlsCa: z.union([z.string(), z.literal("")]).optional(),
dockerTlsCert: z.union([z.string(), z.literal("")]).optional(),
dockerTlsKey: z.union([z.string(), z.literal("")]).optional(),
@@ -350,6 +354,8 @@ const importedMonitorSchema = z
geoCheckLocations: z.array(z.enum(GeoContinents)).default([]),
geoCheckInterval: z.number().min(300000).default(300000),
dockerLogsEnabled: z.boolean().default(false),
dockerAlertOnStopped: z.boolean().default(false),
dockerAlertOnUnhealthy: z.boolean().default(false),
dnsServer: dnsServerValidation.optional(),
dnsRecordType: z.enum(DnsRecordTypes).optional(),
createdAt: z.string().optional(),
@@ -440,6 +446,8 @@ export const monitorResponseSchema = z
geoCheckLocations: z.array(z.enum(GeoContinents)),
geoCheckInterval: z.number(),
dockerLogsEnabled: z.boolean(),
dockerAlertOnStopped: z.boolean(),
dockerAlertOnUnhealthy: z.boolean(),
dockerTlsCa: z.string().optional(),
dockerTlsCert: z.string().optional(),
dockerTlsKeySet: z.boolean().optional(),
+1
View File
@@ -220,6 +220,7 @@ const dockerContainerSchema = new Schema<DockerContainerInfo>(
startedAt: { type: String },
ports: { type: [dockerContainerPortSchema], default: undefined },
mounts: { type: [dockerContainerMountSchema], default: undefined },
exitCode: { type: Number, default: undefined },
},
{ _id: false }
);
-19
View File
@@ -13,7 +13,6 @@ const SERVICE_NAME = "checkService";
export interface ICheckService {
createChecks(checks: Check[]): Promise<Check[]>;
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
getChecksByMonitor(params: {
monitorId: string;
teamId: string;
@@ -135,24 +134,6 @@ export class CheckService implements ICheckService {
return check;
};
toStatusResponse = (check: Check) => {
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
monitorId: check.metadata.monitorId,
teamId: check.metadata.teamId,
type: check.metadata.type,
status: check.status,
code: check.statusCode,
message: check.message,
responseTime: check.responseTime,
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
payload:
check.metadata.type === "hardware"
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
: undefined,
};
return statusResponse;
};
getChecksByMonitor = async ({
monitorId,
teamId,
+8 -6
View File
@@ -45,12 +45,14 @@ const mapMemory = (memory?: CheckMemoryInfo): SnapshotMemoryInfo | undefined =>
};
const mapDisk = (disk?: CheckDiskInfo[]): SnapshotDiskInfo[] | undefined =>
disk?.map((entry) => ({
device: entry.device,
total_bytes: entry.total_bytes,
used_bytes: entry.used_bytes,
usage_percent: entry.usage_percent,
}));
disk
?.filter((entry) => entry != null)
.map((entry) => ({
device: entry.device,
total_bytes: entry.total_bytes,
used_bytes: entry.used_bytes,
usage_percent: entry.usage_percent,
}));
const mapHost = (host?: CheckHostInfo): SnapshotHostInfo | undefined =>
host && {
+47
View File
@@ -0,0 +1,47 @@
import type { DockerContainerInfo } from "@/domain/docker/docker.type.js";
import type { Monitor } from "@/domain/monitors/monitor.type.js";
export const DockerAlertReasons = ["stopped", "unhealthy"] as const;
export type DockerAlertReason = (typeof DockerAlertReasons)[number];
export interface DockerContainerBreach {
name: string;
reason: DockerAlertReason;
exitCode?: number;
}
export const isContainerStopped = (container: DockerContainerInfo): boolean => {
// Container with exit code 0 is probably a one shot container don't report stopped
return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0);
};
export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => {
return container.health === "unhealthy";
};
export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => {
const breaches: DockerContainerBreach[] = [];
for (const container of containers) {
if (monitor.dockerAlertOnStopped && isContainerStopped(container)) {
breaches.push({
name: container.name,
reason: "stopped",
exitCode: container.exitCode,
});
} else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) {
breaches.push({
name: container.name,
reason: "unhealthy",
});
}
}
return breaches;
};
export const describeContainerBreach = (breach: DockerContainerBreach): string => {
if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`;
if (breach.exitCode === undefined) {
return `${breach.name}: stopped`;
}
return `${breach.name}: stopped (exit code ${breach.exitCode})`;
};
+1
View File
@@ -55,6 +55,7 @@ export interface DockerContainerInfo {
startedAt?: string; // ISO date
ports?: DockerContainerPort[];
mounts?: DockerContainerMount[];
exitCode?: number;
}
export interface DockerContainerSummary {
@@ -1,6 +1,5 @@
const SERVICE_NAME = "incidentService";
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { MonitorStatusResponse } from "@/types/network.js";
import type { Check } from "@/domain/checks/check.type.js";
import { AppError } from "@/utils/AppError.js";
import { getDateForRange } from "@/utils/dataUtils.js";
import type { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
@@ -13,13 +12,9 @@ import type { INotificationMessageBuilder } from "@/domain/notifications/notific
import type { ILogger } from "@/utils/logger.js";
import { DateRange } from "@/types/query.js";
const SERVICE_NAME = "incidentService";
export interface IIncidentService {
handleIncident(
monitor: Monitor,
code: number,
decision: MonitorActionDecision,
monitorStatusResponse?: MonitorStatusResponse
): Promise<Incident | null>;
handleIncident(monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise<Incident | null>;
resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise<Incident>;
getIncidentsByTeam(
teamId: string,
@@ -58,12 +53,7 @@ export class IncidentService implements IIncidentService {
this.notificationMessageBuilder = notificationMessageBuilder;
}
handleIncident = async (
monitor: Monitor,
code: number,
decision: MonitorActionDecision,
monitorStatusResponse?: MonitorStatusResponse
): Promise<Incident | null> => {
handleIncident = async (monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise<Incident | null> => {
if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) {
return null;
}
@@ -74,13 +64,13 @@ export class IncidentService implements IIncidentService {
if (activeIncident) {
return activeIncident;
} else {
let statusCode = code;
let statusCode = check.statusCode;
let message: string | undefined;
// For threshold breaches, use 9999 status code and build descriptive message
if (decision.incidentReason === "threshold_breach") {
statusCode = 9999;
message = this.buildThresholdBreachMessage(monitor, monitorStatusResponse);
message = this.notificationMessageBuilder.buildThresholdBreachMessage(monitor, check, decision.thresholdBreaches);
}
const incident = {
@@ -105,20 +95,6 @@ export class IncidentService implements IIncidentService {
return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
};
private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string {
if (!monitorStatusResponse) {
return "Threshold breach detected";
}
const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse);
if (breaches.length === 0) {
return "Threshold breach detected";
}
return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", ");
}
resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => {
try {
if (!incidentId) {
@@ -307,6 +307,14 @@ const MonitorSchema = new Schema<MonitorDocument>(
type: Boolean,
default: false,
},
dockerAlertOnStopped: {
type: Boolean,
default: false,
},
dockerAlertOnUnhealthy: {
type: Boolean,
default: false,
},
dockerTlsCa: {
type: String,
},
@@ -468,6 +468,8 @@ class MongoMonitorsRepository implements IMonitorsRepository {
geoCheckLocations: doc.geoCheckLocations ?? [],
geoCheckInterval: doc.geoCheckInterval ?? 300000,
dockerLogsEnabled: doc.dockerLogsEnabled ?? false,
dockerAlertOnStopped: doc.dockerAlertOnStopped ?? false,
dockerAlertOnUnhealthy: doc.dockerAlertOnUnhealthy ?? false,
dockerTlsCa: doc.dockerTlsCa ?? undefined,
dockerTlsCert: doc.dockerTlsCert ?? undefined,
dockerTlsKeySet: doc.dockerTlsKeySet ?? false,
@@ -51,6 +51,22 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
const MonitorPaths: Record<MonitorType, string> = {
http: "uptime",
port: "uptime",
ping: "uptime",
game: "uptime",
grpc: "uptime",
websocket: "uptime",
dns: "uptime",
unknown: "uptime",
docker: "docker/host",
hardware: "infrastructure",
pagespeed: "pagespeed",
};
export const getMonitorPath = (type: MonitorType): string => MonitorPaths[type];
// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
// however far away the agent is. `unknown` is excluded because no request is made for it.
@@ -134,6 +150,8 @@ export interface Monitor {
geoCheckLocations?: GeoContinent[];
geoCheckInterval?: number;
dockerLogsEnabled?: boolean;
dockerAlertOnStopped?: boolean;
dockerAlertOnUnhealthy?: boolean;
dockerTlsCa?: string;
dockerTlsCert?: string;
// EncryptionService ciphertext. Normally excluded
@@ -222,3 +240,8 @@ export type GamesMap = Record<string, Game>;
export type MonitorScheduleFields = Pick<Monitor, "id" | "type" | "isActive" | "interval" | "geoCheckEnabled" | "geoCheckInterval">;
export type DockerContainerLogsResult = DockerLogPage;
export const HardwareMetricKeys = ["cpu", "memory", "disk", "temp"] as const;
export type HardwareMetricKey = (typeof HardwareMetricKeys)[number];
export type HardwareBreaches = Record<HardwareMetricKey, boolean>;
export type HardwareCounters = Record<HardwareMetricKey, number>;
@@ -1,5 +1,4 @@
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { HardwareMetricKeys, type HardwareBreaches, type HardwareMetricKey, type Monitor } from "@/domain/monitors/monitor.type.js";
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
import type {
NotificationMessage,
@@ -8,15 +7,12 @@ import type {
ThresholdBreach,
NotificationContent,
} from "@/domain/notifications/notification.type.js";
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
import type { Check } from "@/domain/checks/check.type.js";
export interface INotificationMessageBuilder {
buildMessage(
monitor: Monitor,
monitorStatusResponse: MonitorStatusResponse,
decision: MonitorActionDecision,
clientHost: string
): NotificationMessage;
extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[];
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage;
buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string;
}
const SERVICE_NAME = "NotificationMessageBuilder";
@@ -24,15 +20,10 @@ const SERVICE_NAME = "NotificationMessageBuilder";
export class NotificationMessageBuilder implements INotificationMessageBuilder {
static SERVICE_NAME = SERVICE_NAME;
buildMessage(
monitor: Monitor,
monitorStatusResponse: MonitorStatusResponse,
decision: MonitorActionDecision,
clientHost: string
): NotificationMessage {
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage {
const type = this.determineNotificationType(decision, monitor);
const severity = this.determineSeverity(type);
const content = this.buildContent(type, monitor, monitorStatusResponse);
const content = this.buildContent(type, monitor, check, decision);
return {
type,
@@ -48,7 +39,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
clientHost,
metadata: {
teamId: monitor.teamId,
notificationReason: decision.notificationReason || "status_change",
notificationReason: decision.notificationReason,
},
};
}
@@ -59,32 +50,29 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
return "monitor_down";
}
// Threshold breach (only if not down)
if (decision.notificationReason === "threshold_breach") {
return "threshold_breach";
const isDocker = monitor.type === "docker";
switch (decision.notificationReason) {
case "threshold_breach":
return isDocker ? "container_breach" : "threshold_breach";
case "threshold_resolved":
return isDocker ? "container_resolved" : "threshold_resolved";
default:
return "monitor_up";
}
// Recovery from threshold breach (only for hardware monitors)
if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") {
return "threshold_resolved";
}
// Standard recovery (up)
if (monitor.status === "up") {
return "monitor_up";
}
// Default to monitor_up for any other case
return "monitor_up";
}
private determineSeverity(type: NotificationType): NotificationSeverity {
switch (type) {
case "monitor_up":
return "success";
case "monitor_down":
return "critical";
case "container_breach":
return "warning";
case "threshold_breach":
return "warning";
case "monitor_up":
case "container_resolved":
return "success";
case "threshold_resolved":
return "success";
case "test":
@@ -94,34 +82,38 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
}
}
private buildContent(type: NotificationType, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent {
private buildContent(type: NotificationType, monitor: Monitor, check: Check, decision: MonitorActionDecision): NotificationContent {
switch (type) {
case "monitor_down":
return this.buildMonitorDownContent(monitor, monitorStatusResponse);
return this.buildMonitorDownContent(monitor, check);
case "monitor_up":
return this.buildMonitorUpContent(monitor);
case "threshold_breach":
return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<HardwareStatusPayload>);
return this.buildThresholdBreachContent(monitor, check, decision.thresholdBreaches);
case "threshold_resolved":
return this.buildThresholdResolvedContent(monitor);
case "container_breach":
return this.buildContainerBreachContent(monitor, check);
case "container_resolved":
return this.buildContainersRecoveredContent(monitor);
default:
return this.buildDefaultContent(monitor);
}
}
private buildMonitorDownContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent {
private buildMonitorDownContent(monitor: Monitor, check: Check): NotificationContent {
const title = `Monitor Down: ${monitor.name}`;
const summary = `Monitor "${monitor.name}" is currently down and unreachable.`;
const details = [`URL: ${monitor.url}`, `Status: Down`, `Type: ${monitor.type}`];
// Add response code if available
if (monitorStatusResponse.code) {
details.push(`Response Code: ${monitorStatusResponse.code}`);
if (check.statusCode) {
details.push(`Response Code: ${check.statusCode}`);
}
// Add error message if available
if (monitorStatusResponse.message) {
details.push(`Error: ${monitorStatusResponse.message}`);
if (check.message) {
details.push(`Error: ${check.message}`);
}
return {
@@ -145,12 +137,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}
private buildThresholdBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<HardwareStatusPayload>): NotificationContent {
private buildThresholdBreachContent(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): NotificationContent {
const title = `Threshold Exceeded: ${monitor.name}`;
const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`;
const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`];
const thresholds = this.extractThresholdBreaches(monitor, monitorStatusResponse);
const thresholds = this.extractThresholdBreaches(monitor, check, thresholdBreaches);
return {
title,
@@ -174,6 +166,21 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}
private buildContainerBreachContent(monitor: Monitor, check: Check): NotificationContent {
const lines = this.describeContainerBreaches(monitor, check);
const title = `Container Alert: ${monitor.name}`;
const summary = `${lines.length} container(s) on "${monitor.name}" need attention.`;
const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...lines];
return { title, summary, details, timestamp: new Date() };
}
private buildContainersRecoveredContent(monitor: Monitor): NotificationContent {
const title = `Containers Recovered: ${monitor.name}`;
const summary = `All containers on "${monitor.name}" are back to normal.`;
const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`];
return { title, summary, details, timestamp: new Date() };
}
private buildDefaultContent(monitor: Monitor): NotificationContent {
return {
title: `Monitor: ${monitor.name}`,
@@ -183,93 +190,57 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
};
}
public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<HardwareStatusPayload>): ThresholdBreach[] {
private describeBreach(metric: HardwareMetricKey, monitor: Monitor, check: Check): ThresholdBreach {
switch (metric) {
case "cpu": {
const currentValue = (check.cpu?.usage_percent ?? 0) * 100;
return { metric, currentValue, threshold: monitor.cpuAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
}
case "memory": {
const currentValue = (check.memory?.usage_percent ?? 0) * 100;
return { metric, currentValue, threshold: monitor.memoryAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
}
case "disk": {
const currentValue = Math.max(0, ...(check.disk ?? []).map((disk) => disk?.usage_percent ?? 0)) * 100;
return { metric, currentValue, threshold: monitor.diskAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
}
case "temp": {
const currentValue = Math.max(0, ...(check.cpu?.temperature ?? []));
return { metric, currentValue, threshold: monitor.tempAlertThreshold, unit: "°C", formattedValue: `${currentValue.toFixed(1)}°C` };
}
}
}
private extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] {
const breaches: ThresholdBreach[] = [];
// Check if this is a hardware monitor with threshold data
if (monitor.type !== "hardware" || !monitorStatusResponse.payload || typeof monitorStatusResponse.payload === "string") {
if (monitor.type !== "hardware" || thresholdBreaches === undefined) {
return breaches;
}
// Cast to HardwareStatusPayload type
const payload = monitorStatusResponse.payload;
const hardware = payload.data;
if (!hardware) {
return breaches;
}
// Note: usage_percent values in hardware payload are decimals (0-1)
if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && hardware.cpu?.usage_percent !== undefined) {
const cpuUsageDecimal = hardware.cpu.usage_percent;
const cpuPercent = cpuUsageDecimal * 100;
const threshold = monitor.cpuAlertThreshold;
if (cpuPercent > threshold) {
breaches.push({
metric: "cpu",
currentValue: cpuPercent,
threshold,
unit: "%",
formattedValue: `${cpuPercent.toFixed(1)}%`,
});
}
}
// Memory threshold breach
if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && hardware.memory?.usage_percent !== undefined) {
const memoryUsageDecimal = hardware.memory.usage_percent;
const memoryPercent = memoryUsageDecimal * 100;
const threshold = monitor.memoryAlertThreshold;
if (memoryPercent > threshold) {
breaches.push({
metric: "memory",
currentValue: memoryPercent,
threshold,
unit: "%",
formattedValue: `${memoryPercent.toFixed(1)}%`,
});
}
}
// Disk threshold breach
if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(hardware.disk)) {
// Find the highest disk usage
let maxDiskUsageDecimal = 0;
for (const disk of hardware.disk) {
if (disk.usage_percent !== undefined && disk.usage_percent > maxDiskUsageDecimal) {
maxDiskUsageDecimal = disk.usage_percent;
}
}
const maxDiskPercent = maxDiskUsageDecimal * 100;
const threshold = monitor.diskAlertThreshold;
if (maxDiskPercent > threshold) {
breaches.push({
metric: "disk",
currentValue: maxDiskPercent,
threshold,
unit: "%",
formattedValue: `${maxDiskPercent.toFixed(1)}%`,
});
}
}
// Temperature threshold breach
if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && hardware.cpu?.temperature) {
// Temperature is an array in cpu.temperature
const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature];
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
const threshold = monitor.tempAlertThreshold;
if (maxTemp > threshold) {
breaches.push({
metric: "temp",
currentValue: maxTemp,
threshold,
unit: "°C",
formattedValue: `${maxTemp.toFixed(1)}°C`,
});
}
for (const metric of HardwareMetricKeys) {
if (!thresholdBreaches[metric]) continue;
breaches.push(this.describeBreach(metric, monitor, check));
}
return breaches;
}
private describeContainerBreaches(monitor: Monitor, check: Check): string[] {
return findContainerBreaches(monitor, check.containers ?? []).map(describeContainerBreach);
}
public buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string {
if (monitor.type === "docker") {
const lines = this.describeContainerBreaches(monitor, check);
return lines.length > 0 ? lines.join(", ") : "Container alert";
}
const breaches = this.extractThresholdBreaches(monitor, check, thresholdBreaches);
if (breaches.length === 0) {
return "Threshold breach detected";
}
return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", ");
}
}
@@ -1,6 +1,5 @@
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { Notification } from "@/domain/notifications/notification.type.js";
import type { MonitorStatusResponse } from "@/types/network.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
@@ -10,6 +9,7 @@ import type { ISettingsService } from "@/domain/app-settings/app-settings.servic
import { ILogger } from "@/utils/logger.js";
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
import type { NotificationChannel } from "@/domain/notifications/notification.type.js";
import type { Check } from "@/domain/checks/check.type.js";
export type NotificationProviderRegistry = Record<NotificationChannel, INotificationProvider>;
@@ -19,7 +19,7 @@ export interface INotificationsService {
findNotificationsByTeamId: (teamId: string) => Promise<Notification[]>;
updateById(id: string, teamId: string, updateData: Partial<Notification>): Promise<Notification>;
deleteById: (id: string, teamId: string) => Promise<Notification>;
handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise<boolean>;
handleNotifications: (monitor: Monitor, check: Check, decision: MonitorActionDecision) => Promise<boolean>;
sendTestNotification: (notification: Partial<Notification>) => Promise<boolean>;
testAllNotifications: (notificationIds: string[]) => Promise<boolean>;
@@ -63,7 +63,7 @@ export class NotificationsService implements INotificationsService {
private send = async (
notification: Notification,
monitor: Monitor,
monitorStatusResponse: MonitorStatusResponse,
check: Check,
decision: MonitorActionDecision,
notificationMessage: NotificationMessage | undefined
): Promise<boolean> => {
@@ -89,16 +89,16 @@ export class NotificationsService implements INotificationsService {
return await provider.sendMessage(notification, notificationMessage);
};
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
private sendNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => {
const notificationIds = monitor.notifications ?? [];
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
// Build notification message once for all notifications
const settings = this.settingsService.getSettings();
const clientHost = settings.clientHost || "Host not defined";
const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost);
const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, check, decision, clientHost);
const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage));
const tasks = notifications.map((notification) => this.send(notification, monitor, check, decision, notificationMessage));
const outcomes = await Promise.all(tasks);
const succeeded = outcomes.filter(Boolean).length;
@@ -114,13 +114,13 @@ export class NotificationsService implements INotificationsService {
return succeeded === notifications.length;
};
handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
handleNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => {
if (!decision.shouldSendNotification) {
return false;
}
// Send notifications based on decision
return await this.sendNotifications(monitor, monitorStatusResponse, decision);
return await this.sendNotifications(monitor, check, decision);
};
sendTestNotification = async (notification: Partial<Notification>) => {
@@ -148,7 +148,17 @@ export class NotificationsService implements INotificationsService {
testAllNotifications = async (notificationIds: string[]) => {
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
const tasks = notifications.map((notification) => this.sendTestNotification(notification));
const tasks = notifications.map((notification) =>
this.sendTestNotification(notification).catch((error: unknown) => {
this.logger.warn({
message: `Test notification failed: ${error instanceof Error ? error.message : "Unknown error"}`,
service: SERVICE_NAME,
method: "testAllNotifications",
details: { notificationId: notification.id, type: notification.type },
});
return false;
})
);
const outcomes = await Promise.all(tasks);
const succeeded = outcomes.filter(Boolean).length;
const failed = outcomes.length - succeeded;
@@ -1,3 +1,6 @@
import type { HardwareMetricKey } from "@/domain/monitors/monitor.type.js";
import { MonitorActionDecision } from "@/worker/worker.interface.js";
export const NotificationChannels = [
"email",
"slack",
@@ -71,7 +74,8 @@ export interface AlertDiscordPayload {
* Part of notification system unification effort
*/
export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test";
export type NotificationType =
"monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test";
export type NotificationSeverity = "critical" | "warning" | "info" | "success";
@@ -84,7 +88,7 @@ export interface MonitorInfo {
}
export interface ThresholdBreach {
metric: "cpu" | "memory" | "disk" | "temp";
metric: HardwareMetricKey;
currentValue: number;
threshold: number;
unit: string;
@@ -116,6 +120,6 @@ export interface NotificationMessage {
clientHost: string;
metadata: {
teamId: string;
notificationReason: string;
notificationReason: MonitorActionDecision["notificationReason"];
};
}
@@ -1,6 +1,7 @@
const SERVICE_NAME = "EmailProvider";
import type { Notification } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import { AppError } from "@/utils/AppError.js";
import { buildTestEmail } from "@/domain/notifications/providers/utils.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import type { ILogger } from "@/utils/logger.js";
@@ -40,11 +41,13 @@ export class EmailProvider extends NotificationProvider {
return true;
} catch (error: unknown) {
this.logger.warn({
message: "Email test alert failed",
message: `Email test alert failed: ${error instanceof Error ? error.message : "Unknown error"}`,
service: SERVICE_NAME,
method: "sendTestAlert",
details: error instanceof AppError ? error.details : undefined,
stack: error instanceof Error ? error.stack : undefined,
});
if (error instanceof AppError) throw error;
return false;
}
}
@@ -73,9 +76,10 @@ export class EmailProvider extends NotificationProvider {
return true;
} catch (error: unknown) {
this.logger.warn({
message: "Email notification failed",
message: `Email notification failed: ${error instanceof Error ? error.message : "Unknown error"}`,
service: SERVICE_NAME,
method: "sendMessage",
details: error instanceof AppError ? error.details : undefined,
stack: error instanceof Error ? error.stack : undefined,
});
return false;
@@ -92,6 +96,10 @@ export class EmailProvider extends NotificationProvider {
return `Monitor ${message.monitor.name} threshold exceeded`;
case "threshold_resolved":
return `Monitor ${message.monitor.name} thresholds resolved`;
case "container_breach":
return `Monitor ${message.monitor.name} container alert`;
case "container_resolved":
return `Monitor ${message.monitor.name} containers recovered`;
default:
return `Alert: ${message.monitor.name}`;
}
@@ -3,7 +3,7 @@ import got from "got";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { AlertMatrixPayload, Notification } from "@/domain/notifications/notification.type.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import { randomUUID } from "crypto";
export class MatrixProvider extends NotificationProvider {
@@ -172,10 +172,10 @@ export class MatrixProvider extends NotificationProvider {
// Incident link (if incident exists)
if (message.content.incident) {
const incidentUrl = `${message.clientHost}/infrastructure/${message.monitor.id}`;
const url = incidentUrl(message);
plainLines.push("");
plainLines.push(`View Incident: ${incidentUrl}`);
htmlLines.push(`<p><a href="${this.escapeHtml(incidentUrl)}">View Incident</a></p>`);
plainLines.push(`View Incident: ${url}`);
htmlLines.push(`<p><a href="${this.escapeHtml(url)}">View Incident</a></p>`);
}
// Footer with timestamp
@@ -2,7 +2,7 @@ const SERVICE_NAME = "NtfyProvider";
import type { Notification } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import got from "got";
export class NtfyProvider extends NotificationProvider {
@@ -142,7 +142,7 @@ export class NtfyProvider extends NotificationProvider {
}
if (message.content.incident) {
lines.push("", `${message.clientHost}/infrastructure/${message.monitor.id}`);
lines.push("", incidentUrl(message));
}
return lines.join("\n");
@@ -75,7 +75,8 @@ export class PagerDutyProvider extends NotificationProvider {
private buildPagerDutyPayload(notification: Notification, message: NotificationMessage): AlertPagerDutyPayload {
// Map our notification type to PagerDuty event_action
const eventAction = message.type === "monitor_up" || message.type === "threshold_resolved" ? "resolve" : "trigger";
const eventAction =
message.type === "monitor_up" || message.type === "threshold_resolved" || message.type === "container_resolved" ? "resolve" : "trigger";
// Map severity to PagerDuty severity levels
const severityMap: Record<string, string> = {
@@ -2,7 +2,7 @@ const SERVICE_NAME = "PushoverProvider";
import type { Notification } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import got from "got";
export class PushoverProvider extends NotificationProvider {
@@ -101,7 +101,7 @@ export class PushoverProvider extends NotificationProvider {
if (message.content.incident) {
lines.push("");
lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`);
lines.push(`View Incident: ${incidentUrl(message)}`);
}
return lines.join("\n");
@@ -1,5 +1,5 @@
const SERVICE_NAME = "RocketChatProvider";
import { supportsUptimeDetails, type MonitorType } from "@/domain/monitors/monitor.type.js";
import { getMonitorPath, type MonitorType } from "@/domain/monitors/monitor.type.js";
import type { Notification, NotificationMessage } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
@@ -139,19 +139,7 @@ export class RocketChatProvider extends NotificationProvider {
return undefined;
}
if (message.monitor.type === "hardware") {
return `${clientHost}/infrastructure/${message.monitor.id}`;
}
if (message.monitor.type === "pagespeed") {
return `${clientHost}/pagespeed/${message.monitor.id}`;
}
if (supportsUptimeDetails(message.monitor.type as MonitorType)) {
return `${clientHost}/uptime/${message.monitor.id}`;
}
return undefined;
return `${clientHost}/${getMonitorPath(message.monitor.type as MonitorType)}/${message.monitor.id}`;
}
private severityColor(severity: NotificationMessage["severity"]): string {
@@ -1,7 +1,7 @@
const SERVICE_NAME = "SignalgridProvider";
import type { Notification, NotificationMessage, NotificationSeverity } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import got from "got";
export class SignalgridProvider extends NotificationProvider {
@@ -115,7 +115,7 @@ export class SignalgridProvider extends NotificationProvider {
if (message.content.incident) {
lines.push("");
lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`);
lines.push(`View Incident: ${incidentUrl(message)}`);
}
return lines.join("\n");
@@ -2,7 +2,7 @@ const SERVICE_NAME = "SlackProvider";
import type { Notification } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import got, { HTTPError } from "got";
export class SlackProvider extends NotificationProvider {
@@ -161,7 +161,7 @@ export class SlackProvider extends NotificationProvider {
text: "View Incident",
emoji: true,
},
url: `${message.clientHost}/infrastructure/${message.monitor.id}`,
url: incidentUrl(message),
style: "primary",
},
],
@@ -2,7 +2,7 @@ const SERVICE_NAME = "TeamsProvider";
import type { Notification } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import got, { HTTPError } from "got";
// Types for Adaptive Card elements
@@ -254,7 +254,7 @@ export class TeamsProvider extends NotificationProvider {
actions.push({
type: "Action.OpenUrl",
title: "View Incident",
url: `${message.clientHost}/incidents/${message.content.incident.id}`,
url: incidentUrl(message),
});
}
@@ -2,7 +2,7 @@ const SERVICE_NAME = "TelegramProvider";
import type { Notification } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import got from "got";
export class TelegramProvider extends NotificationProvider {
@@ -101,7 +101,7 @@ export class TelegramProvider extends NotificationProvider {
if (message.content.incident) {
lines.push("");
lines.push(`<a href="${message.clientHost}/incidents/${message.monitor.id}">View Incident</a>`);
lines.push(`<a href="${incidentUrl(message)}">View Incident</a>`);
}
return lines.join("\n");
@@ -1,4 +1,8 @@
import { IEmailService } from "@/service/emailService.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
export const incidentUrl = (message: NotificationMessage): string => `${message.clientHost}/incidents/${message.monitor.id}`;
export const buildTestEmail = async (emailService: IEmailService) => {
const context = { testName: "Monitoring System" };
const html = await emailService.buildEmail("testEmailTemplate", context);
@@ -2,7 +2,7 @@ const SERVICE_NAME = "WebhookProvider";
import type { Notification } from "@/domain/notifications/notification.type.js";
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
import got from "got";
export class WebhookProvider extends NotificationProvider {
@@ -74,7 +74,7 @@ export class WebhookProvider extends NotificationProvider {
// Incident link
if (message.content.incident) {
lines.push(`[View Incident](${message.clientHost}/infrastructure/${message.monitor.id})`);
lines.push(`[View Incident](${incidentUrl(message)})`);
}
// Return webhook payload with both text and structured data
+33 -3
View File
@@ -12,6 +12,23 @@ const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const SERVICE_NAME = "EmailService";
const SMTP_FAILURE_REASONS: Record<string, string> = {
EAUTH: "rejected the login. Check the user name and password",
ENOAUTH: "requires a login. Set the user name and password",
EDNS: "could not be resolved. Check the host name",
ECONNECTION: "refused the connection. Check the host and port",
ESOCKET: "refused the connection. Check the host and port",
ETIMEDOUT: "did not respond. Check the port and the secure setting",
ETLS: "failed the TLS handshake. Check the secure and TLS settings",
EPROTOCOL: "sent an invalid response. Check the port and the secure setting",
};
const describeSmtpFailure = (error: unknown, host: string, port: number): string => {
const code = typeof error === "object" && error !== null && "code" in error ? String((error as { code: unknown }).code) : undefined;
const reason = (code !== undefined && SMTP_FAILURE_REASONS[code]) || "failed verification";
return `SMTP host ${host}:${port} ${reason}.`;
};
type MjmlFn = typeof mjml2html;
type FileSystem = typeof fs;
type PathModule = typeof path;
@@ -123,6 +140,15 @@ export class EmailService implements IEmailService {
systemEmailRejectUnauthorized,
} = config;
if (!systemEmailHost) {
throw new AppError({
message: "Email is not configured. Set the system email host in settings.",
status: 400,
service: SERVICE_NAME,
method: "sendEmail",
});
}
const emailConfig = {
host: systemEmailHost,
port: Number(systemEmailPort),
@@ -146,17 +172,20 @@ export class EmailService implements IEmailService {
try {
await this.transporter.verify();
} catch (error: unknown) {
const cause = error instanceof Error ? error.message : "Unknown error";
this.logger.warn({
message: "Email transporter verification failed",
message: `Email transporter verification failed: ${cause}`,
service: SERVICE_NAME,
method: "verifyTransporter",
details: { host: systemEmailHost, port: emailConfig.port },
stack: error instanceof Error ? error.stack : undefined,
});
throw new AppError({
message: "Email transporter verification failed",
message: describeSmtpFailure(error, systemEmailHost, emailConfig.port),
status: 502,
service: SERVICE_NAME,
method: "sendEmail",
details: { cause: error instanceof Error ? error.message : "Unknown error" },
details: { cause },
});
}
@@ -180,6 +209,7 @@ export class EmailService implements IEmailService {
});
throw new AppError({
message: "Failed to send email",
status: 502,
service: SERVICE_NAME,
method: "sendEmail",
details: { cause: error instanceof Error ? error.message : "Unknown error" },
@@ -42,6 +42,8 @@ const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
const DOCKER_TLS_DEFAULT_PORT = 2376;
const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]);
const EXITED_STATUS = /^Exited \((\d+)\)/;
const HEALTH_STATUS = /\((healthy|unhealthy|health: starting)\)$/;
export interface DockerError extends Error {
statusCode?: number;
@@ -140,6 +142,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none";
}
private exitCodeFromStatus = (status: string): number | undefined => {
const match = EXITED_STATUS.exec(status);
return match ? Number(match[1]) : undefined;
};
private healthFromStatus = (status: string): DockerHealthStatus => {
const match = HEALTH_STATUS.exec(status);
if (!match) return "none";
return match[1] === "health: starting" ? "starting" : (match[1] as DockerHealthStatus);
};
private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary {
return {
total: containers.length,
@@ -273,7 +286,10 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
info.health = this.toHealthStatus(inspectResult.value.State?.Health?.Status);
info.ports = this.toPorts(inspectResult.value.NetworkSettings?.Ports);
info.mounts = this.toMounts(inspectResult.value.Mounts);
info.exitCode = inspectResult.value.State?.ExitCode;
} else {
info.exitCode = this.exitCodeFromStatus(summary.Status);
info.health = this.healthFromStatus(summary.Status);
this.logger.warn({
message: `Failed to inspect container ${info.name}`,
service: SERVICE_NAME,
+91 -132
View File
@@ -1,49 +1,27 @@
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js";
import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js";
import type {
DockerStatusPayload,
GameStatusPayload,
GrpcStatusPayload,
HardwareStatusPayload,
HttpStatusPayload,
MonitorStatusResponse,
PageSpeedStatusPayload,
PingStatusPayload,
PortStatusPayload,
StatusChangeResult,
} from "@/types/network.js";
import {
HardwareMetricKeys,
MonitorStatuses,
type HardwareBreaches,
type HardwareCounters,
type Monitor,
type MonitorStatus,
} from "@/domain/monitors/monitor.type.js";
import type { StatusChangeResult } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import type { HardwareStatusMetrics } from "@/types/network.js";
import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js";
import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js";
import { findContainerBreaches } from "@/domain/docker/docker-alert.js";
const SERVICE_NAME = "StatusService";
const HARDWARE_ALERT_COUNTER_START = 5;
const HARDWARE_METRIC_KEYS = ["cpu", "memory", "disk", "temp"] as const;
type HardwareMetricKey = (typeof HARDWARE_METRIC_KEYS)[number];
type HardwareBreaches = Record<HardwareMetricKey, boolean>;
type HardwareCounters = Record<HardwareMetricKey, number>;
export interface IStatusService {
updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise<boolean>;
updateMonitorStatus(
statusResponse: MonitorStatusResponse<
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
>,
check: Check,
monitor: Monitor
): Promise<StatusChangeResult>;
updateMonitorStatus(check: Check, monitor: Monitor): Promise<StatusChangeResult>;
}
export class StatusService implements IStatusService {
@@ -58,32 +36,19 @@ export class StatusService implements IStatusService {
this.monitorStatsRepository = monitorStatsRepository;
}
async updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse) {
private tryUpdateRunningStats = async (check: Check, monitor: Monitor) => {
try {
await this.monitorStatsRepository.updateByMonitorId(monitor.id, {
status: networkResponse.status === true,
responseTime: networkResponse.responseTime ?? 0,
status: check.status === true,
responseTime: check.responseTime ?? 0,
now: Date.now(),
});
return true;
} catch (error: unknown) {
this.logger.error({
service: SERVICE_NAME,
message: error instanceof Error ? error.message : "Unknown error",
method: "updateRunningStats",
stack: error instanceof Error ? error.stack : undefined,
});
return false;
}
}
private tryUpdateRunningStats = async (monitor: Monitor, statusResponse: MonitorStatusResponse) => {
const statsOk = await this.updateRunningStats(monitor, statusResponse);
if (!statsOk) {
this.logger.warn({
service: SERVICE_NAME,
method: "updateMonitorStatus",
message: `Stats update failed for monitor ${monitor.id}`,
message: error instanceof Error ? error.message : "Unknown error",
method: "tryUpdateRunningStats",
stack: error instanceof Error ? error.stack : undefined,
});
}
};
@@ -135,7 +100,7 @@ export class StatusService implements IStatusService {
// Update counters: decrement (floored at 0) if breached, reset to start otherwise.
const nextCounters = { ...counters };
for (const key of HARDWARE_METRIC_KEYS) {
for (const key of HardwareMetricKeys) {
nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START;
}
@@ -147,8 +112,8 @@ export class StatusService implements IStatusService {
if (!reachabilityDown) {
// A counter can only reach zero via the decrement path, which only runs when that
// metric is currently breaching — so anyCounterZero already implies anyBreached.
const anyCounterZero = HARDWARE_METRIC_KEYS.some((k) => nextCounters[k] === 0);
const allNormal = HARDWARE_METRIC_KEYS.every((k) => !breaches[k]);
const anyCounterZero = HardwareMetricKeys.some((k) => nextCounters[k] === 0);
const allNormal = HardwareMetricKeys.every((k) => !breaches[k]);
if (anyCounterZero && currentStatus !== "breached") {
nextStatus = "breached";
@@ -162,26 +127,29 @@ export class StatusService implements IStatusService {
return { nextStatus, transitioned, breaches, nextCounters };
};
updateMonitorStatus = async (
statusResponse: MonitorStatusResponse<
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
>,
check: Check,
monitor: Monitor
): Promise<StatusChangeResult> => {
private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => {
const { currentStatus, reachabilityDown, breaching } = params;
let nextStatus: MonitorStatus = currentStatus;
let transitioned = false;
if (!reachabilityDown) {
if (breaching && currentStatus !== "breached") {
nextStatus = "breached";
transitioned = true;
} else if (!breaching && currentStatus === "breached") {
nextStatus = "up";
transitioned = true;
}
}
return { nextStatus, transitioned };
};
updateMonitorStatus = async (check: Check, monitor: Monitor): Promise<StatusChangeResult> => {
try {
const { status, code } = statusResponse;
const { status } = check;
// Update running stats
await this.tryUpdateRunningStats(monitor, statusResponse);
await this.tryUpdateRunningStats(check, monitor);
const prevStatus = monitor.status;
const checkSnapshot = toCheckSnapshot(check);
@@ -206,69 +174,62 @@ export class StatusService implements IStatusService {
statusChanged = newStatus === "down";
}
// Not enough data points yet — record the check and return
if (projectedWindow.length < monitor.statusWindowSize) {
const updated = await this.monitorsRepository.updateStatusWindowAndChecks(
monitor.id,
monitor.teamId,
check.status,
checkSnapshot,
monitor.statusWindowSize,
MAX_RECENT_CHECKS,
patch
);
return {
monitor: updated,
statusChanged,
prevStatus,
code,
timestamp: Date.now(),
};
}
// First evaluate reachability status changes, which apply to all monitor types
// and take precedence over hardware breaches.
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
if (reachabilityResult.transitioned) {
newStatus = reachabilityResult.nextStatus;
statusChanged = true;
}
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
let thresholdBreaches: HardwareBreaches | undefined;
const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined;
if (monitor.type === "hardware" && hardwarePayload?.data) {
const hardware = this.computeHardwareStatus({
currentStatus: newStatus,
reachabilityDown: newStatus === "down",
metrics: hardwarePayload.data,
thresholds: {
cpu: monitor.cpuAlertThreshold,
memory: monitor.memoryAlertThreshold,
disk: monitor.diskAlertThreshold,
temp: monitor.tempAlertThreshold,
},
counters: {
cpu: monitor.cpuAlertCounter,
memory: monitor.memoryAlertCounter,
disk: monitor.diskAlertCounter,
temp: monitor.tempAlertCounter,
},
});
patch.cpuAlertCounter = hardware.nextCounters.cpu;
patch.memoryAlertCounter = hardware.nextCounters.memory;
patch.diskAlertCounter = hardware.nextCounters.disk;
patch.tempAlertCounter = hardware.nextCounters.temp;
thresholdBreaches = hardware.breaches;
if (hardware.transitioned) {
newStatus = hardware.nextStatus;
if (projectedWindow.length >= monitor.statusWindowSize) {
// First evaluate reachability status changes, which apply to all monitor types
// and take precedence over hardware breaches.
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
if (reachabilityResult.transitioned) {
newStatus = reachabilityResult.nextStatus;
statusChanged = true;
}
}
patch.status = newStatus;
// Evaluate hardware threshold breaches
if (monitor.type === "hardware") {
const hardware = this.computeHardwareStatus({
currentStatus: newStatus,
reachabilityDown: newStatus === "down",
metrics: check,
thresholds: {
cpu: monitor.cpuAlertThreshold,
memory: monitor.memoryAlertThreshold,
disk: monitor.diskAlertThreshold,
temp: monitor.tempAlertThreshold,
},
counters: {
cpu: monitor.cpuAlertCounter,
memory: monitor.memoryAlertCounter,
disk: monitor.diskAlertCounter,
temp: monitor.tempAlertCounter,
},
});
patch.cpuAlertCounter = hardware.nextCounters.cpu;
patch.memoryAlertCounter = hardware.nextCounters.memory;
patch.diskAlertCounter = hardware.nextCounters.disk;
patch.tempAlertCounter = hardware.nextCounters.temp;
thresholdBreaches = hardware.breaches;
if (hardware.transitioned) {
newStatus = hardware.nextStatus;
statusChanged = true;
}
}
if (monitor.type === "docker" && check.containers) {
const breaches = findContainerBreaches(monitor, check.containers);
const docker = this.computeDockerStatus({
currentStatus: newStatus,
reachabilityDown: newStatus === "down",
breaching: breaches.length > 0,
});
if (docker.transitioned) {
newStatus = docker.nextStatus;
statusChanged = true;
}
}
patch.status = newStatus;
}
// Single atomic write: push arrays + set status/counters
const updated = await this.monitorsRepository.updateStatusWindowAndChecks(
@@ -285,8 +246,6 @@ export class StatusService implements IStatusService {
monitor: updated,
statusChanged,
prevStatus,
code,
timestamp: new Date().getTime(),
thresholdBreaches,
};
} catch (error: unknown) {
+2 -9
View File
@@ -10,7 +10,7 @@ import type {
ILighthouseAudit,
} from "@/domain/checks/check.type.js";
import { DockerContainerInfo, DockerContainerLogs, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { DnsRecordType, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js";
import type { DnsRecordType, HardwareBreaches, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js";
import type { QueryResult } from "gamedig";
@@ -182,14 +182,7 @@ export type StatusChangeResult = {
monitor: Monitor;
statusChanged: boolean;
prevStatus: MonitorStatus;
code: number;
timestamp: number;
thresholdBreaches?: {
cpu: boolean;
memory: boolean;
disk: boolean;
temp: boolean;
};
thresholdBreaches?: HardwareBreaches;
};
export type MonitorStatusResponseOverrides<T> = Partial<Omit<MonitorStatusResponse<T>, "monitorId" | "teamId" | "type">>;
+12 -15
View File
@@ -60,21 +60,18 @@ class Logger implements ILogger {
if (formattedDetails) msg += ` (details: ${formattedDetails})`;
if (typeof stack === "string") {
const stackTrace = stack
.split("\n")
.slice(1) // Remove first line (error message)
.map((line: string) => {
const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/);
if (match) {
return {
function: match[1],
file: match[2],
line: parseInt(match[3] ?? "0", 10),
column: parseInt(match[4] ?? "0", 10),
};
}
return line.trim();
});
const stackTrace = stack.split("\n").map((line: string) => {
const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/);
if (match) {
return {
function: match[1],
file: match[2],
line: parseInt(match[3] ?? "0", 10),
column: parseInt(match[4] ?? "0", 10),
};
}
return line.trim();
});
msg += ` (stack: ${JSON.stringify(stackTrace, null, 2)})`;
}
@@ -8,6 +8,6 @@ export class IncidentReactor implements IMonitorReactor {
constructor(private incidentService: IIncidentService) {}
react = async (evaluation: MonitorEvaluation) => {
await this.incidentService.handleIncident(evaluation.monitor, evaluation.statusChange.code, evaluation.decision, evaluation.status);
await this.incidentService.handleIncident(evaluation.monitor, evaluation.decision, evaluation.check);
};
}
@@ -7,7 +7,6 @@ export class NotificationReactor implements IMonitorReactor {
constructor(private notificationService: INotificationsService) {}
react = async (evaluation: MonitorEvaluation) => {
if (!evaluation.decision.shouldSendNotification) return;
await this.notificationService.handleNotifications(evaluation.monitor, evaluation.status, evaluation.decision);
await this.notificationService.handleNotifications(evaluation.monitor, evaluation.check, evaluation.decision);
};
}
+3 -11
View File
@@ -1,7 +1,6 @@
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { HardwareBreaches, Monitor } from "@/domain/monitors/monitor.type.js";
import { Check } from "@/domain/checks/check.type.js";
import { Job, JobType } from "@/domain/jobs/job.type.js";
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
import type { QueueMode } from "@/domain/app-settings/app-settings.type.js";
@@ -12,20 +11,13 @@ export interface MonitorActionDecision {
shouldResolveIncident: boolean;
shouldSendNotification: boolean;
incidentReason: "status_down" | "threshold_breach" | null;
notificationReason: "status_change" | "threshold_breach" | null;
thresholdBreaches?: {
cpu?: boolean;
memory?: boolean;
disk?: boolean;
temp?: boolean;
};
notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null;
thresholdBreaches?: HardwareBreaches;
}
export type MonitorEvaluation = {
monitor: Monitor;
status: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>; // raw result from networkService.requestStatus
check: Check;
statusChange: StatusChangeResult; // from statusService.updateMonitorStatus
decision: MonitorActionDecision;
};
+24 -43
View File
@@ -3,7 +3,6 @@ import { ICheckService } from "@/domain/checks/check.service.js";
import { Check } from "@/domain/checks/check.type.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
import type { EgressStatus } from "@/domain/egress/egress.type.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
@@ -21,7 +20,13 @@ import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js";
const SERVICE_NAME = "WorkerPipeline";
const NO_ACTION: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
// The three entry points: the queue calls the two handlers, the buffer calls ingestChecks.
// produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer.
export interface IWorkerPipeline {
@@ -128,7 +133,7 @@ export class WorkerPipeline implements IWorkerPipeline {
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
const egressStatus = this.isTransportFailure(monitor, status) ? await this.deps.egressService.assessAfterFailure() : null;
if (egressStatus !== null) {
check.egressStatus = egressStatus;
}
@@ -171,20 +176,6 @@ export class WorkerPipeline implements IWorkerPipeline {
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
}
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
try {
return await this.deps.egressService.assessAfterFailure();
} catch (error: unknown) {
this.deps.logger.warn({
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
service: SERVICE_NAME,
method: "assessEgress",
});
return null;
}
}
// ****************************************************************
// Stage 2: Ingestion
// Called by the buffer, checks are inserted and evaluation jobs created
@@ -240,8 +231,7 @@ export class WorkerPipeline implements IWorkerPipeline {
let current = monitor;
for (const check of checks) {
const status = this.deps.checkService.toStatusResponse(check);
const evaluation = await this.evaluateCheck(status, check, current);
const evaluation = await this.evaluateCheck(check, current);
await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications
const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]);
if (!leaseHeld) {
@@ -253,7 +243,7 @@ export class WorkerPipeline implements IWorkerPipeline {
});
return;
}
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
current = evaluation.monitor; // fresh statusWindow/status/counters for the next check
}
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
@@ -272,48 +262,34 @@ export class WorkerPipeline implements IWorkerPipeline {
// A check taken while the instance had no outbound connectivity says nothing about the target.
// It is left out of the status window, running stats and monitor status entirely, so that no incident
// opens for it and no spurious "resolved"/"up" fires once egress returns.
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
private skipDegradedEgressCheck = (check: Check, monitor: Monitor): MonitorEvaluation => {
this.deps.logger.debug({
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
service: SERVICE_NAME,
method: "evaluateCheck",
});
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
return {
monitor,
status,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
decision,
decision: { ...NO_ACTION },
};
};
evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
evaluateCheck = async (check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
if (check.egressStatus === "degraded") {
return this.skipDegradedEgressCheck(status, check, monitor);
return this.skipDegradedEgressCheck(check, monitor);
}
const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor);
const statusChange = await this.deps.statusService.updateMonitorStatus(check, monitor);
const decision = this.decide(statusChange);
return { monitor: statusChange.monitor, status, check, statusChange, decision };
return { monitor: statusChange.monitor, check, decision };
};
private decide = (statusChange: StatusChangeResult): MonitorActionDecision => {
const { monitor, statusChanged, prevStatus } = statusChange;
const decision: MonitorActionDecision = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
const decision: MonitorActionDecision = { ...NO_ACTION };
if (!statusChanged) return decision;
if (monitor.status === "down") {
@@ -322,11 +298,16 @@ export class WorkerPipeline implements IWorkerPipeline {
decision.incidentReason = "status_down";
decision.notificationReason = "status_change";
} else if (monitor.status === "breached") {
if (statusChange.thresholdBreaches) decision.thresholdBreaches = statusChange.thresholdBreaches;
decision.shouldCreateIncident = true;
decision.shouldSendNotification = true;
decision.incidentReason = "threshold_breach";
decision.notificationReason = "threshold_breach";
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
} else if (monitor.status === "up" && prevStatus === "breached") {
decision.shouldResolveIncident = true;
decision.shouldSendNotification = true;
decision.notificationReason = "threshold_resolved";
} else if (monitor.status === "up" && prevStatus === "down") {
decision.shouldResolveIncident = true;
decision.shouldSendNotification = true;
decision.notificationReason = "status_change";
+16 -28
View File
@@ -5,17 +5,17 @@ import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts";
import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts";
import { StatusService } from "../../src/service/statusService.ts";
import { IncidentService } from "../../src/domain/incidents/incident.service.ts";
import { CheckService } from "../../src/domain/checks/check.service.ts";
import { InMemoryMonitorsRepository } from "./InMemoryMonitorsRepository.ts";
import { InMemoryIncidentsRepository } from "./InMemoryIncidentsRepository.ts";
import { createMockLogger } from "./createMockLogger.ts";
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
import type { MonitorStatusResponse } from "../../src/types/network.ts";
import type { Check } from "../../src/domain/checks/check.type.ts";
import type { MonitorActionDecision } from "../../src/worker/worker.interface.ts";
import type { MonitorStatusResponse } from "../../src/types/network.ts";
import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts";
import type { EgressStatus } from "../../src/domain/egress/egress.type.ts";
let checkCounter = 0;
export const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "mon-1",
@@ -45,20 +45,6 @@ export const makeStatusResponse = (status: boolean, code: number): MonitorStatus
responseTime: status ? 150 : 0,
});
export const makeCheck = (status: boolean, code: number): Check => {
const now = new Date().toISOString();
return {
id: `check-${++checkCounter}`,
metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" },
status,
statusCode: code,
responseTime: status ? 150 : 0,
message: status ? "OK" : "Service Unavailable",
createdAt: now,
updatedAt: now,
};
};
const createStubMonitorStatsRepo = () => ({
findByMonitorId: jest.fn().mockRejectedValue(new Error("no stats")),
create: jest.fn().mockResolvedValue({}),
@@ -77,7 +63,7 @@ export interface HeartbeatTestHarness {
networkService: { requestStatus: jest.Mock };
bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock };
maintenanceWindowsRepo: { findByMonitorId: jest.Mock };
messageBuilder: { extractThresholdBreaches: jest.Mock };
messageBuilder: { buildThresholdBreachMessage: jest.Mock };
egressService: { assessAfterFailure: jest.Mock };
heartbeatJob: (monitor: Monitor) => Promise<void>;
setNextResponse: (status: boolean, code: number) => void;
@@ -86,8 +72,6 @@ export interface HeartbeatTestHarness {
}
export function createHeartbeatTestHarness(): HeartbeatTestHarness {
checkCounter = 0;
const monitorsRepo = new InMemoryMonitorsRepository();
const incidentsRepo = new InMemoryIncidentsRepository();
const logger = createMockLogger() as any;
@@ -95,7 +79,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
const statusService = new StatusService(logger, monitorsRepo as any, createStubMonitorStatsRepo() as any);
const messageBuilder = { extractThresholdBreaches: jest.fn().mockReturnValue([]) };
const messageBuilder = { buildThresholdBreachMessage: jest.fn().mockReturnValue("") };
const incidentService = new IncidentService(logger, incidentsRepo, monitorsRepo as any, { findById: jest.fn() } as any, messageBuilder as any);
const notificationsService = { handleNotifications: jest.fn().mockResolvedValue(true) };
@@ -111,11 +95,9 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
return Promise.resolve(makeStatusResponse(nextStatus, nextCode));
}),
};
const checkService = {
toCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => {
return makeCheck(response.status, response.code);
}),
};
// The real mapper, so the stored check carries the hardware/docker payload the evaluator reads.
// Only toCheck is reached; the repositories are never touched.
const checkService = new CheckService(monitorsRepo as any, logger, {} as any);
const setNextResponse = (status: boolean, code: number) => {
nextResponse = null;
@@ -137,7 +119,13 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
nextEgressStatus = status;
};
const notificationReactor = new NotificationReactor(notificationsService as any);
// The real NotificationsService ignores decisions with shouldSendNotification false; the stub is reached only for
// the ones it would act on, so tests can assert on handleNotifications calls as "notifications sent".
const guardedNotificationsService = {
handleNotifications: (monitor: Monitor, check: Check, decision: MonitorActionDecision) =>
decision.shouldSendNotification ? notificationsService.handleNotifications(monitor, check, decision) : Promise.resolve(false),
};
const notificationReactor = new NotificationReactor(guardedNotificationsService as any);
const incidentReactor = new IncidentReactor(incidentService as any);
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
@@ -165,7 +153,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
if (!monitor) return;
const result = await pipeline.produce(monitor);
if (!result) return; // skipped (e.g. maintenance window)
const evaluation = await pipeline.evaluateCheck(result.status, result.check, monitor);
const evaluation = await pipeline.evaluateCheck(result.check, monitor);
await reactorDispatcher.dispatch(evaluation);
};
@@ -131,9 +131,9 @@ describe("Evaluate cursor with more than one processing worker", () => {
bufferService,
dockerLogsService: { buildDockerLogs: jest.fn<any>() } as any,
statusService: {
updateMonitorStatus: async (_status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
updateMonitorStatus: async (check: Check, monitor: Monitor) => {
evaluatedCheckIds.push(check.id);
return { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 };
return { monitor, statusChanged: false, prevStatus: monitor.status };
},
} as any,
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) } as any,
@@ -49,7 +49,7 @@ describe("Heartbeat job: hardware threshold breach", () => {
h.setNextResponseFull(makeHardwareResponse(0.9));
// Configure message builder for when incident is created
h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]);
h.messageBuilder.buildThresholdBreachMessage.mockReturnValue("CPU: 90% (threshold: 80%)");
// 4 heartbeats: counter goes 5→4→3→2→1, status still "up"
for (let i = 0; i < 4; i++) {
@@ -191,7 +191,7 @@ describe("Heartbeat job: hardware threshold breach", () => {
h.monitorsRepo.seed(monitor);
h.setNextResponseFull(makeHardwareResponse(0.9));
h.messageBuilder.extractThresholdBreaches.mockReturnValue([{ metric: "cpu", formattedValue: "90%", threshold: 80, unit: "%" }]);
h.messageBuilder.buildThresholdBreachMessage.mockReturnValue("CPU: 90% (threshold: 80%)");
// 5 heartbeats to trigger breach, then 5 more while still breaching
for (let i = 0; i < 10; i++) {
@@ -7,6 +7,7 @@ import type { IUsersRepository } from "../../src/domain/users/user.repository.in
import type { INotificationMessageBuilder } from "../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
import type { MonitorActionDecision } from "../../src/worker/worker.interface.ts";
import type { Check } from "../../src/domain/checks/check.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -30,6 +31,17 @@ const makeDecision = (overrides?: Partial<MonitorActionDecision>): MonitorAction
...overrides,
});
const makeCheck = (statusCode: number): Check => ({
id: "check-1",
metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" },
status: statusCode < 400,
statusCode,
responseTime: 100,
message: statusCode < 400 ? "OK" : "Service Unavailable",
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
});
// ── Test suite ───────────────────────────────────────────────────────────────
describe("Incident lifecycle (integration)", () => {
@@ -43,7 +55,7 @@ describe("Incident lifecycle (integration)", () => {
repo = new InMemoryIncidentsRepository();
monitorsRepo = { findById: jest.fn() } as unknown as jest.Mocked<IMonitorsRepository>;
usersRepo = { findById: jest.fn() } as unknown as jest.Mocked<IUsersRepository>;
messageBuilder = { extractThresholdBreaches: jest.fn() } as unknown as jest.Mocked<INotificationMessageBuilder>;
messageBuilder = { buildThresholdBreachMessage: jest.fn() } as unknown as jest.Mocked<INotificationMessageBuilder>;
service = new IncidentService(createMockLogger() as any, repo, monitorsRepo, usersRepo, messageBuilder);
});
@@ -53,7 +65,7 @@ describe("Incident lifecycle (integration)", () => {
const monitor = makeMonitor({ status: "down" });
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
const incident = await service.handleIncident(monitor, 503, decision);
const incident = await service.handleIncident(monitor, decision, makeCheck(503));
expect(incident).not.toBeNull();
expect(incident!.monitorId).toBe("mon-1");
@@ -74,8 +86,8 @@ describe("Incident lifecycle (integration)", () => {
const monitor = makeMonitor({ status: "down" });
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
const first = await service.handleIncident(monitor, 503, decision);
const second = await service.handleIncident(monitor, 503, decision);
const first = await service.handleIncident(monitor, decision, makeCheck(503));
const second = await service.handleIncident(monitor, decision, makeCheck(503));
expect(second!.id).toBe(first!.id);
expect(repo.getAll()).toHaveLength(1);
@@ -86,10 +98,10 @@ describe("Incident lifecycle (integration)", () => {
it("auto-resolves an incident when monitor recovers", async () => {
const monitor = makeMonitor({ status: "down" });
const createDecision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
const created = await service.handleIncident(monitor, 503, createDecision);
const created = await service.handleIncident(monitor, createDecision, makeCheck(503));
const resolveDecision = makeDecision({ shouldResolveIncident: true });
const resolved = await service.handleIncident(monitor, 200, resolveDecision);
const resolved = await service.handleIncident(monitor, resolveDecision, makeCheck(200));
expect(resolved).not.toBeNull();
expect(resolved!.id).toBe(created!.id);
@@ -104,7 +116,7 @@ describe("Incident lifecycle (integration)", () => {
const monitor = makeMonitor({ status: "up" });
const decision = makeDecision({ shouldResolveIncident: true });
const result = await service.handleIncident(monitor, 200, decision);
const result = await service.handleIncident(monitor, decision, makeCheck(200));
expect(result).toBeNull();
});
@@ -114,7 +126,7 @@ describe("Incident lifecycle (integration)", () => {
it("manually resolves an active incident with comment", async () => {
const monitor = makeMonitor({ status: "down" });
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
const created = await service.handleIncident(monitor, 500, decision);
const created = await service.handleIncident(monitor, decision, makeCheck(500));
const resolved = await service.resolveIncident(created!.id, "user-1", "team-1", "Root cause identified", "user@test.com");
@@ -129,7 +141,7 @@ describe("Incident lifecycle (integration)", () => {
it("throws when manually resolving an already-resolved incident", async () => {
const monitor = makeMonitor({ status: "down" });
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
const created = await service.handleIncident(monitor, 500, decision);
const created = await service.handleIncident(monitor, decision, makeCheck(500));
await service.resolveIncident(created!.id, "user-1", "team-1");
@@ -140,13 +152,10 @@ describe("Incident lifecycle (integration)", () => {
it("creates a threshold breach incident with statusCode 9999 and descriptive message", async () => {
const monitor = makeMonitor({ status: "breached", type: "hardware" });
(messageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([
{ metric: "cpu", formattedValue: "92%", threshold: 80, unit: "%" },
{ metric: "memory", formattedValue: "88%", threshold: 85, unit: "%" },
]);
(messageBuilder.buildThresholdBreachMessage as jest.Mock).mockReturnValue("CPU: 92% (threshold: 80%), MEMORY: 88% (threshold: 85%)");
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
const incident = await service.handleIncident(monitor, 200, decision, { monitorId: "mon-1" } as any);
const incident = await service.handleIncident(monitor, decision, makeCheck(200));
expect(incident!.statusCode).toBe(9999);
expect(incident!.message).toBe("CPU: 92% (threshold: 80%), MEMORY: 88% (threshold: 85%)");
@@ -158,7 +167,7 @@ describe("Incident lifecycle (integration)", () => {
const monitor = makeMonitor({ status: "up" });
const decision = makeDecision();
const result = await service.handleIncident(monitor, 200, decision);
const result = await service.handleIncident(monitor, decision, makeCheck(200));
expect(result).toBeNull();
expect(repo.getAll()).toHaveLength(0);
@@ -170,19 +179,19 @@ describe("Incident lifecycle (integration)", () => {
const monitor = makeMonitor({ status: "down" });
// First outage
const first = await service.handleIncident(monitor, 503, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }));
const first = await service.handleIncident(monitor, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), makeCheck(503));
expect(first!.status).toBe(true);
// Manually resolved
await service.resolveIncident(first!.id, "user-1", "team-1", "Restarted server");
// Second outage — new incident since previous was resolved
const second = await service.handleIncident(monitor, 502, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }));
const second = await service.handleIncident(monitor, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }), makeCheck(502));
expect(second!.id).not.toBe(first!.id);
expect(second!.statusCode).toBe(502);
// Auto-resolve
const resolved = await service.handleIncident(monitor, 200, makeDecision({ shouldResolveIncident: true }));
const resolved = await service.handleIncident(monitor, makeDecision({ shouldResolveIncident: true }), makeCheck(200));
expect(resolved!.id).toBe(second!.id);
expect(resolved!.status).toBe(false);
expect(resolved!.resolutionType).toBe("automatic");
@@ -200,14 +209,14 @@ describe("Incident lifecycle (integration)", () => {
const monitorB = makeMonitor({ id: "mon-b", status: "down" });
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" });
const incidentA = await service.handleIncident(monitorA, 500, decision);
const incidentB = await service.handleIncident(monitorB, 502, decision);
const incidentA = await service.handleIncident(monitorA, decision, makeCheck(500));
const incidentB = await service.handleIncident(monitorB, decision, makeCheck(502));
expect(incidentA!.id).not.toBe(incidentB!.id);
expect(repo.getAll()).toHaveLength(2);
// Resolving monitor A does not affect monitor B
const resolved = await service.handleIncident(monitorA, 200, makeDecision({ shouldResolveIncident: true }));
const resolved = await service.handleIncident(monitorA, makeDecision({ shouldResolveIncident: true }), makeCheck(200));
expect(resolved!.id).toBe(incidentA!.id);
const bStillActive = await repo.findActiveByMonitorId("mon-b", "team-1");
@@ -229,6 +229,18 @@ describe("DockerProvider", () => {
});
});
it("leaves the exit code undefined for Capture containers", async () => {
const response = makeCaptureResponse({
payload: { data: [makeCaptureContainer({ status: "exited", running: false, stats: undefined })], errors: [] },
});
const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) });
const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" }));
expect(result.payload?.containers[0]?.state).toBe("exited");
expect(result.payload?.containers[0]?.exitCode).toBeUndefined();
});
it("keeps state-derived Capture health separate from Docker healthchecks", async () => {
const response = makeCaptureResponse({
payload: {
@@ -558,6 +570,16 @@ describe("DockerProvider", () => {
);
});
it("reads the exit code from inspect", async () => {
const inspect = jest.fn().mockResolvedValue(makeInspect({ State: { StartedAt: "2026-08-28T10:00:00.000Z", ExitCode: 137 } }));
const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status: "Exited (137) 2 minutes ago" })]);
const { provider } = setup({ inspect, listContainers });
const result = await provider.handle(makeMonitor());
expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ state: "exited", exitCode: 137, health: "none" }));
});
it("counts running, stopped, and unhealthy states in the summary", async () => {
const containers = [
makeContainer({ Id: "a".repeat(64) }),
@@ -682,11 +704,40 @@ describe("DockerProvider", () => {
expect(result.status).toBe(true);
expect(result.payload?.containers).toHaveLength(1);
expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ health: "none" }));
expect(result.payload?.containers[0]?.restartCount).toBeUndefined();
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Failed to inspect") }));
});
it.each([
["Up 3 hours (healthy)", "healthy"],
["Up 3 hours (unhealthy)", "unhealthy"],
["Up 5 seconds (health: starting)", "starting"],
["Up 3 hours", "none"],
["Exited (1) 2 minutes ago", "none"],
])("reads health from the list status %s when inspect rejects", async (Status, health) => {
const inspect = jest.fn().mockRejectedValue(new Error("inspect failed"));
const listContainers = jest.fn().mockResolvedValue([makeContainer({ Status })]);
const { provider } = setup({ inspect, listContainers });
const result = await provider.handle(makeMonitor());
expect(result.payload?.containers[0]?.health).toBe(health);
});
it.each([
["Exited (0) 2 minutes ago", 0],
["Exited (137) 2 minutes ago", 137],
["Up 3 hours", undefined],
])("reads the exit code from the list status %s when inspect rejects", async (Status, exitCode) => {
const inspect = jest.fn().mockRejectedValue(new Error("inspect failed"));
const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status })]);
const { provider } = setup({ inspect, listContainers });
const result = await provider.handle(makeMonitor());
expect(result.payload?.containers[0]?.exitCode).toBe(exitCode);
});
it("keeps the container without metrics when stats rejects", async () => {
const stats = jest.fn().mockRejectedValue(new Error("stats failed"));
const { provider } = setup({ stats });
@@ -4,6 +4,7 @@ import { createMockLogger } from "../../../helpers/createMockLogger.ts";
import { makeNotification, makeMessage } from "../../../helpers/notificationMessage.ts";
import { testNotificationProviderContract } from "../../../helpers/notificationProviderContract.ts";
import type { IEmailService } from "../../../../src/service/emailService.ts";
import { AppError } from "../../../../src/utils/AppError.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -50,11 +51,29 @@ describe("EmailProvider", () => {
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build test email content" }));
});
it("returns false when sendEmail rejects", async () => {
it("returns false and logs the cause when sendEmail rejects with a plain error", async () => {
const { provider, emailService, logger } = createProvider();
(emailService.sendEmail as jest.Mock).mockRejectedValue(new Error("SMTP auth failed"));
expect(await provider.sendTestAlert(makeNotification())).toBe(false);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email test alert failed" }));
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email test alert failed: SMTP auth failed" }));
});
it("rethrows an AppError from sendEmail so the caller can report the reason", async () => {
const { provider, emailService, logger } = createProvider();
const error = new AppError({
message: "Email is not configured. Set the system email host in settings.",
status: 400,
details: { cause: "no host" },
});
(emailService.sendEmail as jest.Mock).mockRejectedValue(error);
await expect(provider.sendTestAlert(makeNotification())).rejects.toBe(error);
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({
message: "Email test alert failed: Email is not configured. Set the system email host in settings.",
details: { cause: "no host" },
})
);
});
});
@@ -84,11 +103,29 @@ describe("EmailProvider", () => {
// NotificationsService fans providers out through Promise.all, so a failed email must
// resolve to false rather than reject and take the other channels down with it.
it("returns false when sendEmail rejects", async () => {
it("returns false and logs the cause when sendEmail rejects", async () => {
const { provider, emailService, logger } = createProvider();
(emailService.sendEmail as jest.Mock).mockRejectedValue(new Error("SMTP auth failed"));
expect(await provider.sendMessage(makeNotification() as any, makeMessage())).toBe(false);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email notification failed" }));
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email notification failed: SMTP auth failed" }));
});
it("returns false rather than throwing when sendEmail rejects with an AppError", async () => {
const { provider, emailService, logger } = createProvider();
(emailService.sendEmail as jest.Mock).mockRejectedValue(
new AppError({
message: "SMTP verification failed for smtp.example.com:587: 535 Username and Password not accepted",
status: 502,
details: { cause: "ECONNREFUSED" },
})
);
expect(await provider.sendMessage(makeNotification() as any, makeMessage())).toBe(false);
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({
message: "Email notification failed: SMTP verification failed for smtp.example.com:587: 535 Username and Password not accepted",
details: { cause: "ECONNREFUSED" },
})
);
});
it("builds correct subject for monitor_down", async () => {
@@ -115,6 +152,18 @@ describe("EmailProvider", () => {
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor thresholds resolved", expect.anything());
});
it("builds correct subject for container_breach", async () => {
const { provider, emailService } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" }));
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor container alert", expect.anything());
});
it("builds correct subject for container_resolved", async () => {
const { provider, emailService } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" }));
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor containers recovered", expect.anything());
});
it("builds default subject for unknown type", async () => {
const { provider, emailService } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "test" }));
@@ -105,6 +105,14 @@ describe("MatrixProvider", () => {
expect(body.formatted_body).toContain("View Incident");
});
it("links the incident to the monitor's incidents page", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
const body = mockGotPut.mock.calls[0][1].json;
expect(body.body).toContain("View Incident: https://app.example.com/incidents/mon-1");
expect(body.formatted_body).toContain('href="https://app.example.com/incidents/mon-1"');
});
it("omits optional sections when not present", async () => {
const { provider } = createProvider();
const msg = makeMessage();
@@ -106,7 +106,7 @@ describe("NtfyProvider", () => {
it("includes incident links in text", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNtfyNotification() as any, makeMessageWithIncident());
expect(mockGotPost.mock.calls[0][1].body).toContain("/infrastructure/mon-1");
expect(mockGotPost.mock.calls[0][1].body).toContain("https://app.example.com/incidents/mon-1");
});
});
@@ -89,6 +89,18 @@ describe("PagerDutyProvider", () => {
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve");
});
it("uses 'resolve' event_action for container_resolved", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" }));
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve");
});
it("uses 'trigger' event_action for container_breach", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" }));
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("trigger");
});
it("includes threshold info in summary and custom_details", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessageWithThresholds());
@@ -99,6 +99,12 @@ describe("PushoverProvider", () => {
expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident");
});
it("links the incident to the monitor's incidents page", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident: https://app.example.com/incidents/mon-1");
});
it("omits optional sections when not present", async () => {
const { provider } = createProvider();
const msg = makeMessage();
@@ -134,6 +134,7 @@ describe("RocketChatProvider", () => {
["ping", "https://app.example.com/uptime/mon-1"],
["pagespeed", "https://app.example.com/pagespeed/mon-1"],
["hardware", "https://app.example.com/infrastructure/mon-1"],
["docker", "https://app.example.com/docker/host/mon-1"],
])("links %s monitors to %s", async (type, expectedLink) => {
const { provider } = createProvider();
@@ -139,6 +139,12 @@ describe("SignalgridProvider", () => {
expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident");
});
it("links the incident to the monitor's incidents page", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident: https://app.example.com/incidents/mon-1");
});
it("omits optional sections when not present", async () => {
const { provider } = createProvider();
const msg = makeMessage();
@@ -96,6 +96,14 @@ describe("SlackProvider", () => {
expect(text).toContain("Additional Information");
});
it("links the incident button to the monitor's incidents page", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
const blocks = mockGotPost.mock.calls[0][1].json.blocks;
const actions = blocks.find((b: any) => b.type === "actions");
expect(actions.elements[0].url).toBe("https://app.example.com/incidents/mon-1");
});
it("omits threshold, details, and incident sections when not present", async () => {
const { provider } = createProvider();
const msg = makeMessage();
@@ -154,7 +154,7 @@ describe("TeamsProvider", () => {
expect(card.actions).toHaveLength(1);
expect(card.actions[0].type).toBe("Action.OpenUrl");
expect(card.actions[0].title).toBe("View Incident");
expect(card.actions[0].url).toContain("/incidents/inc-1");
expect(card.actions[0].url).toBe("https://app.example.com/incidents/mon-1");
});
it("omits actions when no incident is present", async () => {
@@ -99,6 +99,12 @@ describe("TelegramProvider", () => {
expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident");
});
it("links the incident to the monitor's incidents page", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
expect(mockGotPost.mock.calls[0][1].json.text).toContain('<a href="https://app.example.com/incidents/mon-1">View Incident</a>');
});
it("omits optional sections when not present", async () => {
const { provider } = createProvider();
const msg = makeMessage();
@@ -86,6 +86,12 @@ describe("WebhookProvider", () => {
expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident");
});
it("links the incident to the monitor's incidents page", async () => {
const { provider } = createProvider();
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
expect(mockGotPost.mock.calls[0][1].json.text).toContain("[View Incident](https://app.example.com/incidents/mon-1)");
});
it("omits threshold and incident sections when not present", async () => {
const { provider } = createProvider();
const msg = makeMessage();
@@ -4,8 +4,6 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts";
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -281,84 +279,6 @@ describe("CheckService", () => {
});
});
// ── toStatusResponse round-trip (Phase 2 §5 guard) ─────────────────────────
// The DB-queue split persists a Check, then the evaluator rebuilds the status response from it.
// toStatusResponse is only correct because every field a downstream consumer reads survives
// toCheck → toStatusResponse. These tests pin that: if a future change drops a consumed field
// from the round-trip, they fail instead of silently regressing the evaluator/reactors.
describe("toStatusResponse round-trip", () => {
const makeHardwareStatus = (overrides?: Partial<MonitorStatusResponse>): MonitorStatusResponse =>
makeStatusResponse({
type: "hardware",
status: true,
code: 200,
message: "OK",
responseTime: 42,
payload: {
data: {
cpu: { usage_percent: 0.92 },
memory: { usage_percent: 0.81 },
disk: [{ device: "/dev/sda", usage_percent: 0.75 }],
host: { os: "linux" },
net: [{ name: "eth0" }],
},
} as HardwareStatusPayload,
...overrides,
} as any);
it("uptime: preserves every consumed scalar field", () => {
const { service } = createService();
const status = makeStatusResponse({ type: "http", status: false, code: 503, message: "Service Unavailable", responseTime: 87 });
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
expect(rebuilt.monitorId).toBe(status.monitorId);
expect(rebuilt.teamId).toBe(status.teamId);
expect(rebuilt.type).toBe(status.type);
expect(rebuilt.status).toBe(status.status);
expect(rebuilt.code).toBe(status.code);
expect(rebuilt.message).toBe(status.message);
expect(rebuilt.responseTime).toBe(status.responseTime);
});
it("hardware: preserves payload.data metrics verbatim, same units (usage_percent stays a 0–1 decimal)", () => {
const { service } = createService();
const status = makeHardwareStatus();
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
const original = status.payload as HardwareStatusPayload;
const roundtripped = rebuilt.payload as HardwareStatusPayload;
expect(roundtripped.data).toEqual(original.data);
expect(roundtripped.data.cpu!.usage_percent).toBe(0.92); // not rescaled to 92
});
it("hardware: a real consumer (extractThresholdBreaches) behaves identically on raw vs round-tripped status", () => {
const { service } = createService();
const builder = new NotificationMessageBuilder();
const monitor = {
id: "mon-1",
type: "hardware",
cpuAlertThreshold: 90, // 0.92 → 92% breaches
memoryAlertThreshold: 90, // 0.81 → 81% does not
diskAlertThreshold: 90, // 0.75 → 75% does not
tempAlertThreshold: 100,
} as Monitor;
const status = makeHardwareStatus();
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
const fromRaw = builder.extractThresholdBreaches(monitor, status as MonitorStatusResponse<HardwareStatusPayload>);
const fromRebuilt = builder.extractThresholdBreaches(monitor, rebuilt as MonitorStatusResponse<HardwareStatusPayload>);
// sanity: the fixture actually triggers a breach, so the equality below is not vacuous
expect(fromRaw).toHaveLength(1);
expect(fromRaw[0].metric).toBe("cpu");
expect(fromRebuilt).toEqual(fromRaw);
});
});
// ── createChecks ─────────────────────────────────────────────────────────
describe("createChecks", () => {
+46 -3
View File
@@ -335,17 +335,60 @@ describe("EmailService", () => {
const { service, logger } = createService({ nodemailer });
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig())).rejects.toThrow(
"Email transporter verification failed"
"SMTP host smtp.example.com:587 failed verification."
);
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({
message: "Email transporter verification failed",
message: "Email transporter verification failed: SMTP auth failed",
service: "EmailService",
method: "verifyTransporter",
details: { host: "smtp.example.com", port: 587 },
})
);
});
it.each([
["EAUTH", "SMTP host smtp.example.com:587 rejected the login. Check the user name and password."],
["ETIMEDOUT", "SMTP host smtp.example.com:587 did not respond. Check the port and the secure setting."],
["EDNS", "SMTP host smtp.example.com:587 could not be resolved. Check the host name."],
["ECONNECTION", "SMTP host smtp.example.com:587 refused the connection. Check the host and port."],
["ETLS", "SMTP host smtp.example.com:587 failed the TLS handshake. Check the secure and TLS settings."],
])("describes a %s verification failure without the server's reply text", async (code, expected) => {
const transporter = createMockTransporter();
(transporter.verify as jest.Mock).mockRejectedValue(Object.assign(new Error("535-5.7.8 Username and Password not accepted"), { code }));
const nodemailer = createMockNodemailer(transporter);
const { service } = createService({ nodemailer });
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig())).rejects.toMatchObject({
message: expected,
status: 502,
details: { cause: "535-5.7.8 Username and Password not accepted" },
});
});
it("throws a 400 without building a transport when the host is not configured", async () => {
const nodemailer = createMockNodemailer();
const { service } = createService({ nodemailer });
await expect(
service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig({ systemEmailHost: undefined }))
).rejects.toMatchObject({
status: 400,
message: "Email is not configured. Set the system email host in settings.",
});
expect(nodemailer.createTransport).not.toHaveBeenCalled();
});
it("treats an empty host as not configured", async () => {
const { service } = createService();
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig({ systemEmailHost: "" }))).rejects.toMatchObject(
{
status: 400,
}
);
});
it("reports the underlying cause when verification fails", async () => {
const transporter = createMockTransporter();
(transporter.verify as jest.Mock).mockRejectedValue(new Error("SMTP auth failed"));
@@ -353,7 +396,7 @@ describe("EmailService", () => {
const { service } = createService({ nodemailer });
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig())).rejects.toMatchObject({
status: 500,
status: 502,
details: { cause: "SMTP auth failed" },
});
});
@@ -8,6 +8,7 @@ import type { INotificationMessageBuilder } from "../../../src/domain/notificati
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { Incident } from "../../../src/domain/incidents/incident.type.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -35,7 +36,7 @@ const createUsersRepo = () =>
const createMessageBuilder = () =>
({
extractThresholdBreaches: jest.fn(),
buildThresholdBreachMessage: jest.fn(),
}) as unknown as jest.Mocked<INotificationMessageBuilder>;
const createService = (overrides?: {
@@ -79,6 +80,19 @@ const makeIncident = (overrides?: Partial<Incident>): Incident =>
...overrides,
}) as Incident;
const makeCheck = (overrides?: Partial<Check>): Check =>
({
id: "check-1",
metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" },
status: true,
statusCode: 200,
responseTime: 100,
message: "OK",
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
...overrides,
}) as Check;
const makeDecision = (overrides?: Partial<MonitorActionDecision>): MonitorActionDecision => ({
shouldCreateIncident: false,
shouldResolveIncident: false,
@@ -96,7 +110,7 @@ describe("IncidentService", () => {
describe("handleIncident", () => {
it("returns null when neither create nor resolve is requested", async () => {
const { service } = createService();
const result = await service.handleIncident(makeMonitor(), 200, makeDecision());
const result = await service.handleIncident(makeMonitor(), makeDecision(), makeCheck());
expect(result).toBeNull();
});
@@ -105,7 +119,11 @@ describe("IncidentService", () => {
const { service, incidentsRepository } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(existing);
const result = await service.handleIncident(makeMonitor(), 500, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }));
const result = await service.handleIncident(
makeMonitor(),
makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }),
makeCheck({ statusCode: 500 })
);
expect(result).toBe(existing);
expect(incidentsRepository.create).not.toHaveBeenCalled();
@@ -113,11 +131,15 @@ describe("IncidentService", () => {
it("creates a new incident when shouldCreateIncident and no active incident exists", async () => {
const created = makeIncident();
const { service, incidentsRepository } = createService();
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
const result = await service.handleIncident(makeMonitor(), 500, makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }));
const result = await service.handleIncident(
makeMonitor(),
makeDecision({ shouldCreateIncident: true, incidentReason: "status_down" }),
makeCheck({ statusCode: 500 })
);
expect(result).toBe(created);
expect(incidentsRepository.create).toHaveBeenCalledWith(
@@ -128,6 +150,7 @@ describe("IncidentService", () => {
statusCode: 500,
})
);
expect(notificationMessageBuilder.buildThresholdBreachMessage).not.toHaveBeenCalled();
});
it("uses status code 9999 and builds message for threshold_breach incidents", async () => {
@@ -135,13 +158,15 @@ describe("IncidentService", () => {
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
(notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([
{ metric: "cpu", formattedValue: "95%", threshold: 80, unit: "%" },
]);
(notificationMessageBuilder.buildThresholdBreachMessage as jest.Mock).mockReturnValue("CPU: 95% (threshold: 80%)");
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
await service.handleIncident(makeMonitor(), 200, decision, { monitorId: "mon-1" } as any);
const monitor = makeMonitor({ type: "hardware" });
const check = makeCheck({ metadata: { monitorId: "mon-1", teamId: "team-1", type: "hardware" }, cpu: { usage_percent: 0.95 } });
const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false };
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach", thresholdBreaches });
await service.handleIncident(monitor, decision, check);
expect(notificationMessageBuilder.buildThresholdBreachMessage).toHaveBeenCalledWith(monitor, check, thresholdBreaches);
expect(incidentsRepository.create).toHaveBeenCalledWith(
expect.objectContaining({
statusCode: 9999,
@@ -150,49 +175,6 @@ describe("IncidentService", () => {
);
});
it("uses fallback message when monitorStatusResponse is undefined for threshold_breach", async () => {
const created = makeIncident();
const { service, incidentsRepository } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
await service.handleIncident(makeMonitor(), 200, decision, undefined);
expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" }));
});
it("uses fallback message when extractThresholdBreaches returns empty array", async () => {
const created = makeIncident();
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
(notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([]);
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
await service.handleIncident(makeMonitor(), 200, decision, { monitorId: "mon-1" } as any);
expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" }));
});
it("joins multiple threshold breaches with commas", async () => {
const created = makeIncident();
const { service, incidentsRepository, notificationMessageBuilder } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
(notificationMessageBuilder.extractThresholdBreaches as jest.Mock).mockReturnValue([
{ metric: "cpu", formattedValue: "95%", threshold: 80, unit: "%" },
{ metric: "memory", formattedValue: "90%", threshold: 80, unit: "%" },
]);
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
await service.handleIncident(makeMonitor(), 200, decision, { monitorId: "mon-1" } as any);
expect(incidentsRepository.create).toHaveBeenCalledWith(
expect.objectContaining({ message: "CPU: 95% (threshold: 80%), MEMORY: 90% (threshold: 80%)" })
);
});
it("resolves active incident when shouldResolveIncident", async () => {
const active = makeIncident();
const resolved = makeIncident({ status: false, endTime: "123", resolutionType: "automatic" });
@@ -200,7 +182,7 @@ describe("IncidentService", () => {
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(active);
(incidentsRepository.updateById as jest.Mock).mockResolvedValue(resolved);
const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true }));
const result = await service.handleIncident(makeMonitor(), makeDecision({ shouldResolveIncident: true }), makeCheck());
expect(result).toBe(resolved);
expect(incidentsRepository.updateById).toHaveBeenCalledWith(
@@ -214,7 +196,7 @@ describe("IncidentService", () => {
const { service, incidentsRepository } = createService();
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
const result = await service.handleIncident(makeMonitor(), 200, makeDecision({ shouldResolveIncident: true }));
const result = await service.handleIncident(makeMonitor(), makeDecision({ shouldResolveIncident: true }), makeCheck());
expect(result).toBeNull();
});
@@ -1,7 +1,8 @@
import { describe, expect, it, beforeEach } from "@jest/globals";
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
import type { HardwareBreaches, Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -30,26 +31,38 @@ const makeDecision = (overrides?: Partial<MonitorActionDecision>): MonitorAction
...overrides,
});
const makeStatusResponse = (overrides?: Partial<MonitorStatusResponse>): MonitorStatusResponse =>
({
monitorId: "mon-1",
teamId: "team-1",
type: "http",
status: false,
code: 500,
message: "Internal Server Error",
...overrides,
}) as MonitorStatusResponse;
const makeCheck = (overrides?: Partial<Check>): Check => ({
id: "check-1",
metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" },
status: false,
responseTime: 100,
statusCode: 500,
message: "Internal Server Error",
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
...overrides,
});
const makeHardwarePayload = (overrides?: Partial<HardwareStatusPayload["data"]>): HardwareStatusPayload =>
({
data: {
cpu: { usage_percent: 0.5, temperature: [45] },
memory: { usage_percent: 0.6 },
disk: [{ usage_percent: 0.7 }],
...overrides,
},
}) as HardwareStatusPayload;
const makeContainers = (containers: Partial<DockerContainerInfo>[]): DockerContainerInfo[] =>
containers.map((container, index) => ({
id: `container-${index}`,
name: `container-${index}`,
image: "nginx:latest",
state: "running",
status: "Up 3 hours",
health: "none",
...container,
}));
const makeDockerMonitor = (overrides?: Partial<Monitor>): Monitor =>
makeMonitor({
type: "docker",
status: "breached",
url: "unix:///var/run/docker.sock",
dockerAlertOnStopped: true,
dockerAlertOnUnhealthy: true,
...overrides,
});
// ── Tests ────────────────────────────────────────────────────────────────────
@@ -72,9 +85,9 @@ describe("NotificationMessageBuilder", () => {
it("builds a monitor_down message", () => {
const monitor = makeMonitor({ status: "down" });
const decision = makeDecision();
const response = makeStatusResponse();
const check = makeCheck();
const msg = builder.buildMessage(monitor, response, decision, "https://app.example.com");
const msg = builder.buildMessage(monitor, check, decision, "https://app.example.com");
expect(msg.type).toBe("monitor_down");
expect(msg.severity).toBe("critical");
@@ -97,7 +110,7 @@ describe("NotificationMessageBuilder", () => {
const monitor = makeMonitor({ status: "up", type: "http" });
const decision = makeDecision({ notificationReason: "status_change" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com");
const msg = builder.buildMessage(monitor, makeCheck(), decision, "https://app.example.com");
expect(msg.type).toBe("monitor_up");
expect(msg.severity).toBe("success");
@@ -106,13 +119,13 @@ describe("NotificationMessageBuilder", () => {
it("builds a threshold_breach message", () => {
const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 });
const decision = makeDecision({ notificationReason: "threshold_breach" });
const response = makeStatusResponse({
type: "hardware",
payload: makeHardwarePayload({ cpu: { usage_percent: 0.9, temperature: [50] } as any }),
} as any);
const decision = makeDecision({
notificationReason: "threshold_breach",
thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false },
});
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const msg = builder.buildMessage(monitor, response, decision, "https://app.example.com");
const msg = builder.buildMessage(monitor, check, decision, "https://app.example.com");
expect(msg.type).toBe("threshold_breach");
expect(msg.severity).toBe("warning");
@@ -123,23 +136,14 @@ describe("NotificationMessageBuilder", () => {
it("builds a threshold_resolved message for hardware monitor recovering", () => {
const monitor = makeMonitor({ status: "up", type: "hardware" });
const decision = makeDecision({ notificationReason: "status_change" });
const decision = makeDecision({ notificationReason: "threshold_resolved" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com");
const msg = builder.buildMessage(monitor, makeCheck(), decision, "https://app.example.com");
expect(msg.type).toBe("threshold_resolved");
expect(msg.severity).toBe("success");
expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor");
});
it("uses notificationReason from decision, falling back to status_change", () => {
const monitor = makeMonitor({ status: "down" });
const decision = makeDecision({ notificationReason: null });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com");
expect(msg.metadata.notificationReason).toBe("status_change");
});
});
// ── determineNotificationType (via buildMessage) ─────────────────────
@@ -149,7 +153,7 @@ describe("NotificationMessageBuilder", () => {
const monitor = makeMonitor({ status: "down" });
const decision = makeDecision({ notificationReason: "threshold_breach" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
const msg = builder.buildMessage(monitor, makeCheck(), decision, "");
expect(msg.type).toBe("monitor_down");
});
@@ -158,25 +162,58 @@ describe("NotificationMessageBuilder", () => {
const monitor = makeMonitor({ status: "up", type: "hardware" });
const decision = makeDecision({ notificationReason: "threshold_breach" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
const msg = builder.buildMessage(monitor, makeCheck(), decision, "");
expect(msg.type).toBe("threshold_breach");
});
it("returns threshold_resolved for hardware monitor with status_change and up", () => {
it("returns threshold_resolved for hardware monitor with threshold_resolved reason", () => {
const monitor = makeMonitor({ status: "up", type: "hardware" });
const decision = makeDecision({ notificationReason: "threshold_resolved" });
const msg = builder.buildMessage(monitor, makeCheck(), decision, "");
expect(msg.type).toBe("threshold_resolved");
});
it("returns monitor_up for hardware monitor recovering from down with status_change", () => {
const monitor = makeMonitor({ status: "up", type: "hardware" });
const decision = makeDecision({ notificationReason: "status_change" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
const msg = builder.buildMessage(monitor, makeCheck(), decision, "");
expect(msg.type).toBe("threshold_resolved");
expect(msg.type).toBe("monitor_up");
});
it("returns container_breach for docker monitor with threshold_breach reason", () => {
const decision = makeDecision({ notificationReason: "threshold_breach" });
const msg = builder.buildMessage(makeDockerMonitor(), makeCheck(), decision, "");
expect(msg.type).toBe("container_breach");
});
it("returns container_resolved for docker monitor with threshold_resolved reason", () => {
const decision = makeDecision({ notificationReason: "threshold_resolved" });
const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeCheck(), decision, "");
expect(msg.type).toBe("container_resolved");
});
it("returns monitor_down for docker monitor that is down regardless of reason", () => {
const decision = makeDecision({ notificationReason: "threshold_breach" });
const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeCheck(), decision, "");
expect(msg.type).toBe("monitor_down");
});
it("returns monitor_up for non-hardware monitor with status up", () => {
const monitor = makeMonitor({ status: "up", type: "http" });
const decision = makeDecision({ notificationReason: "status_change" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
const msg = builder.buildMessage(monitor, makeCheck(), decision, "");
expect(msg.type).toBe("monitor_up");
});
@@ -185,7 +222,7 @@ describe("NotificationMessageBuilder", () => {
const monitor = makeMonitor({ status: "unknown" as any });
const decision = makeDecision({ notificationReason: null });
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
const msg = builder.buildMessage(monitor, makeCheck(), decision, "");
expect(msg.type).toBe("monitor_up");
});
@@ -195,14 +232,14 @@ describe("NotificationMessageBuilder", () => {
describe("determineSeverity", () => {
it("returns critical for monitor_down", () => {
const msg = builder.buildMessage(makeMonitor({ status: "down" }), makeStatusResponse(), makeDecision(), "");
const msg = builder.buildMessage(makeMonitor({ status: "down" }), makeCheck(), makeDecision(), "");
expect(msg.severity).toBe("critical");
});
it("returns warning for threshold_breach", () => {
const msg = builder.buildMessage(
makeMonitor({ status: "up", type: "hardware" }),
makeStatusResponse(),
makeCheck(),
makeDecision({ notificationReason: "threshold_breach" }),
""
);
@@ -212,7 +249,7 @@ describe("NotificationMessageBuilder", () => {
it("returns success for monitor_up", () => {
const msg = builder.buildMessage(
makeMonitor({ status: "up", type: "http" }),
makeStatusResponse(),
makeCheck(),
makeDecision({ notificationReason: "status_change" }),
""
);
@@ -222,8 +259,23 @@ describe("NotificationMessageBuilder", () => {
it("returns success for threshold_resolved", () => {
const msg = builder.buildMessage(
makeMonitor({ status: "up", type: "hardware" }),
makeStatusResponse(),
makeDecision({ notificationReason: "status_change" }),
makeCheck(),
makeDecision({ notificationReason: "threshold_resolved" }),
""
);
expect(msg.severity).toBe("success");
});
it("returns warning for container_breach", () => {
const msg = builder.buildMessage(makeDockerMonitor(), makeCheck(), makeDecision({ notificationReason: "threshold_breach" }), "");
expect(msg.severity).toBe("warning");
});
it("returns success for container_resolved", () => {
const msg = builder.buildMessage(
makeDockerMonitor({ status: "up" }),
makeCheck(),
makeDecision({ notificationReason: "threshold_resolved" }),
""
);
expect(msg.severity).toBe("success");
@@ -236,9 +288,9 @@ describe("NotificationMessageBuilder", () => {
describe("monitor_down", () => {
it("includes response code and error message when present", () => {
const monitor = makeMonitor({ status: "down" });
const response = makeStatusResponse({ code: 503, message: "Service Unavailable" });
const check = makeCheck({ statusCode: 503, message: "Service Unavailable" });
const msg = builder.buildMessage(monitor, response, makeDecision(), "");
const msg = builder.buildMessage(monitor, check, makeDecision(), "");
expect(msg.content.details).toContain("Response Code: 503");
expect(msg.content.details).toContain("Error: Service Unavailable");
@@ -246,9 +298,9 @@ describe("NotificationMessageBuilder", () => {
it("omits response code when falsy", () => {
const monitor = makeMonitor({ status: "down" });
const response = makeStatusResponse({ code: 0, message: "" });
const check = makeCheck({ statusCode: 0, message: "" });
const msg = builder.buildMessage(monitor, response, makeDecision(), "");
const msg = builder.buildMessage(monitor, check, makeDecision(), "");
expect(msg.content.details).not.toContainEqual(expect.stringContaining("Response Code:"));
expect(msg.content.details).not.toContainEqual(expect.stringContaining("Error:"));
@@ -256,7 +308,7 @@ describe("NotificationMessageBuilder", () => {
it("includes URL, Status, and Type in details", () => {
const monitor = makeMonitor({ status: "down" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), "");
expect(msg.content.details).toContain("URL: https://example.com");
expect(msg.content.details).toContain("Status: Down");
@@ -265,13 +317,13 @@ describe("NotificationMessageBuilder", () => {
it("sets summary text", () => {
const monitor = makeMonitor({ status: "down" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), "");
expect(msg.content.summary).toBe('Monitor "Test Monitor" is currently down and unreachable.');
});
it("sets timestamp", () => {
const monitor = makeMonitor({ status: "down" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), "");
expect(msg.content.timestamp).toBeInstanceOf(Date);
});
});
@@ -279,7 +331,7 @@ describe("NotificationMessageBuilder", () => {
describe("monitor_up", () => {
it("includes recovery details", () => {
const monitor = makeMonitor({ status: "up", type: "http" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), "");
expect(msg.content.title).toBe("Monitor Recovered: Test Monitor");
expect(msg.content.summary).toBe('Monitor "Test Monitor" is back up and operational.');
@@ -288,30 +340,33 @@ describe("NotificationMessageBuilder", () => {
});
describe("threshold_breach", () => {
it("includes threshold breaches in content", () => {
it("includes the breaches flagged on the decision in content", () => {
const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.9, temperature: [50] } as any }),
} as any);
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const decision = makeDecision({
notificationReason: "threshold_breach",
thresholdBreaches: { cpu: true, memory: false, disk: false, temp: false },
});
const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), "");
const msg = builder.buildMessage(monitor, check, decision, "");
expect(msg.content.thresholds).toEqual(
expect.arrayContaining([
expect.objectContaining({
metric: "cpu",
currentValue: 90,
threshold: 80,
}),
])
);
expect(msg.content.thresholds).toEqual([expect.objectContaining({ metric: "cpu", currentValue: 90, threshold: 80 })]);
});
it("leaves thresholds empty when the decision carries no breaches", () => {
const monitor = makeMonitor({ status: "up", type: "hardware", cpuAlertThreshold: 80 });
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const msg = builder.buildMessage(monitor, check, makeDecision({ notificationReason: "threshold_breach" }), "");
expect(msg.content.thresholds).toEqual([]);
});
});
describe("threshold_resolved", () => {
it("includes resolved details", () => {
const monitor = makeMonitor({ status: "up", type: "hardware" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
const msg = builder.buildMessage(monitor, makeCheck(), makeDecision({ notificationReason: "threshold_resolved" }), "");
expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor");
expect(msg.content.summary).toBe('Monitor "Test Monitor" thresholds have returned to normal.');
@@ -319,6 +374,61 @@ describe("NotificationMessageBuilder", () => {
});
});
describe("container_breach", () => {
it("lists one detail line per breaching container", () => {
const check = makeCheck({
status: true,
containers: makeContainers([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]),
});
const msg = builder.buildMessage(makeDockerMonitor(), check, makeDecision({ notificationReason: "threshold_breach" }), "");
expect(msg.content.title).toBe("Container Alert: Test Monitor");
expect(msg.content.summary).toBe('2 container(s) on "Test Monitor" need attention.');
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "db: stopped (exit code 1)", "api: unhealthy"]);
expect(msg.content.thresholds).toBeUndefined();
});
it("only lists containers matching an enabled switch", () => {
const check = makeCheck({
status: true,
containers: makeContainers([
{ name: "db", state: "exited", exitCode: 1 },
{ name: "api", health: "unhealthy" },
]),
});
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false });
const msg = builder.buildMessage(monitor, check, makeDecision({ notificationReason: "threshold_breach" }), "");
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "api: unhealthy"]);
});
it("lists no containers when the check carries no container data", () => {
const check = makeCheck({ status: false, message: "connect ECONNREFUSED" });
const msg = builder.buildMessage(makeDockerMonitor(), check, makeDecision({ notificationReason: "threshold_breach" }), "");
expect(msg.content.summary).toBe('0 container(s) on "Test Monitor" need attention.');
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker"]);
});
});
describe("container_resolved", () => {
it("includes recovered details", () => {
const msg = builder.buildMessage(
makeDockerMonitor({ status: "up" }),
makeCheck(),
makeDecision({ notificationReason: "threshold_resolved" }),
""
);
expect(msg.content.title).toBe("Containers Recovered: Test Monitor");
expect(msg.content.summary).toBe('All containers on "Test Monitor" are back to normal.');
expect(msg.content.details).toContain("Status: Up");
});
});
describe("default content", () => {
it("builds default content for unhandled notification type", () => {
// Force default by creating a scenario where type falls through
@@ -329,7 +439,7 @@ describe("NotificationMessageBuilder", () => {
// Instead, let's verify the default case exists by checking that
// non-standard types would get default content
const monitor = makeMonitor({ status: "up", type: "http" });
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
const msg = builder.buildMessage(monitor, makeCheck(), makeDecision(), "");
// monitor_up is handled, so content should be monitor_up specific
expect(msg.content.title).toBe("Monitor Recovered: Test Monitor");
@@ -356,7 +466,7 @@ describe("NotificationMessageBuilder", () => {
describe("buildContent edge cases", () => {
it("returns default content for unhandled notification type", () => {
const monitor = makeMonitor({ status: "up", type: "http" });
const result = (builder as any).buildContent("test", monitor, makeStatusResponse());
const result = (builder as any).buildContent("test", monitor, makeCheck());
expect(result.title).toBe("Monitor: Test Monitor");
expect(result.summary).toBe('Status update for monitor "Test Monitor".');
@@ -369,410 +479,199 @@ describe("NotificationMessageBuilder", () => {
// ── extractThresholdBreaches ─────────────────────────────────────────
describe("extractThresholdBreaches", () => {
it("returns empty array for non-hardware monitor", () => {
const monitor = makeMonitor({ type: "http" });
const response = makeStatusResponse();
const none: HardwareBreaches = { cpu: false, memory: false, disk: false, temp: false };
const hardwareMonitor = (overrides?: Partial<Monitor>) =>
makeMonitor({
type: "hardware",
cpuAlertThreshold: 80,
memoryAlertThreshold: 70,
diskAlertThreshold: 85,
tempAlertThreshold: 65,
...overrides,
});
const breaches = builder.extractThresholdBreaches(monitor, response);
it("returns empty when no breaches are supplied", () => {
const check = makeCheck({ cpu: { usage_percent: 0.99, temperature: [99] } });
expect(breaches).toEqual([]);
expect(builder.extractThresholdBreaches(hardwareMonitor(), check, undefined)).toEqual([]);
});
it("returns empty array when payload is missing", () => {
const monitor = makeMonitor({ type: "hardware" });
const response = makeStatusResponse({ payload: undefined } as any);
it("returns empty when every flag is false, whatever the check says", () => {
const check = makeCheck({
cpu: { usage_percent: 0.99, temperature: [99] },
memory: { usage_percent: 0.99 },
disk: [{ usage_percent: 0.99 }],
});
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches).toEqual([]);
expect(builder.extractThresholdBreaches(hardwareMonitor(), check, none)).toEqual([]);
});
it("returns empty array when hardware data is missing", () => {
const monitor = makeMonitor({ type: "hardware" });
const response = makeStatusResponse({ payload: { data: undefined } } as any);
it("returns empty for a non-hardware monitor even when flags are set", () => {
const check = makeCheck({ cpu: { usage_percent: 0.99 } });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches).toEqual([]);
expect(builder.extractThresholdBreaches(makeMonitor({ type: "http" }), check, { ...none, cpu: true })).toEqual([]);
});
// ── CPU ──────────────────────────────────────────────────────────
// The flags are the verdict. The builder formats what it is told is breaching and never re-runs the comparison.
it("reports a flagged metric without re-testing it against the threshold", () => {
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [50] } });
describe("cpu threshold", () => {
it("detects CPU breach when usage exceeds threshold", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.9, temperature: [50] } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches).toEqual([{ metric: "cpu", currentValue: 50, threshold: 80, unit: "%", formattedValue: "50.0%" }]);
});
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "cpu",
currentValue: 90,
threshold: 80,
unit: "%",
formattedValue: "90.0%",
})
);
describe("cpu", () => {
it("formats cpu usage as a percentage when flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
expect(breaches).toEqual([{ metric: "cpu", currentValue: 90, threshold: 80, unit: "%", formattedValue: "90.0%" }]);
});
it("does not report CPU breach when usage is below threshold", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [50] } as any }),
} as any);
it("ignores cpu when not flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.9, temperature: [50] } });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpuAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: undefined });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.99 } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpuAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: null as any });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.99 } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpu usage_percent is undefined", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: undefined } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
it("skips CPU check when cpu object is undefined", () => {
const monitor = makeMonitor({ type: "hardware", cpuAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: undefined as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true });
expect(breaches.find((b) => b.metric === "cpu")).toBeUndefined();
});
});
// ── Memory ───────────────────────────────────────────────────────
describe("memory", () => {
it("formats memory usage as a percentage when flagged", () => {
const check = makeCheck({ memory: { usage_percent: 0.85 } });
describe("memory threshold", () => {
it("detects memory breach when usage exceeds threshold", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ memory: { usage_percent: 0.85 } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "memory",
currentValue: 85,
threshold: 70,
unit: "%",
formattedValue: "85.0%",
})
);
expect(breaches).toEqual([{ metric: "memory", currentValue: 85, threshold: 70, unit: "%", formattedValue: "85.0%" }]);
});
it("does not report memory breach when usage is below threshold", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 90 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ memory: { usage_percent: 0.5 } as any }),
} as any);
it("ignores memory when not flagged", () => {
const check = makeCheck({ memory: { usage_percent: 0.85 } });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memoryAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: undefined });
const response = makeStatusResponse({
payload: makeHardwarePayload({ memory: { usage_percent: 0.99 } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memoryAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: null as any });
const response = makeStatusResponse({
payload: makeHardwarePayload({ memory: { usage_percent: 0.99 } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memory usage_percent is undefined", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ memory: { usage_percent: undefined } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
it("skips memory check when memory object is undefined", () => {
const monitor = makeMonitor({ type: "hardware", memoryAlertThreshold: 70 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ memory: undefined as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
expect(breaches.find((b) => b.metric === "memory")).toBeUndefined();
});
});
// ── Disk ─────────────────────────────────────────────────────────
describe("disk", () => {
it("formats the highest usage across disks when flagged", () => {
const check = makeCheck({ disk: [{ usage_percent: 0.5 }, { usage_percent: 0.95 }, { usage_percent: 0.7 }] });
describe("disk threshold", () => {
it("detects disk breach using highest usage across multiple disks", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({
disk: [{ usage_percent: 0.5 }, { usage_percent: 0.95 }, { usage_percent: 0.7 }] as any,
}),
} as any);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "disk",
currentValue: 95,
threshold: 80,
formattedValue: "95.0%",
})
);
expect(breaches).toEqual([{ metric: "disk", currentValue: 95, threshold: 85, unit: "%", formattedValue: "95.0%" }]);
});
it("does not report disk breach when all disks are below threshold", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 90 });
const response = makeStatusResponse({
payload: makeHardwarePayload({
disk: [{ usage_percent: 0.5 }, { usage_percent: 0.6 }] as any,
}),
} as any);
it("treats a disk with no usage figure as zero when finding the highest", () => {
const check = makeCheck({ disk: [{ usage_percent: undefined }, { usage_percent: 0.3 }] });
const breaches = builder.extractThresholdBreaches(monitor, response);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, disk: true });
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
expect(breaches).toEqual([expect.objectContaining({ metric: "disk", currentValue: 30 })]);
});
it("skips disk check when diskAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: undefined });
const response = makeStatusResponse({
payload: makeHardwarePayload({ disk: [{ usage_percent: 0.99 }] as any }),
} as any);
it("ignores disk when not flagged", () => {
const check = makeCheck({ disk: [{ usage_percent: 0.95 }] });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
it("skips disk check when diskAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: null as any });
const response = makeStatusResponse({
payload: makeHardwarePayload({ disk: [{ usage_percent: 0.99 }] as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
it("skips disk check when disk is not an array", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ disk: "not-an-array" as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
it("skips disks with undefined usage_percent", () => {
const monitor = makeMonitor({ type: "hardware", diskAlertThreshold: 50 });
const response = makeStatusResponse({
payload: makeHardwarePayload({
disk: [{ usage_percent: undefined }, { usage_percent: 0.3 }] as any,
}),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true });
expect(breaches.find((b) => b.metric === "disk")).toBeUndefined();
});
});
// ── Temperature ──────────────────────────────────────────────────
describe("temperature", () => {
it("formats the highest core temperature when flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [65, 75, 68] } });
describe("temperature threshold", () => {
it("detects temperature breach from array of temperatures", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [65, 75, 68] } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, temp: true });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "temp",
currentValue: 75,
threshold: 70,
unit: "°C",
formattedValue: "75.0°C",
})
);
expect(breaches).toEqual([{ metric: "temp", currentValue: 75, threshold: 65, unit: "°C", formattedValue: "75.0°C" }]);
});
it("handles single temperature value (non-array)", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: 80 } as any }),
} as any);
it("ignores temperature when not flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.5, temperature: [99] } });
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches).toContainEqual(
expect.objectContaining({
metric: "temp",
currentValue: 80,
threshold: 70,
})
);
});
it("does not report temperature breach when below threshold", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [60, 65] } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
// A temperature exactly AT the threshold must not report a breach here, matching
// StatusService.computeHardwareStatus's own strict `>` comparison (cpu/memory/disk
// already agree with this file on `>`; temp alone used to diverge with `>=`, so an
// incident-driving check at exactly the threshold never fired but the notification
// content for a breach some OTHER metric triggered could still list temp as breaching).
it("does not report temperature breach when exactly at threshold", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [80] } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when tempAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [99] } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when tempAlertThreshold is null", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: null as any });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [99] } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when cpu.temperature is falsy", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: null } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when cpu object is undefined", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 70 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: undefined as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, cpu: true });
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
});
// ── Combined ─────────────────────────────────────────────────────
describe("combined thresholds", () => {
it("detects multiple breaches simultaneously", () => {
const monitor = makeMonitor({
type: "hardware",
cpuAlertThreshold: 80,
memoryAlertThreshold: 70,
diskAlertThreshold: 85,
tempAlertThreshold: 65,
describe("combined", () => {
it("reports every flagged metric in cpu, memory, disk, temp order", () => {
const check = makeCheck({
cpu: { usage_percent: 0.9, temperature: [70] },
memory: { usage_percent: 0.85 },
disk: [{ usage_percent: 0.95 }],
});
const response = makeStatusResponse({
payload: makeHardwarePayload({
cpu: { usage_percent: 0.9, temperature: [70] } as any,
memory: { usage_percent: 0.85 } as any,
disk: [{ usage_percent: 0.95 }] as any,
}),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { cpu: true, memory: true, disk: true, temp: true });
const metrics = breaches.map((b) => b.metric);
expect(metrics).toContain("cpu");
expect(metrics).toContain("memory");
expect(metrics).toContain("disk");
expect(metrics).toContain("temp");
expect(breaches.map((b) => b.metric)).toEqual(["cpu", "memory", "disk", "temp"]);
});
it("reports only the flagged subset", () => {
const check = makeCheck({
cpu: { usage_percent: 0.9, temperature: [70] },
memory: { usage_percent: 0.85 },
disk: [{ usage_percent: 0.95 }],
});
const breaches = builder.extractThresholdBreaches(hardwareMonitor(), check, { ...none, memory: true, temp: true });
expect(breaches.map((b) => b.metric)).toEqual(["memory", "temp"]);
});
});
});
// ── buildThresholdBreachMessage ─────────────────────────────────────────
describe("buildThresholdBreachMessage", () => {
const none: HardwareBreaches = { cpu: false, memory: false, disk: false, temp: false };
const hardwareMonitor = () => makeMonitor({ type: "hardware", cpuAlertThreshold: 80, memoryAlertThreshold: 80 });
it("describes a single flagged metric", () => {
const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] } });
const message = builder.buildThresholdBreachMessage(hardwareMonitor(), check, { ...none, cpu: true });
expect(message).toBe("CPU: 95.0% (threshold: 80%)");
});
it("joins several flagged metrics with a comma", () => {
const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] }, memory: { usage_percent: 0.9 } });
const message = builder.buildThresholdBreachMessage(hardwareMonitor(), check, { ...none, cpu: true, memory: true });
expect(message).toBe("CPU: 95.0% (threshold: 80%), MEMORY: 90.0% (threshold: 80%)");
});
it("falls back when no metric is flagged", () => {
const check = makeCheck({ cpu: { usage_percent: 0.95, temperature: [50] } });
expect(builder.buildThresholdBreachMessage(hardwareMonitor(), check, none)).toBe("Threshold breach detected");
expect(builder.buildThresholdBreachMessage(hardwareMonitor(), check, undefined)).toBe("Threshold breach detected");
});
it("lists the breaching containers for a docker monitor", () => {
const check = makeCheck({
status: true,
containers: makeContainers([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]),
});
const message = builder.buildThresholdBreachMessage(makeDockerMonitor(), check, undefined);
expect(message).toBe("db: stopped (exit code 1), api: unhealthy");
});
it("falls back when no container is breaching", () => {
const check = makeCheck({ status: true, containers: makeContainers([{ name: "web" }]) });
expect(builder.buildThresholdBreachMessage(makeDockerMonitor(), check, undefined)).toBe("Container alert");
});
});
});
@@ -3,7 +3,7 @@ import { NotificationsService } from "../../../src/domain/notifications/notifica
import { createMockLogger } from "../../helpers/createMockLogger.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
import type { Notification } from "../../../src/domain/notifications/notification.type.ts";
import type { MonitorStatusResponse } from "../../../src/types/network.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -33,7 +33,7 @@ const createSettingsService = (clientHost = "https://app.example.com") => ({
const createMessageBuilder = () => ({
buildMessage: jest.fn().mockReturnValue({ type: "monitor_down", content: { title: "Down" } }),
extractThresholdBreaches: jest.fn(),
buildThresholdBreachMessage: jest.fn(),
});
const createService = (overrides?: Record<string, unknown>) => {
@@ -130,7 +130,17 @@ const makeDecision = (overrides?: Partial<MonitorActionDecision>): MonitorAction
...overrides,
});
const makeStatusResponse = () => ({ monitorId: "mon-1", status: false, code: 500 }) as unknown as MonitorStatusResponse;
const makeCheck = (overrides?: Partial<Check>): Check => ({
id: "check-1",
metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" },
status: false,
responseTime: 100,
statusCode: 500,
message: "Internal Server Error",
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
...overrides,
});
// ── Tests ────────────────────────────────────────────────────────────────────
@@ -140,7 +150,7 @@ describe("NotificationsService", () => {
describe("handleNotifications", () => {
it("returns false when shouldSendNotification is false", async () => {
const { service } = createService();
const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision({ shouldSendNotification: false }));
const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision({ shouldSendNotification: false }));
expect(result).toBe(false);
});
@@ -148,7 +158,7 @@ describe("NotificationsService", () => {
const { service, notificationsRepository, emailProvider } = createService();
(notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification({ type: "email" })]);
const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision());
const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision());
expect(result).toBe(true);
expect(emailProvider.sendMessage).toHaveBeenCalledTimes(1);
@@ -160,7 +170,7 @@ describe("NotificationsService", () => {
const deps = createService();
(deps.notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification({ type })]);
await deps.service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision());
await deps.service.handleNotifications(makeMonitor(), makeCheck(), makeDecision());
const providerMap: Record<string, ReturnType<typeof createProvider>> = {
webhook: deps.webhookProvider,
@@ -183,7 +193,7 @@ describe("NotificationsService", () => {
const { service, notificationsRepository, logger } = createService();
(notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification({ type: "carrier_pigeon" as any })]);
const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision());
const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision());
expect(result).toBe(false);
expect(logger.warn).toHaveBeenCalledWith(
@@ -197,7 +207,7 @@ describe("NotificationsService", () => {
const { service, notificationsRepository, logger } = createService({ notificationMessageBuilder });
(notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification()]);
const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision());
const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision());
expect(result).toBe(false);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Notification message not provided" }));
@@ -207,7 +217,7 @@ describe("NotificationsService", () => {
const { service, notificationsRepository } = createService();
(notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([]);
const result = await service.handleNotifications(makeMonitor({ notifications: undefined as any }), makeStatusResponse(), makeDecision());
const result = await service.handleNotifications(makeMonitor({ notifications: undefined as any }), makeCheck(), makeDecision());
expect(result).toBe(true);
expect(notificationsRepository.findNotificationsByIds).toHaveBeenCalledWith([]);
@@ -222,7 +232,7 @@ describe("NotificationsService", () => {
emailProvider.sendMessage.mockResolvedValue(true);
slackProvider.sendMessage.mockResolvedValue(false);
const result = await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision());
const result = await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision());
expect(result).toBe(false);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 success, 1 failure") }));
@@ -233,7 +243,7 @@ describe("NotificationsService", () => {
const { service, notificationsRepository, notificationMessageBuilder } = createService({ settingsService });
(notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([makeNotification()]);
await service.handleNotifications(makeMonitor(), makeStatusResponse(), makeDecision());
await service.handleNotifications(makeMonitor(), makeCheck(), makeDecision());
expect(notificationMessageBuilder.buildMessage).toHaveBeenCalledWith(
expect.anything(),
@@ -311,6 +321,23 @@ describe("NotificationsService", () => {
const result = await service.testAllNotifications(["notif-1"]);
expect(result).toBe(false);
});
it("counts a provider that throws as a failed test and still tests the others", async () => {
const { service, notificationsRepository, emailProvider, slackProvider, logger } = createService();
emailProvider.sendTestAlert.mockRejectedValue(new Error("Email is not configured. Set the system email host in settings."));
(notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([
makeNotification({ type: "email" }),
makeNotification({ type: "slack" }),
]);
const result = await service.testAllNotifications(["notif-1", "notif-2"]);
expect(result).toBe(false);
expect(slackProvider.sendTestAlert).toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ message: "Test notification failed: Email is not configured. Set the system email host in settings." })
);
});
});
// ── CRUD operations ──────────────────────────────────────────────────────
+288 -209
View File
@@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts";
import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts";
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts";
@@ -98,18 +98,6 @@ const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
...overrides,
}) as Monitor;
const makeStatusResponse = (overrides?: Partial<MonitorStatusResponse>): MonitorStatusResponse =>
({
monitorId: "mon-1",
teamId: "team-1",
type: "http",
status: true,
code: 200,
message: "OK",
responseTime: 100,
...overrides,
}) as MonitorStatusResponse;
const makeCheck = (overrides?: Partial<Check>): Check =>
({
id: "check-1",
@@ -143,92 +131,86 @@ const makeExistingStats = (overrides?: Record<string, unknown>) => ({
// ── Tests ────────────────────────────────────────────────────────────────────
describe("StatusService", () => {
// ── updateRunningStats ───────────────────────────────────────────────────
// ── running stats (via updateMonitorStatus) ──────────────────────────────
describe("updateRunningStats", () => {
// The service now delegates the entire running-stats computation to the repository's
// atomic `updateByMonitorId`, so the service-level tests only verify forwarding and error
// handling. The math (running average, uptimePercentage, timeOfLastFailure state machine)
// is a repository-level concern and is exercised by the repository tests.
describe("running stats", () => {
// The service delegates the running-stats computation to the repository's atomic `updateByMonitorId`,
// so these cases only verify what is forwarded and that a failure never blocks the status update.
// The math (running average, uptimePercentage, timeOfLastFailure) is exercised by the repository tests.
const run = async (check: Check) => {
const monitor = makeMonitor();
const deps = createService();
(deps.monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(deps.monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({});
const result = await deps.service.updateMonitorStatus(check, monitor);
return { ...deps, result };
};
it("forwards a successful check to updateByMonitorId", async () => {
const { service, monitorStatsRepository } = createService();
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({});
const { monitorStatsRepository } = await run(makeCheck({ responseTime: 50 }));
const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse({ responseTime: 50 }));
expect(result).toBe(true);
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith(
"mon-1",
expect.objectContaining({
status: true,
responseTime: 50,
now: expect.any(Number),
})
expect.objectContaining({ status: true, responseTime: 50, now: expect.any(Number) })
);
});
it("forwards a failed check to updateByMonitorId", async () => {
const { service, monitorStatsRepository } = createService();
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({});
const { monitorStatsRepository } = await run(makeCheck({ status: false, responseTime: 100 }));
const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse({ status: false, responseTime: 100 }));
expect(result).toBe(true);
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false, responseTime: 100 }));
});
it("passes 0 when responseTime is undefined", async () => {
const { service, monitorStatsRepository } = createService();
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({});
await service.updateRunningStats(makeMonitor(), makeStatusResponse({ responseTime: undefined }));
const { monitorStatsRepository } = await run(makeCheck({ responseTime: undefined as any }));
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 }));
});
it("forwards responseTime of 0 as 0 (falsy but defined)", async () => {
const { service, monitorStatsRepository } = createService();
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({});
await service.updateRunningStats(makeMonitor(), makeStatusResponse({ responseTime: 0 }));
const { monitorStatsRepository } = await run(makeCheck({ responseTime: 0 }));
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ responseTime: 0 }));
});
it("coerces a non-true status to false (e.g. undefined)", async () => {
const { service, monitorStatsRepository } = createService();
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockResolvedValue({});
await service.updateRunningStats(makeMonitor(), makeStatusResponse({ status: undefined as unknown as boolean }));
const { monitorStatsRepository } = await run(makeCheck({ status: undefined as any }));
expect(monitorStatsRepository.updateByMonitorId).toHaveBeenCalledWith("mon-1", expect.objectContaining({ status: false }));
});
it("returns false and logs error when updateByMonitorId throws", async () => {
const { service, logger, monitorStatsRepository } = createService();
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("db write failed"));
it("warns once and still updates the status when updateByMonitorId throws", async () => {
const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" });
const { service, logger, monitorsRepository, monitorStatsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down"));
const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse());
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result).toBe(false);
expect(logger.error).toHaveBeenCalledWith(
expect(result.monitor.status).toBe("up");
expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({
service: "StatusService",
method: "updateRunningStats",
message: "db write failed",
method: "tryUpdateRunningStats",
message: "stats db down",
stack: expect.any(String),
})
);
expect(logger.error).not.toHaveBeenCalled();
});
it("logs error with 'Unknown error' for non-Error exceptions", async () => {
const { service, logger, monitorStatsRepository } = createService();
it("warns with 'Unknown error' for non-Error rejections", async () => {
const monitor = makeMonitor();
const { service, logger, monitorsRepository, monitorStatsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue("string error");
const result = await service.updateRunningStats(makeMonitor(), makeStatusResponse());
await service.updateMonitorStatus(makeCheck(), monitor);
expect(result).toBe(false);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
});
});
@@ -241,7 +223,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.prevStatus).toBe("up");
@@ -252,7 +234,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(monitorsRepository.findById).not.toHaveBeenCalled();
expect(result.prevStatus).toBe("up");
@@ -265,7 +247,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
// Atomic push should have been called with the correct status and window size
expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith(
@@ -286,7 +268,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }), monitor);
await service.updateMonitorStatus(makeCheck({ id: "new-check" }), monitor);
expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS);
expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" }));
@@ -304,7 +286,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -322,7 +304,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("up");
@@ -339,7 +321,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -359,7 +341,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -376,7 +358,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
@@ -388,7 +370,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
await service.updateMonitorStatus(makeCheck(), monitor);
expect(monitor.statusWindow).toEqual([true]);
});
@@ -399,7 +381,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
await service.updateMonitorStatus(makeCheck(), monitor);
expect(monitor.recentChecks).toHaveLength(1);
});
@@ -409,7 +391,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error"));
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Failed to update monitor");
await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Failed to update monitor");
});
it("throws AppError with 'Unknown error' for non-Error exceptions", async () => {
@@ -417,7 +399,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error");
await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Unknown error");
await expect(service.updateMonitorStatus(makeCheck(), monitor)).rejects.toThrow("Unknown error");
});
it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => {
@@ -435,7 +417,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
@@ -453,7 +435,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -475,7 +457,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.monitor.status).toBe("up");
});
@@ -493,7 +475,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.monitor.status).toBe("down");
});
@@ -514,7 +496,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.monitor.status).toBe("up");
expect(result.statusChanged).toBe(false); // was effectively up already — no notification storm
@@ -531,7 +513,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.monitor.status).toBe("down");
expect(result.statusChanged).toBe(true); // down must surface so an incident is opened
@@ -548,7 +530,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -565,7 +547,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -580,7 +562,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -596,7 +578,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
@@ -611,7 +593,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: false }), monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -631,7 +613,7 @@ describe("StatusService", () => {
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.monitor.status).toBe("up");
});
@@ -646,43 +628,12 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor);
const result = await service.updateMonitorStatus(makeCheck({ status: true }), monitor);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
});
it("logs a warning but still succeeds when running-stats update fails mid-flight", async () => {
const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" });
const { service, logger, monitorsRepository, monitorStatsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
(monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down"));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor);
expect(result.monitor.status).toBe("up");
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({
service: "StatusService",
method: "updateMonitorStatus",
message: expect.stringContaining("Stats update failed"),
})
);
});
it("returns code and timestamp in result", async () => {
const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5 });
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck(), monitor);
expect(result.code).toBe(201);
expect(result.timestamp).toBeGreaterThan(0);
});
// ── Hardware threshold breach tests ──────────────────────────────────
describe("hardware threshold breaches", () => {
@@ -703,13 +654,12 @@ describe("StatusService", () => {
...overrides,
});
const makeHardwareResponse = (payload: HardwareStatusPayload, overrides?: Partial<MonitorStatusResponse<HardwareStatusPayload>>) =>
makeStatusResponse({
type: "hardware",
const makeHardwareCheck = (metrics: Pick<Check, "cpu" | "memory" | "disk" | "host">, overrides?: Partial<Check>) =>
makeCheck({
status: true,
payload,
...metrics,
...overrides,
} as any) as MonitorStatusResponse<HardwareStatusPayload>;
});
it("detects CPU threshold breach and decrements counter", async () => {
const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 });
@@ -717,8 +667,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.cpu).toBe(true);
expect(result.thresholdBreaches?.memory).toBe(false);
@@ -734,8 +684,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.memory).toBe(true);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
@@ -749,10 +699,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.disk).toBe(true);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
@@ -766,10 +714,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.temp).toBe(true);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
@@ -788,10 +734,13 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} },
} as any);
await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({
cpu: { usage_percent: 0.1, temperature: [30] },
memory: { usage_percent: 0.1 },
disk: [{ usage_percent: 0.1 }],
host: {},
});
await service.updateMonitorStatus(check, monitor);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
expect(patch.cpuAlertCounter).toBe(5);
@@ -809,8 +758,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.monitor.status).toBe("breached");
expect(result.statusChanged).toBe(false);
@@ -822,10 +771,13 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({
cpu: { usage_percent: 0.1, temperature: [30] },
memory: { usage_percent: 0.1 },
disk: [{ usage_percent: 0.1 }],
host: {},
});
const result = await service.updateMonitorStatus(check, monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("up");
@@ -844,11 +796,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
status: false,
data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.monitor.status).toBe("down");
});
@@ -859,8 +808,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.cpu).toBe(false);
});
@@ -871,8 +820,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.memory).toBe(false);
});
@@ -883,10 +832,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.temp).toBe(false);
});
@@ -897,36 +844,25 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.temp).toBe(false);
});
it("skips threshold evaluation when payload is undefined", async () => {
const monitor = makeHardwareMonitor();
it("reports no breaches when the check carries no hardware metrics", async () => {
// A stored hardware check with no metrics (cpu/memory/disk/host all absent) must not
// throw and must not report any breach.
const monitor = makeHardwareMonitor({ cpuAlertCounter: 1 });
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse(undefined as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const result = await service.updateMonitorStatus(makeCheck(), monitor);
expect(result.thresholdBreaches).toBeUndefined();
});
it("skips threshold evaluation when payload.data is undefined", async () => {
const monitor = makeHardwareMonitor();
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
expect(result.thresholdBreaches).toBeUndefined();
expect(result.thresholdBreaches).toEqual({ cpu: false, memory: false, disk: false, temp: false });
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
});
it("does not set thresholdBreaches for non-hardware monitors", async () => {
@@ -939,7 +875,7 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor);
const result = await service.updateMonitorStatus(makeCheck(), monitor);
expect(result.thresholdBreaches).toBeUndefined();
});
@@ -950,10 +886,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.disk).toBe(false);
});
@@ -964,10 +898,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.disk).toBe(false);
});
@@ -978,10 +910,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.cpu).toBe(false);
expect(result.thresholdBreaches?.temp).toBe(false);
@@ -993,10 +923,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.memory).toBe(false);
});
@@ -1007,10 +935,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.thresholdBreaches?.disk).toBe(false);
});
@@ -1023,8 +949,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any);
const result = await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} });
const result = await service.updateMonitorStatus(check, monitor);
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
expect(patch.cpuAlertCounter).toBe(2);
@@ -1046,11 +972,8 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
status: false,
data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} },
} as any);
const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor);
const check = makeHardwareCheck({ cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, { status: false });
const result = await service.updateMonitorStatus(check, monitor);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
@@ -1067,10 +990,13 @@ describe("StatusService", () => {
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const response = makeHardwareResponse({
data: { cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {} },
} as any);
await service.updateMonitorStatus(response, makeCheck(), monitor);
const check = makeHardwareCheck({
cpu: { usage_percent: 0.9, temperature: [90] },
memory: { usage_percent: 0.9 },
disk: [{ usage_percent: 0.95 }],
host: {},
});
await service.updateMonitorStatus(check, monitor);
expect(monitor.cpuAlertCounter).toBe(0);
expect(monitor.memoryAlertCounter).toBe(0);
@@ -1086,6 +1012,159 @@ describe("StatusService", () => {
// they're private implementation details; exposing them publicly would
// widen the API surface just for testability.
describe("docker container alerts", () => {
const makeDockerMonitor = (overrides?: Partial<Monitor>) =>
makeMonitor({
type: "docker",
url: "unix:///var/run/docker.sock",
statusWindow: [true, true, true, true],
statusWindowSize: 5,
status: "up",
dockerAlertOnStopped: true,
dockerAlertOnUnhealthy: true,
...overrides,
});
const makeContainers = (containers: Partial<DockerContainerInfo>[]): DockerContainerInfo[] =>
containers.map((container, index) => ({
id: `container-${index}`,
name: `container-${index}`,
image: "nginx:latest",
state: "running",
status: "Up 3 hours",
health: "none",
...container,
}));
const makeDockerCheck = (containers: DockerContainerInfo[] | null, overrides?: Partial<Check>) =>
makeCheck({
status: true,
...(containers ? { containers } : {}),
...overrides,
});
const run = async (monitor: Monitor, check: Check) => {
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(check, monitor);
return { result, monitorsRepository };
};
const stopped = { name: "db", state: "exited" as const, exitCode: 1 };
const unhealthy = { name: "api", health: "unhealthy" as const };
it("goes to breached on the first check with a stopped container", async () => {
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped])));
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("breached");
});
it("goes to breached on the first check with an unhealthy container", async () => {
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([unhealthy])));
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("breached");
});
it("stays up on a stopped container when both switches are off", async () => {
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false });
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped, unhealthy])));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
});
it("stays up on a stopped container when only the unhealthy switch is on", async () => {
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false });
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped])));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
});
it("stays up on an exited container with code 0", async () => {
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "job", state: "exited", exitCode: 0 }])));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
});
it("stays breached without a transition while a container is still down", async () => {
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([stopped])));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("breached");
});
it("stays up without a transition while every container is clean", async () => {
const { result } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([{ name: "web" }])));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
});
it("recovers from breached to up on the first clean check", async () => {
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerCheck(makeContainers([{ name: "web" }])));
expect(result.statusChanged).toBe(true);
expect(result.prevStatus).toBe("breached");
expect(result.monitor.status).toBe("up");
});
it("recovers from breached to up when the breaching switch is turned off", async () => {
const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false });
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped])));
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("up");
});
it("does not override down with breached when the host is unreachable", async () => {
const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
const check = makeDockerCheck(null, { status: false, statusCode: 5000 });
const { result } = await run(monitor, check);
expect(result.monitor.status).toBe("down");
});
it("transitions from breached to down when the reachability threshold trips", async () => {
const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
const check = makeDockerCheck(null, { status: false, statusCode: 5000 });
const { result } = await run(monitor, check);
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
});
it("does not evaluate containers while the status window is short", async () => {
const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 });
const { result } = await run(monitor, makeDockerCheck(makeContainers([stopped])));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("up");
});
it("does not evaluate a failed check with a null payload", async () => {
const monitor = makeDockerMonitor({ status: "breached" });
const check = makeDockerCheck(null, { status: false, statusCode: 5000 });
const { result } = await run(monitor, check);
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("breached");
});
it("does not touch the hardware counters or thresholdBreaches", async () => {
const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerCheck(makeContainers([stopped])));
expect(result.thresholdBreaches).toBeUndefined();
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
expect(patch.cpuAlertCounter).toBeUndefined();
expect(patch.status).toBe("breached");
});
});
describe("computeReachability (pure)", () => {
const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => {
const { service } = createService();
@@ -64,8 +64,6 @@ const makeStatusChange = (overrides: Partial<StatusChangeResult> & { status?: st
monitor: { id: "m1", status } as any,
statusChanged: false,
prevStatus: "up",
code: 200,
timestamp: 0,
...rest,
} as StatusChangeResult;
};
@@ -92,7 +90,6 @@ const createPipeline = (overrides?: Record<string, any>) => {
},
checkService: {
toCheck: jest.fn<any>().mockReturnValue({ id: "check-1" }),
toStatusResponse: jest.fn<any>().mockReturnValue({ monitorId: "m1", status: true, code: 200, message: "OK" }),
createChecks: jest.fn<any>().mockResolvedValue([]),
},
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
@@ -519,24 +516,6 @@ describe("WorkerPipeline", () => {
expect(check.egressStatus).toBe("ok");
});
it("buffers the check when the egress assessment rejects", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
egressService: { assessAfterFailure: jest.fn<any>().mockRejectedValue(new Error("settings unavailable")) },
});
const result = await pipeline.produce(makeMonitor());
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
expect(check).not.toHaveProperty("egressStatus");
expect(result).toEqual({ status: failingStatus, check });
expect(deps.logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "WorkerPipeline", method: "assessEgress" })
);
});
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
const check: Record<string, unknown> = { id: "check-1" };
const { pipeline, deps } = createPipeline({
@@ -624,7 +603,7 @@ describe("WorkerPipeline", () => {
it("loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
const job = makeEvaluateJob("c1");
const { pipeline, deps } = createPipeline({
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) },
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([makeCheck({ id: "c1" })]) },
});
await pipeline.handleEvaluate(job);
@@ -661,7 +640,9 @@ describe("WorkerPipeline", () => {
it("stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
const job = makeEvaluateJob("c1", "c2", "c3");
const { pipeline, deps } = createPipeline({
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) },
checksRepository: {
findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([makeCheck({ id: "c1" }), makeCheck({ id: "c2" }), makeCheck({ id: "c3" })]),
},
jobsRepository: {
upsertEvaluate: jest.fn<any>(),
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
@@ -679,7 +660,7 @@ describe("WorkerPipeline", () => {
it("pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
const job = makeEvaluateJob("c1", "c-gone");
const { pipeline, deps } = createPipeline({
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }, // c-gone was cleaned up
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([makeCheck({ id: "c1" })]) }, // c-gone was cleaned up
});
await pipeline.handleEvaluate(job);
@@ -695,7 +676,7 @@ describe("WorkerPipeline", () => {
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
const { pipeline, deps } = createPipeline({
monitorsRepository: { findByIdLean: jest.fn<any>().mockResolvedValue(initialMonitor), updateById: jest.fn<any>() },
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) },
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([makeCheck({ id: "c1" }), makeCheck({ id: "c2" })]) },
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(makeStatusChange({ monitor: postWriteMonitor })) },
});
@@ -705,38 +686,34 @@ describe("WorkerPipeline", () => {
expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2);
// First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first
expect(deps.statusService.updateMonitorStatus.mock.calls[0][2]).toBe(initialMonitor);
expect(deps.statusService.updateMonitorStatus.mock.calls[1][2]).toBe(postWriteMonitor);
expect(deps.statusService.updateMonitorStatus.mock.calls[0][1]).toBe(initialMonitor);
expect(deps.statusService.updateMonitorStatus.mock.calls[1][1]).toBe(postWriteMonitor);
});
});
describe("evaluateCheck", () => {
it("threads status, check, and monitor through to the status service", async () => {
it("threads check and monitor through to the status service", async () => {
const { pipeline, deps } = createPipeline();
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const check = { id: "check-1" } as any;
const check = makeCheck({ status: true, statusCode: 200, message: "OK" });
const monitor = makeMonitor({ status: "up" });
await pipeline.evaluateCheck(status, check, monitor);
await pipeline.evaluateCheck(check, monitor);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor);
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(check, monitor);
});
it("returns an evaluation that opens an incident on a down transition", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1" } as any;
const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 });
const check = makeCheck({ status: false, statusCode: 500, message: "Error" });
const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up" });
const { pipeline } = createPipeline({
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
});
const result = await pipeline.evaluateCheck(status, check, makeMonitor());
const result = await pipeline.evaluateCheck(check, makeMonitor());
expect(result).toMatchObject({
monitor: statusChange.monitor,
status,
check,
statusChange,
decision: expect.objectContaining({
shouldCreateIncident: true,
shouldSendNotification: true,
@@ -746,22 +723,20 @@ describe("WorkerPipeline", () => {
});
it("returns an evaluation that resolves an incident on recovery", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down" });
const { pipeline } = createPipeline({
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
});
const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor({ status: "down" }));
const result = await pipeline.evaluateCheck(makeCheck({ status: true, statusCode: 200, message: "OK" }), makeMonitor({ status: "down" }));
expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
});
it("produces a no-op decision when status did not change", async () => {
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
const { pipeline } = createPipeline();
const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor());
const result = await pipeline.evaluateCheck(makeCheck({ status: true, statusCode: 200, message: "OK" }), makeMonitor());
expect(result.decision).toMatchObject({
shouldCreateIncident: false,
@@ -773,17 +748,15 @@ describe("WorkerPipeline", () => {
// ── degraded egress ───────────────────────────────────────────────────
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "degraded" } as any;
const check = makeCheck({ status: false, statusCode: 500, message: "Error", egressStatus: "degraded" });
const monitor = makeMonitor({ status: "up" });
const { pipeline, deps } = createPipeline();
const result = await pipeline.evaluateCheck(status, check, monitor);
const result = await pipeline.evaluateCheck(check, monitor);
expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled();
expect(result.monitor).toBe(monitor);
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
expect(result).not.toHaveProperty("statusChange");
expect(result.decision).toEqual({
shouldCreateIncident: false,
shouldResolveIncident: false,
@@ -795,13 +768,12 @@ describe("WorkerPipeline", () => {
});
it("evaluates a check flagged with egress ok as normal", async () => {
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
const check = { id: "check-1", egressStatus: "ok" } as any;
const check = makeCheck({ status: false, statusCode: 500, message: "Error", egressStatus: "ok" });
const { pipeline, deps } = createPipeline();
await pipeline.evaluateCheck(status, check, makeMonitor());
await pipeline.evaluateCheck(check, makeMonitor());
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(check, expect.anything());
});
});
@@ -812,11 +784,7 @@ describe("WorkerPipeline", () => {
const { pipeline } = createPipeline({
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
});
const result = await pipeline.evaluateCheck(
{ monitorId: "m1", status: true, code: 200, message: "OK" } as any,
{ id: "check-1" } as any,
makeMonitor()
);
const result = await pipeline.evaluateCheck(makeCheck({ status: true, statusCode: 200, message: "OK" }), makeMonitor());
return result.decision;
};
@@ -832,7 +800,7 @@ describe("WorkerPipeline", () => {
});
it("creates incident and notifies when monitor goes down", async () => {
const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 }));
const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({
shouldCreateIncident: true,
shouldSendNotification: true,
@@ -861,14 +829,70 @@ describe("WorkerPipeline", () => {
});
});
it("resolves incident when monitor recovers from breached", async () => {
it("resolves incident with threshold_resolved when monitor recovers from breached", async () => {
const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" }));
expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
expect(decision).toMatchObject({
shouldResolveIncident: true,
shouldSendNotification: true,
shouldCreateIncident: false,
notificationReason: "threshold_resolved",
});
});
it("does not create or resolve for unhandled status transitions", async () => {
const decision = await decide(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false });
});
it("carries the threshold breaches onto a breached decision", async () => {
const thresholdBreaches = { cpu: true, memory: false, disk: true, temp: false };
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up", thresholdBreaches }));
expect(decision.thresholdBreaches).toEqual(thresholdBreaches);
});
it("leaves threshold breaches off a down decision even when the status change carries them", async () => {
const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false };
const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", thresholdBreaches }));
expect(decision).not.toHaveProperty("thresholdBreaches");
});
it("leaves threshold breaches off a no-op decision", async () => {
const thresholdBreaches = { cpu: true, memory: false, disk: false, temp: false };
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: false, prevStatus: "breached", thresholdBreaches }));
expect(decision).not.toHaveProperty("thresholdBreaches");
});
// StatusService only sets thresholdBreaches for hardware monitors; a docker breach must not leave the key behind holding undefined.
it("leaves threshold breaches off a breached decision when the status change carries none", async () => {
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" }));
expect(decision).toMatchObject({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
expect(decision).not.toHaveProperty("thresholdBreaches");
});
// Each decision starts from a shared template. A transition must not leak its flags into the next decision
// the same pipeline makes, whether that is a no-op evaluation or a degraded-egress short-circuit.
it("starts every decision clean after a transition on the same pipeline", async () => {
const updateMonitorStatus = jest
.fn<any>()
.mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up" }))
.mockResolvedValueOnce(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "down" }));
const { pipeline } = createPipeline({ statusService: { updateMonitorStatus } });
const failing = makeCheck({ status: false, statusCode: 500, message: "Error" });
const first = await pipeline.evaluateCheck(failing, makeMonitor());
const second = await pipeline.evaluateCheck(failing, makeMonitor({ status: "down" }));
const degraded = await pipeline.evaluateCheck(makeCheck({ ...failing, egressStatus: "degraded" }), makeMonitor({ status: "down" }));
expect(first.decision.shouldCreateIncident).toBe(true);
const clean = {
shouldCreateIncident: false,
shouldResolveIncident: false,
shouldSendNotification: false,
incidentReason: null,
notificationReason: null,
};
expect(second.decision).toEqual(clean);
expect(degraded.decision).toEqual(clean);
});
});
});
+104
View File
@@ -0,0 +1,104 @@
import { describe, expect, it } from "@jest/globals";
import { describeContainerBreach, findContainerBreaches, isContainerStopped, isContainerUnhealthy } from "../../../src/domain/docker/docker-alert.ts";
import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
const makeContainer = (overrides?: Partial<DockerContainerInfo>): DockerContainerInfo => ({
id: "abc123",
name: "web",
image: "nginx:latest",
state: "running",
status: "Up 3 hours",
health: "none",
...overrides,
});
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "mon-1",
type: "docker",
dockerAlertOnStopped: false,
dockerAlertOnUnhealthy: false,
...overrides,
}) as Monitor;
describe("isContainerStopped", () => {
it("treats dead as stopped", () => {
expect(isContainerStopped(makeContainer({ state: "dead" }))).toBe(true);
});
it.each([1, 137, 143])("treats exited with code %s as stopped", (exitCode) => {
expect(isContainerStopped(makeContainer({ state: "exited", exitCode }))).toBe(true);
});
it("does not treat exited with code 0 as stopped", () => {
expect(isContainerStopped(makeContainer({ state: "exited", exitCode: 0 }))).toBe(false);
});
it("treats exited with an unknown exit code as stopped", () => {
expect(isContainerStopped(makeContainer({ state: "exited" }))).toBe(true);
});
it.each(["running", "restarting", "paused", "created", "removing"] as const)("does not treat %s as stopped", (state) => {
expect(isContainerStopped(makeContainer({ state, exitCode: 1 }))).toBe(false);
});
});
describe("isContainerUnhealthy", () => {
it("treats unhealthy as unhealthy", () => {
expect(isContainerUnhealthy(makeContainer({ health: "unhealthy" }))).toBe(true);
});
it.each(["healthy", "starting", "none"] as const)("does not treat %s as unhealthy", (health) => {
expect(isContainerUnhealthy(makeContainer({ health }))).toBe(false);
});
});
describe("findContainerBreaches", () => {
const stopped = makeContainer({ name: "db", state: "exited", exitCode: 1 });
const unhealthy = makeContainer({ name: "api", health: "unhealthy" });
it("returns nothing when both switches are off", () => {
expect(findContainerBreaches(makeMonitor(), [stopped, unhealthy])).toEqual([]);
});
it("reports only stopped containers when only the stopped switch is on", () => {
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true }), [stopped, unhealthy]);
expect(breaches).toEqual([{ name: "db", reason: "stopped", exitCode: 1 }]);
});
it("reports only unhealthy containers when only the unhealthy switch is on", () => {
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnUnhealthy: true }), [stopped, unhealthy]);
expect(breaches).toEqual([{ name: "api", reason: "unhealthy" }]);
});
it("reports both conditions when both switches are on", () => {
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [stopped, unhealthy]);
expect(breaches.map((b) => b.reason)).toEqual(["stopped", "unhealthy"]);
});
it("reports a container that is both stopped and unhealthy once, as stopped", () => {
const both = makeContainer({ name: "worker", state: "dead", health: "unhealthy" });
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [both]);
expect(breaches).toEqual([{ name: "worker", reason: "stopped", exitCode: undefined }]);
});
it("ignores healthy running containers", () => {
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [makeContainer()]);
expect(breaches).toEqual([]);
});
});
describe("describeContainerBreach", () => {
it("describes an unhealthy container", () => {
expect(describeContainerBreach({ name: "api", reason: "unhealthy" })).toBe("api: unhealthy");
});
it("describes a stopped container with its exit code", () => {
expect(describeContainerBreach({ name: "db", reason: "stopped", exitCode: 137 })).toBe("db: stopped (exit code 137)");
});
it("describes a stopped container without an exit code", () => {
expect(describeContainerBreach({ name: "db", reason: "stopped" })).toBe("db: stopped");
});
});
@@ -1,68 +0,0 @@
import { describe, expect, it } from "@jest/globals";
import type { SSLDetails, SSLOptions } from "ssl-checker";
import { fetchMonitorCertificate } from "../../../src/api/controllers/controllerUtils.ts";
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
const makeMonitor = (url: string, port?: number): Monitor =>
({
url,
port,
type: "http",
}) as unknown as Monitor;
const cert: SSLDetails = {
daysRemaining: 30,
valid: true,
validFrom: "2026-01-01T00:00:00Z",
validTo: "2026-12-31T00:00:00Z",
validFor: ["example.com"],
};
const setup = () => {
const calls: { hostname: string; options?: SSLOptions }[] = [];
const checker = async (hostname: string, options?: SSLOptions) => {
calls.push({ hostname, options });
return cert;
};
return { calls, checker };
};
describe("fetchMonitorCertificate", () => {
it("uses the default port when none is specified", async () => {
const { calls, checker } = setup();
await fetchMonitorCertificate(checker, makeMonitor("https://example.com/health"));
expect(calls).toEqual([{ hostname: "example.com", options: undefined }]);
});
it("uses the port from the URL", async () => {
const { calls, checker } = setup();
await fetchMonitorCertificate(checker, makeMonitor("https://example.com:54321/health"));
expect(calls).toEqual([{ hostname: "example.com", options: { port: 54321 } }]);
});
it("falls back to monitor.port when the URL has no port", async () => {
const { calls, checker } = setup();
await fetchMonitorCertificate(checker, makeMonitor("https://example.com/health", 8443));
expect(calls).toEqual([{ hostname: "example.com", options: { port: 8443 } }]);
});
it("prefers the URL port over monitor.port", async () => {
const { calls, checker } = setup();
await fetchMonitorCertificate(checker, makeMonitor("https://www.myurl.com:52345/health", 443));
expect(calls).toEqual([{ hostname: "www.myurl.com", options: { port: 52345 } }]);
});
it("throws when the certificate has no expiry", async () => {
const checker = async () => ({ ...cert, validTo: undefined }) as unknown as SSLDetails;
await expect(fetchMonitorCertificate(checker, makeMonitor("https://example.com"))).rejects.toThrow("Certificate not found");
});
});
@@ -247,6 +247,20 @@ describe("monitorValidation — strategy gating", () => {
expect(parsed.monitors[0].dockerLogsEnabled).toBe(false);
});
it("defaults the docker alert switches to false on imported docker monitors", () => {
const parsed = importMonitorsBodyValidation.parse({
monitors: [
{
name: "Imported Docker",
type: "docker",
url: "unix:///var/run/docker.sock",
},
],
});
expect(parsed.monitors[0].dockerAlertOnStopped).toBe(false);
expect(parsed.monitors[0].dockerAlertOnUnhealthy).toBe(false);
});
it("rejects strategy on imported non-pagespeed monitors", () => {
expect(() =>
importMonitorsBodyValidation.parse({
@@ -818,3 +832,43 @@ describe("monitorValidation — Docker TLS credentials", () => {
});
});
});
describe("monitorValidation — docker alert switches", () => {
describe("createMonitorBodyValidation", () => {
it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on a docker monitor", () => {
const parsed = createMonitorBodyValidation.parse({
name: "Docker host",
type: "docker",
url: "unix:///var/run/docker.sock",
dockerAlertOnStopped: true,
dockerAlertOnUnhealthy: true,
});
expect(parsed.dockerAlertOnStopped).toBe(true);
expect(parsed.dockerAlertOnUnhealthy).toBe(true);
});
it("leaves the switches undefined when omitted", () => {
const parsed = createMonitorBodyValidation.parse({
name: "Docker host",
type: "docker",
url: "unix:///var/run/docker.sock",
});
expect(parsed.dockerAlertOnStopped).toBeUndefined();
expect(parsed.dockerAlertOnUnhealthy).toBeUndefined();
});
});
describe("editMonitorBodyValidation", () => {
it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on edits", () => {
const parsed = editMonitorBodyValidation.parse({
dockerAlertOnStopped: true,
dockerAlertOnUnhealthy: false,
});
expect(parsed.dockerAlertOnStopped).toBe(true);
expect(parsed.dockerAlertOnUnhealthy).toBe(false);
});
});
});