Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Malena Aguiar
|
10e127d0db |
No files matched your search
@@ -85,8 +85,6 @@ export const getMonitorDefaults = (
|
||||
dockerTlsKey: "",
|
||||
dockerTlsKeySet: data?.dockerTlsKeySet ?? false,
|
||||
dockerLogsEnabled: data?.dockerLogsEnabled ?? false,
|
||||
dockerAlertOnStopped: data?.dockerAlertOnStopped ?? false,
|
||||
dockerAlertOnUnhealthy: data?.dockerAlertOnUnhealthy ?? false,
|
||||
};
|
||||
break;
|
||||
case "game":
|
||||
|
||||
@@ -617,6 +617,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(0) && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.general.title")}
|
||||
@@ -757,6 +758,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(0) && watchedType === "docker" && isDockerTlsUrl(watchedUrl) && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.dockerTls.title")}
|
||||
@@ -814,6 +816,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(1) && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.frequency.title")}
|
||||
@@ -834,6 +837,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Alert Thresholds - only for hardware type */}
|
||||
{watchedType === "hardware" && showStep(1) && (
|
||||
<ConfigBox
|
||||
@@ -889,6 +893,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(1) && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.incidents.title")}
|
||||
@@ -915,6 +920,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(1) && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.notifications.title")}
|
||||
@@ -927,6 +933,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(1) && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.tags.title")}
|
||||
@@ -953,6 +960,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(1) && watchedType === "docker" && !isCaptureDocker && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.dockerLogs.title")}
|
||||
@@ -965,28 +973,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{showStep(1) && watchedType === "docker" && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.dockerAlerts.title")}
|
||||
subtitle={t("pages.createMonitor.form.dockerAlerts.description")}
|
||||
rightContent={
|
||||
<Stack gap={theme.spacing(LAYOUT.MD)}>
|
||||
<FormSwitchField
|
||||
name="dockerAlertOnStopped"
|
||||
label={t(
|
||||
"pages.createMonitor.form.dockerAlerts.option.onStopped.label"
|
||||
)}
|
||||
/>
|
||||
<FormSwitchField
|
||||
name="dockerAlertOnUnhealthy"
|
||||
label={t(
|
||||
"pages.createMonitor.form.dockerAlerts.option.onUnhealthy.label"
|
||||
)}
|
||||
/>
|
||||
</Stack>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(2) &&
|
||||
(watchedType === "http" ||
|
||||
watchedType === "grpc" ||
|
||||
@@ -1002,6 +989,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(2) && watchedType === "http" && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.advanced.title")}
|
||||
@@ -1101,6 +1089,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(2) && supportsGeoCheck(watchedType) && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.geoChecks.title")}
|
||||
@@ -1133,6 +1122,7 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
<Stack
|
||||
direction="row"
|
||||
justifyContent={!isEditMode ? "space-between" : "flex-end"}
|
||||
|
||||
@@ -19,7 +19,6 @@ const DockerMonitorsPage = () => {
|
||||
controller={c}
|
||||
bulkActions
|
||||
showTypeFilter={false}
|
||||
summaryProps={{ showBreached: true }}
|
||||
>
|
||||
<DockerMonitorsTable
|
||||
monitors={c.monitors || []}
|
||||
|
||||
@@ -169,8 +169,6 @@ export interface Monitor {
|
||||
geoCheckLocations?: GeoContinent[];
|
||||
geoCheckInterval?: number;
|
||||
dockerLogsEnabled?: boolean;
|
||||
dockerAlertOnStopped?: boolean;
|
||||
dockerAlertOnUnhealthy?: boolean;
|
||||
dockerTlsCa?: string;
|
||||
dockerTlsCert?: string;
|
||||
dockerTlsKeySet?: boolean;
|
||||
|
||||
@@ -20,7 +20,7 @@ export const getMonitorPath = (type: MonitorType): string => {
|
||||
websocket: "uptime",
|
||||
dns: "uptime",
|
||||
unknown: "uptime",
|
||||
docker: "docker/host",
|
||||
docker: "docker",
|
||||
hardware: "infrastructure",
|
||||
pagespeed: "pagespeed",
|
||||
};
|
||||
|
||||
@@ -152,8 +152,6 @@ const dockerSchema = baseSchema.extend({
|
||||
// Read only mirror of the server flag, stripped on submit
|
||||
dockerTlsKeySet: z.boolean(),
|
||||
dockerLogsEnabled: z.boolean().register(monitorStepRegistry, { step: 1 }),
|
||||
dockerAlertOnStopped: z.boolean().register(monitorStepRegistry, { step: 1 }),
|
||||
dockerAlertOnUnhealthy: z.boolean().register(monitorStepRegistry, { step: 1 }),
|
||||
});
|
||||
|
||||
// Game server monitor schema
|
||||
|
||||
@@ -557,7 +557,7 @@
|
||||
},
|
||||
"status": {
|
||||
"down": "down",
|
||||
"breached": "alert",
|
||||
"breached": "threshold exceeded",
|
||||
"initializing": "initializing",
|
||||
"maintenance": "maintenance",
|
||||
"paused": "paused",
|
||||
@@ -827,18 +827,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"dockerAlerts": {
|
||||
"title": "Container alerts",
|
||||
"description": "Mark this host as breached while any container on it is stopped or failing its health check, and notify on the change. Containers that exit with code 0 do not count as stopped. Capture does not report exit codes, so on Capture hosts every stopped container counts.",
|
||||
"option": {
|
||||
"onStopped": {
|
||||
"label": "Alert when a container is stopped"
|
||||
},
|
||||
"onUnhealthy": {
|
||||
"label": "Alert when a container is unhealthy"
|
||||
}
|
||||
}
|
||||
},
|
||||
"stats": {
|
||||
"title": "Monitor history",
|
||||
"description": "Clear all monitoring history and statistics for your team. This action is irreversible.",
|
||||
@@ -1697,7 +1685,7 @@
|
||||
"status": {
|
||||
"up": "Operational",
|
||||
"down": "Down",
|
||||
"breached": "Alert",
|
||||
"breached": "Threshold exceeded",
|
||||
"maintenance": "Maintenance",
|
||||
"paused": "Paused",
|
||||
"initializing": "Initializing"
|
||||
@@ -1737,13 +1725,13 @@
|
||||
"monitoringSummary": "Monitoring {{count}} service · Updates every 10 seconds",
|
||||
"monitoringSummary_other": "Monitoring {{count}} services · Updates every 10 seconds",
|
||||
"allUp": "All systems operational",
|
||||
"allBreached": "All systems have active alerts",
|
||||
"allBreached": "All systems have exceeded thresholds",
|
||||
"allMaintenance": "All systems are under maintenance",
|
||||
"allDown": "All systems are down",
|
||||
"allPaused": "All systems are paused",
|
||||
"allInitializing": "All systems are initializing",
|
||||
"breachedAndDown": "Some systems have active alerts and some are down",
|
||||
"breached": "Some systems have active alerts",
|
||||
"breachedAndDown": "Some systems have exceeded thresholds and are down",
|
||||
"breached": "Some systems have exceeded thresholds",
|
||||
"degraded": "Some systems are experiencing issues",
|
||||
"maintenanceAndDown": "Some systems are down during maintenance",
|
||||
"maintenance": "Some systems are under maintenance",
|
||||
|
||||
@@ -466,12 +466,6 @@
|
||||
"dockerLogsEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnStopped": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnUnhealthy": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerTlsCa": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -529,8 +523,6 @@
|
||||
"geoCheckLocations",
|
||||
"geoCheckInterval",
|
||||
"dockerLogsEnabled",
|
||||
"dockerAlertOnStopped",
|
||||
"dockerAlertOnUnhealthy",
|
||||
"teamId",
|
||||
"userId",
|
||||
"createdAt",
|
||||
@@ -702,12 +694,6 @@
|
||||
"dockerLogsEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnStopped": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnUnhealthy": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerTlsCa": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -767,8 +753,6 @@
|
||||
"geoCheckLocations",
|
||||
"geoCheckInterval",
|
||||
"dockerLogsEnabled",
|
||||
"dockerAlertOnStopped",
|
||||
"dockerAlertOnUnhealthy",
|
||||
"teamId",
|
||||
"userId",
|
||||
"createdAt",
|
||||
@@ -7202,12 +7186,6 @@
|
||||
"dockerLogsEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnStopped": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnUnhealthy": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerTlsCa": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -7267,8 +7245,6 @@
|
||||
"geoCheckLocations",
|
||||
"geoCheckInterval",
|
||||
"dockerLogsEnabled",
|
||||
"dockerAlertOnStopped",
|
||||
"dockerAlertOnUnhealthy",
|
||||
"teamId",
|
||||
"userId",
|
||||
"createdAt",
|
||||
@@ -7785,12 +7761,6 @@
|
||||
"dockerLogsEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnStopped": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnUnhealthy": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerTlsCa": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -7850,8 +7820,6 @@
|
||||
"geoCheckLocations",
|
||||
"geoCheckInterval",
|
||||
"dockerLogsEnabled",
|
||||
"dockerAlertOnStopped",
|
||||
"dockerAlertOnUnhealthy",
|
||||
"teamId",
|
||||
"userId",
|
||||
"createdAt",
|
||||
@@ -9340,12 +9308,6 @@
|
||||
"dockerLogsEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnStopped": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnUnhealthy": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerTlsCa": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -10071,14 +10033,6 @@
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
},
|
||||
"dockerAlertOnStopped": {
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
},
|
||||
"dockerAlertOnUnhealthy": {
|
||||
"type": "boolean",
|
||||
"default": false
|
||||
},
|
||||
"dnsServer": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -10646,12 +10600,6 @@
|
||||
"dockerLogsEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnStopped": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerAlertOnUnhealthy": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dockerTlsCa": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
||||
@@ -10,7 +10,10 @@ type WhoisModule = typeof whoiser;
|
||||
export const fetchMonitorCertificate = async (checker: SSLCheckerType, monitor: Monitor): Promise<SSLDetails> => {
|
||||
const monitorUrl = new URL(monitor.url);
|
||||
const hostname = monitorUrl.hostname;
|
||||
const cert = await checker(hostname);
|
||||
// An explicit port in the URL is the most visible to the user, so it takes precedence over monitor.port.
|
||||
const urlPort = monitorUrl.port ? Number(monitorUrl.port) : undefined;
|
||||
const port = urlPort ?? monitor.port;
|
||||
const cert = await checker(hostname, port !== undefined ? { port } : undefined);
|
||||
if (cert?.validTo === null || cert?.validTo === undefined) {
|
||||
throw new Error("Certificate not found");
|
||||
}
|
||||
|
||||
@@ -35,18 +35,11 @@ class NotificationController implements INotificationController {
|
||||
|
||||
testNotification = catchAsync(async (req: Request, res: Response) => {
|
||||
const notification = testNotificationBodyValidation.parse(req.body);
|
||||
let success = false;
|
||||
let reason: string | undefined;
|
||||
try {
|
||||
success = await this.notificationsService.sendTestNotification(notification);
|
||||
} catch (error: unknown) {
|
||||
if (!(error instanceof AppError)) throw error;
|
||||
reason = error.message;
|
||||
}
|
||||
const success = await this.notificationsService.sendTestNotification(notification);
|
||||
|
||||
return res.status(200).json({
|
||||
success,
|
||||
msg: success ? "Notification sent successfully" : (reason ?? "Notification could not be sent — check the destination details."),
|
||||
msg: success ? "Notification sent successfully" : "Notification could not be sent — check the destination details.",
|
||||
details: { service: SERVICE_NAME },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,8 +9,6 @@ import { IProxiesService } from "@/domain/proxies/proxy.service.js";
|
||||
import { IEgressStateService } from "@/domain/egress/egress-state.service.js";
|
||||
import { Settings } from "@/domain/app-settings/app-settings.type.js";
|
||||
|
||||
const SERVICE_NAME = "SettingsController";
|
||||
|
||||
export interface ISettingsController {
|
||||
getAppSettings: RequestHandler;
|
||||
updateAppSettings: RequestHandler;
|
||||
@@ -121,7 +119,7 @@ class SettingsController implements ISettingsController {
|
||||
|
||||
const html = await this.emailService.buildEmail("testEmailTemplate", context);
|
||||
if (!html) {
|
||||
throw new AppError({ message: "Failed to build email template.", status: 500, service: SERVICE_NAME, method: "sendTestEmail" });
|
||||
throw new AppError({ message: "Failed to build email template.", status: 500 });
|
||||
}
|
||||
let messageId: string;
|
||||
try {
|
||||
@@ -144,11 +142,8 @@ class SettingsController implements ISettingsController {
|
||||
// Surface the underlying SMTP failure: diagnosing the settings is the whole
|
||||
// point of the test email endpoint.
|
||||
throw new AppError({
|
||||
message: error instanceof AppError ? error.message : "Failed to send test email.",
|
||||
status: error instanceof AppError ? error.status : 500,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendTestEmail",
|
||||
details: error instanceof AppError ? error.details : undefined,
|
||||
message: error instanceof Error ? `Failed to send test email. ${error.message}` : "Failed to send test email.",
|
||||
status: 500,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -220,8 +220,6 @@ export const createMonitorBodyValidation = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
|
||||
geoCheckInterval: z.number().min(300000).optional(),
|
||||
dockerLogsEnabled: z.boolean().optional(),
|
||||
dockerAlertOnStopped: z.boolean().optional(),
|
||||
dockerAlertOnUnhealthy: z.boolean().optional(),
|
||||
dockerTlsCa: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsCert: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsKey: z.union([z.string(), z.literal("")]).optional(),
|
||||
@@ -272,8 +270,6 @@ export const editMonitorBodyValidation = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
|
||||
geoCheckInterval: z.number().min(300000).optional(),
|
||||
dockerLogsEnabled: z.boolean().optional(),
|
||||
dockerAlertOnStopped: z.boolean().optional(),
|
||||
dockerAlertOnUnhealthy: z.boolean().optional(),
|
||||
dockerTlsCa: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsCert: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsKey: z.union([z.string(), z.literal("")]).optional(),
|
||||
@@ -354,8 +350,6 @@ const importedMonitorSchema = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)).default([]),
|
||||
geoCheckInterval: z.number().min(300000).default(300000),
|
||||
dockerLogsEnabled: z.boolean().default(false),
|
||||
dockerAlertOnStopped: z.boolean().default(false),
|
||||
dockerAlertOnUnhealthy: z.boolean().default(false),
|
||||
dnsServer: dnsServerValidation.optional(),
|
||||
dnsRecordType: z.enum(DnsRecordTypes).optional(),
|
||||
createdAt: z.string().optional(),
|
||||
@@ -446,8 +440,6 @@ export const monitorResponseSchema = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)),
|
||||
geoCheckInterval: z.number(),
|
||||
dockerLogsEnabled: z.boolean(),
|
||||
dockerAlertOnStopped: z.boolean(),
|
||||
dockerAlertOnUnhealthy: z.boolean(),
|
||||
dockerTlsCa: z.string().optional(),
|
||||
dockerTlsCert: z.string().optional(),
|
||||
dockerTlsKeySet: z.boolean().optional(),
|
||||
|
||||
@@ -220,7 +220,6 @@ const dockerContainerSchema = new Schema<DockerContainerInfo>(
|
||||
startedAt: { type: String },
|
||||
ports: { type: [dockerContainerPortSchema], default: undefined },
|
||||
mounts: { type: [dockerContainerMountSchema], default: undefined },
|
||||
exitCode: { type: Number, default: undefined },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
@@ -135,17 +135,6 @@ export class CheckService implements ICheckService {
|
||||
return check;
|
||||
};
|
||||
|
||||
private toPayload = (check: Check): HardwareStatusPayload | DockerStatusPayload | undefined => {
|
||||
switch (check.metadata.type) {
|
||||
case "hardware":
|
||||
return { data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload;
|
||||
case "docker":
|
||||
return check.containers ? { containers: check.containers, summary: check.containerSummary } : undefined;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
toStatusResponse = (check: Check) => {
|
||||
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
|
||||
monitorId: check.metadata.monitorId,
|
||||
@@ -156,7 +145,10 @@ export class CheckService implements ICheckService {
|
||||
message: check.message,
|
||||
responseTime: check.responseTime,
|
||||
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
|
||||
payload: this.toPayload(check),
|
||||
payload:
|
||||
check.metadata.type === "hardware"
|
||||
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
|
||||
: undefined,
|
||||
};
|
||||
return statusResponse;
|
||||
};
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
import type { DockerContainerInfo } from "@/domain/docker/docker.type.js";
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
|
||||
export const DockerAlertReasons = ["stopped", "unhealthy"] as const;
|
||||
export type DockerAlertReason = (typeof DockerAlertReasons)[number];
|
||||
|
||||
export interface DockerContainerBreach {
|
||||
name: string;
|
||||
reason: DockerAlertReason;
|
||||
exitCode?: number;
|
||||
}
|
||||
|
||||
export const isContainerStopped = (container: DockerContainerInfo): boolean => {
|
||||
// Container with exit code 0 is probably a one shot container don't report stopped
|
||||
return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0);
|
||||
};
|
||||
|
||||
export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => {
|
||||
return container.health === "unhealthy";
|
||||
};
|
||||
|
||||
export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => {
|
||||
const breaches: DockerContainerBreach[] = [];
|
||||
for (const container of containers) {
|
||||
if (monitor.dockerAlertOnStopped && isContainerStopped(container)) {
|
||||
breaches.push({
|
||||
name: container.name,
|
||||
reason: "stopped",
|
||||
exitCode: container.exitCode,
|
||||
});
|
||||
} else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) {
|
||||
breaches.push({
|
||||
name: container.name,
|
||||
reason: "unhealthy",
|
||||
});
|
||||
}
|
||||
}
|
||||
return breaches;
|
||||
};
|
||||
|
||||
export const describeContainerBreach = (breach: DockerContainerBreach): string => {
|
||||
if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`;
|
||||
if (breach.exitCode === undefined) {
|
||||
return `${breach.name}: stopped`;
|
||||
}
|
||||
return `${breach.name}: stopped (exit code ${breach.exitCode})`;
|
||||
};
|
||||
@@ -55,7 +55,6 @@ export interface DockerContainerInfo {
|
||||
startedAt?: string; // ISO date
|
||||
ports?: DockerContainerPort[];
|
||||
mounts?: DockerContainerMount[];
|
||||
exitCode?: number;
|
||||
}
|
||||
|
||||
export interface DockerContainerSummary {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
const SERVICE_NAME = "incidentService";
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { getDateForRange } from "@/utils/dataUtils.js";
|
||||
import type { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
|
||||
@@ -12,7 +12,6 @@ import type { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import { DateRange } from "@/types/query.js";
|
||||
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
||||
|
||||
export interface IIncidentService {
|
||||
handleIncident(
|
||||
@@ -107,12 +106,6 @@ export class IncidentService implements IIncidentService {
|
||||
};
|
||||
|
||||
private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string {
|
||||
if (monitor.type === "docker") {
|
||||
const containers = (monitorStatusResponse?.payload as DockerStatusPayload | undefined)?.containers ?? [];
|
||||
const breaches = findContainerBreaches(monitor, containers);
|
||||
return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert";
|
||||
}
|
||||
|
||||
if (!monitorStatusResponse) {
|
||||
return "Threshold breach detected";
|
||||
}
|
||||
|
||||
@@ -307,14 +307,6 @@ const MonitorSchema = new Schema<MonitorDocument>(
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
dockerAlertOnStopped: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
dockerAlertOnUnhealthy: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
dockerTlsCa: {
|
||||
type: String,
|
||||
},
|
||||
|
||||
@@ -468,8 +468,6 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
geoCheckLocations: doc.geoCheckLocations ?? [],
|
||||
geoCheckInterval: doc.geoCheckInterval ?? 300000,
|
||||
dockerLogsEnabled: doc.dockerLogsEnabled ?? false,
|
||||
dockerAlertOnStopped: doc.dockerAlertOnStopped ?? false,
|
||||
dockerAlertOnUnhealthy: doc.dockerAlertOnUnhealthy ?? false,
|
||||
dockerTlsCa: doc.dockerTlsCa ?? undefined,
|
||||
dockerTlsCert: doc.dockerTlsCert ?? undefined,
|
||||
dockerTlsKeySet: doc.dockerTlsKeySet ?? false,
|
||||
|
||||
@@ -51,22 +51,6 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
|
||||
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
|
||||
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
|
||||
|
||||
const MonitorPaths: Record<MonitorType, string> = {
|
||||
http: "uptime",
|
||||
port: "uptime",
|
||||
ping: "uptime",
|
||||
game: "uptime",
|
||||
grpc: "uptime",
|
||||
websocket: "uptime",
|
||||
dns: "uptime",
|
||||
unknown: "uptime",
|
||||
docker: "docker/host",
|
||||
hardware: "infrastructure",
|
||||
pagespeed: "pagespeed",
|
||||
};
|
||||
|
||||
export const getMonitorPath = (type: MonitorType): string => MonitorPaths[type];
|
||||
|
||||
// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
|
||||
// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
|
||||
// however far away the agent is. `unknown` is excluded because no request is made for it.
|
||||
@@ -150,8 +134,6 @@ export interface Monitor {
|
||||
geoCheckLocations?: GeoContinent[];
|
||||
geoCheckInterval?: number;
|
||||
dockerLogsEnabled?: boolean;
|
||||
dockerAlertOnStopped?: boolean;
|
||||
dockerAlertOnUnhealthy?: boolean;
|
||||
dockerTlsCa?: string;
|
||||
dockerTlsCert?: string;
|
||||
// EncryptionService ciphertext. Normally excluded
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
import type {
|
||||
NotificationMessage,
|
||||
@@ -8,7 +8,6 @@ import type {
|
||||
ThresholdBreach,
|
||||
NotificationContent,
|
||||
} from "@/domain/notifications/notification.type.js";
|
||||
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
||||
|
||||
export interface INotificationMessageBuilder {
|
||||
buildMessage(
|
||||
@@ -60,29 +59,32 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
return "monitor_down";
|
||||
}
|
||||
|
||||
const isDocker = monitor.type === "docker";
|
||||
switch (decision.notificationReason) {
|
||||
case "threshold_breach":
|
||||
return isDocker ? "container_breach" : "threshold_breach";
|
||||
case "threshold_resolved":
|
||||
return isDocker ? "container_resolved" : "threshold_resolved";
|
||||
default:
|
||||
return "monitor_up";
|
||||
// Threshold breach (only if not down)
|
||||
if (decision.notificationReason === "threshold_breach") {
|
||||
return "threshold_breach";
|
||||
}
|
||||
|
||||
// Recovery from threshold breach (only for hardware monitors)
|
||||
if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") {
|
||||
return "threshold_resolved";
|
||||
}
|
||||
|
||||
// Standard recovery (up)
|
||||
if (monitor.status === "up") {
|
||||
return "monitor_up";
|
||||
}
|
||||
|
||||
// Default to monitor_up for any other case
|
||||
return "monitor_up";
|
||||
}
|
||||
|
||||
private determineSeverity(type: NotificationType): NotificationSeverity {
|
||||
switch (type) {
|
||||
case "monitor_up":
|
||||
return "success";
|
||||
case "monitor_down":
|
||||
return "critical";
|
||||
case "container_breach":
|
||||
return "warning";
|
||||
case "threshold_breach":
|
||||
return "warning";
|
||||
case "container_resolved":
|
||||
return "success";
|
||||
case "monitor_up":
|
||||
case "threshold_resolved":
|
||||
return "success";
|
||||
case "test":
|
||||
@@ -102,10 +104,6 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<HardwareStatusPayload>);
|
||||
case "threshold_resolved":
|
||||
return this.buildThresholdResolvedContent(monitor);
|
||||
case "container_breach":
|
||||
return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<DockerStatusPayload>);
|
||||
case "container_resolved":
|
||||
return this.buildContainersRecoveredContent(monitor);
|
||||
default:
|
||||
return this.buildDefaultContent(monitor);
|
||||
}
|
||||
@@ -176,23 +174,6 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
};
|
||||
}
|
||||
|
||||
private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<DockerStatusPayload>): NotificationContent {
|
||||
const payload = monitorStatusResponse.payload;
|
||||
const containers = payload && typeof payload !== "string" ? payload.containers : [];
|
||||
const breaches = findContainerBreaches(monitor, containers);
|
||||
const title = `Container Alert: ${monitor.name}`;
|
||||
const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`;
|
||||
const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...breaches.map(describeContainerBreach)];
|
||||
return { title, summary, details, timestamp: new Date() };
|
||||
}
|
||||
|
||||
private buildContainersRecoveredContent(monitor: Monitor): NotificationContent {
|
||||
const title = `Containers Recovered: ${monitor.name}`;
|
||||
const summary = `All containers on "${monitor.name}" are back to normal.`;
|
||||
const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`];
|
||||
return { title, summary, details, timestamp: new Date() };
|
||||
}
|
||||
|
||||
private buildDefaultContent(monitor: Monitor): NotificationContent {
|
||||
return {
|
||||
title: `Monitor: ${monitor.name}`,
|
||||
|
||||
@@ -148,17 +148,7 @@ export class NotificationsService implements INotificationsService {
|
||||
|
||||
testAllNotifications = async (notificationIds: string[]) => {
|
||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||
const tasks = notifications.map((notification) =>
|
||||
this.sendTestNotification(notification).catch((error: unknown) => {
|
||||
this.logger.warn({
|
||||
message: `Test notification failed: ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "testAllNotifications",
|
||||
details: { notificationId: notification.id, type: notification.type },
|
||||
});
|
||||
return false;
|
||||
})
|
||||
);
|
||||
const tasks = notifications.map((notification) => this.sendTestNotification(notification));
|
||||
const outcomes = await Promise.all(tasks);
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
const failed = outcomes.length - succeeded;
|
||||
|
||||
@@ -71,8 +71,7 @@ export interface AlertDiscordPayload {
|
||||
* Part of notification system unification effort
|
||||
*/
|
||||
|
||||
export type NotificationType =
|
||||
"monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test";
|
||||
export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test";
|
||||
|
||||
export type NotificationSeverity = "critical" | "warning" | "info" | "success";
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
const SERVICE_NAME = "EmailProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { buildTestEmail } from "@/domain/notifications/providers/utils.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
@@ -41,13 +40,11 @@ export class EmailProvider extends NotificationProvider {
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: `Email test alert failed: ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
message: "Email test alert failed",
|
||||
service: SERVICE_NAME,
|
||||
method: "sendTestAlert",
|
||||
details: error instanceof AppError ? error.details : undefined,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
if (error instanceof AppError) throw error;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -76,10 +73,9 @@ export class EmailProvider extends NotificationProvider {
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: `Email notification failed: ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
message: "Email notification failed",
|
||||
service: SERVICE_NAME,
|
||||
method: "sendMessage",
|
||||
details: error instanceof AppError ? error.details : undefined,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
return false;
|
||||
@@ -96,10 +92,6 @@ export class EmailProvider extends NotificationProvider {
|
||||
return `Monitor ${message.monitor.name} threshold exceeded`;
|
||||
case "threshold_resolved":
|
||||
return `Monitor ${message.monitor.name} thresholds resolved`;
|
||||
case "container_breach":
|
||||
return `Monitor ${message.monitor.name} container alert`;
|
||||
case "container_resolved":
|
||||
return `Monitor ${message.monitor.name} containers recovered`;
|
||||
default:
|
||||
return `Alert: ${message.monitor.name}`;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import got from "got";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { AlertMatrixPayload, Notification } from "@/domain/notifications/notification.type.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
export class MatrixProvider extends NotificationProvider {
|
||||
@@ -172,10 +172,10 @@ export class MatrixProvider extends NotificationProvider {
|
||||
|
||||
// Incident link (if incident exists)
|
||||
if (message.content.incident) {
|
||||
const url = incidentUrl(message);
|
||||
const incidentUrl = `${message.clientHost}/infrastructure/${message.monitor.id}`;
|
||||
plainLines.push("");
|
||||
plainLines.push(`View Incident: ${url}`);
|
||||
htmlLines.push(`<p><a href="${this.escapeHtml(url)}">View Incident</a></p>`);
|
||||
plainLines.push(`View Incident: ${incidentUrl}`);
|
||||
htmlLines.push(`<p><a href="${this.escapeHtml(incidentUrl)}">View Incident</a></p>`);
|
||||
}
|
||||
|
||||
// Footer with timestamp
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "NtfyProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class NtfyProvider extends NotificationProvider {
|
||||
@@ -142,7 +142,7 @@ export class NtfyProvider extends NotificationProvider {
|
||||
}
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("", incidentUrl(message));
|
||||
lines.push("", `${message.clientHost}/infrastructure/${message.monitor.id}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -75,8 +75,7 @@ export class PagerDutyProvider extends NotificationProvider {
|
||||
|
||||
private buildPagerDutyPayload(notification: Notification, message: NotificationMessage): AlertPagerDutyPayload {
|
||||
// Map our notification type to PagerDuty event_action
|
||||
const eventAction =
|
||||
message.type === "monitor_up" || message.type === "threshold_resolved" || message.type === "container_resolved" ? "resolve" : "trigger";
|
||||
const eventAction = message.type === "monitor_up" || message.type === "threshold_resolved" ? "resolve" : "trigger";
|
||||
|
||||
// Map severity to PagerDuty severity levels
|
||||
const severityMap: Record<string, string> = {
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "PushoverProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class PushoverProvider extends NotificationProvider {
|
||||
@@ -101,7 +101,7 @@ export class PushoverProvider extends NotificationProvider {
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("");
|
||||
lines.push(`View Incident: ${incidentUrl(message)}`);
|
||||
lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const SERVICE_NAME = "RocketChatProvider";
|
||||
import { getMonitorPath, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { supportsUptimeDetails, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import type { Notification, NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
@@ -139,7 +139,19 @@ export class RocketChatProvider extends NotificationProvider {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return `${clientHost}/${getMonitorPath(message.monitor.type as MonitorType)}/${message.monitor.id}`;
|
||||
if (message.monitor.type === "hardware") {
|
||||
return `${clientHost}/infrastructure/${message.monitor.id}`;
|
||||
}
|
||||
|
||||
if (message.monitor.type === "pagespeed") {
|
||||
return `${clientHost}/pagespeed/${message.monitor.id}`;
|
||||
}
|
||||
|
||||
if (supportsUptimeDetails(message.monitor.type as MonitorType)) {
|
||||
return `${clientHost}/uptime/${message.monitor.id}`;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private severityColor(severity: NotificationMessage["severity"]): string {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const SERVICE_NAME = "SignalgridProvider";
|
||||
import type { Notification, NotificationMessage, NotificationSeverity } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class SignalgridProvider extends NotificationProvider {
|
||||
@@ -115,7 +115,7 @@ export class SignalgridProvider extends NotificationProvider {
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("");
|
||||
lines.push(`View Incident: ${incidentUrl(message)}`);
|
||||
lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "SlackProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import got, { HTTPError } from "got";
|
||||
|
||||
export class SlackProvider extends NotificationProvider {
|
||||
@@ -161,7 +161,7 @@ export class SlackProvider extends NotificationProvider {
|
||||
text: "View Incident",
|
||||
emoji: true,
|
||||
},
|
||||
url: incidentUrl(message),
|
||||
url: `${message.clientHost}/infrastructure/${message.monitor.id}`,
|
||||
style: "primary",
|
||||
},
|
||||
],
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "TeamsProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import got, { HTTPError } from "got";
|
||||
|
||||
// Types for Adaptive Card elements
|
||||
@@ -254,7 +254,7 @@ export class TeamsProvider extends NotificationProvider {
|
||||
actions.push({
|
||||
type: "Action.OpenUrl",
|
||||
title: "View Incident",
|
||||
url: incidentUrl(message),
|
||||
url: `${message.clientHost}/incidents/${message.content.incident.id}`,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "TelegramProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class TelegramProvider extends NotificationProvider {
|
||||
@@ -101,7 +101,7 @@ export class TelegramProvider extends NotificationProvider {
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("");
|
||||
lines.push(`<a href="${incidentUrl(message)}">View Incident</a>`);
|
||||
lines.push(`<a href="${message.clientHost}/incidents/${message.monitor.id}">View Incident</a>`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
import { IEmailService } from "@/service/emailService.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
|
||||
export const incidentUrl = (message: NotificationMessage): string => `${message.clientHost}/incidents/${message.monitor.id}`;
|
||||
|
||||
export const buildTestEmail = async (emailService: IEmailService) => {
|
||||
const context = { testName: "Monitoring System" };
|
||||
const html = await emailService.buildEmail("testEmailTemplate", context);
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "WebhookProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class WebhookProvider extends NotificationProvider {
|
||||
@@ -74,7 +74,7 @@ export class WebhookProvider extends NotificationProvider {
|
||||
|
||||
// Incident link
|
||||
if (message.content.incident) {
|
||||
lines.push(`[View Incident](${incidentUrl(message)})`);
|
||||
lines.push(`[View Incident](${message.clientHost}/infrastructure/${message.monitor.id})`);
|
||||
}
|
||||
|
||||
// Return webhook payload with both text and structured data
|
||||
|
||||
@@ -12,23 +12,6 @@ const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const SERVICE_NAME = "EmailService";
|
||||
|
||||
const SMTP_FAILURE_REASONS: Record<string, string> = {
|
||||
EAUTH: "rejected the login. Check the user name and password",
|
||||
ENOAUTH: "requires a login. Set the user name and password",
|
||||
EDNS: "could not be resolved. Check the host name",
|
||||
ECONNECTION: "refused the connection. Check the host and port",
|
||||
ESOCKET: "refused the connection. Check the host and port",
|
||||
ETIMEDOUT: "did not respond. Check the port and the secure setting",
|
||||
ETLS: "failed the TLS handshake. Check the secure and TLS settings",
|
||||
EPROTOCOL: "sent an invalid response. Check the port and the secure setting",
|
||||
};
|
||||
|
||||
const describeSmtpFailure = (error: unknown, host: string, port: number): string => {
|
||||
const code = typeof error === "object" && error !== null && "code" in error ? String((error as { code: unknown }).code) : undefined;
|
||||
const reason = (code !== undefined && SMTP_FAILURE_REASONS[code]) || "failed verification";
|
||||
return `SMTP host ${host}:${port} ${reason}.`;
|
||||
};
|
||||
type MjmlFn = typeof mjml2html;
|
||||
type FileSystem = typeof fs;
|
||||
type PathModule = typeof path;
|
||||
@@ -140,15 +123,6 @@ export class EmailService implements IEmailService {
|
||||
systemEmailRejectUnauthorized,
|
||||
} = config;
|
||||
|
||||
if (!systemEmailHost) {
|
||||
throw new AppError({
|
||||
message: "Email is not configured. Set the system email host in settings.",
|
||||
status: 400,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendEmail",
|
||||
});
|
||||
}
|
||||
|
||||
const emailConfig = {
|
||||
host: systemEmailHost,
|
||||
port: Number(systemEmailPort),
|
||||
@@ -172,20 +146,17 @@ export class EmailService implements IEmailService {
|
||||
try {
|
||||
await this.transporter.verify();
|
||||
} catch (error: unknown) {
|
||||
const cause = error instanceof Error ? error.message : "Unknown error";
|
||||
this.logger.warn({
|
||||
message: `Email transporter verification failed: ${cause}`,
|
||||
message: "Email transporter verification failed",
|
||||
service: SERVICE_NAME,
|
||||
method: "verifyTransporter",
|
||||
details: { host: systemEmailHost, port: emailConfig.port },
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
throw new AppError({
|
||||
message: describeSmtpFailure(error, systemEmailHost, emailConfig.port),
|
||||
status: 502,
|
||||
message: "Email transporter verification failed",
|
||||
service: SERVICE_NAME,
|
||||
method: "sendEmail",
|
||||
details: { cause },
|
||||
details: { cause: error instanceof Error ? error.message : "Unknown error" },
|
||||
});
|
||||
}
|
||||
|
||||
@@ -209,7 +180,6 @@ export class EmailService implements IEmailService {
|
||||
});
|
||||
throw new AppError({
|
||||
message: "Failed to send email",
|
||||
status: 502,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendEmail",
|
||||
details: { cause: error instanceof Error ? error.message : "Unknown error" },
|
||||
|
||||
@@ -42,8 +42,6 @@ const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
|
||||
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
|
||||
const DOCKER_TLS_DEFAULT_PORT = 2376;
|
||||
const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]);
|
||||
const EXITED_STATUS = /^Exited \((\d+)\)/;
|
||||
const HEALTH_STATUS = /\((healthy|unhealthy|health: starting)\)$/;
|
||||
|
||||
export interface DockerError extends Error {
|
||||
statusCode?: number;
|
||||
@@ -142,17 +140,6 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none";
|
||||
}
|
||||
|
||||
private exitCodeFromStatus = (status: string): number | undefined => {
|
||||
const match = EXITED_STATUS.exec(status);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
};
|
||||
|
||||
private healthFromStatus = (status: string): DockerHealthStatus => {
|
||||
const match = HEALTH_STATUS.exec(status);
|
||||
if (!match) return "none";
|
||||
return match[1] === "health: starting" ? "starting" : (match[1] as DockerHealthStatus);
|
||||
};
|
||||
|
||||
private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary {
|
||||
return {
|
||||
total: containers.length,
|
||||
@@ -286,10 +273,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
info.health = this.toHealthStatus(inspectResult.value.State?.Health?.Status);
|
||||
info.ports = this.toPorts(inspectResult.value.NetworkSettings?.Ports);
|
||||
info.mounts = this.toMounts(inspectResult.value.Mounts);
|
||||
info.exitCode = inspectResult.value.State?.ExitCode;
|
||||
} else {
|
||||
info.exitCode = this.exitCodeFromStatus(summary.Status);
|
||||
info.health = this.healthFromStatus(summary.Status);
|
||||
this.logger.warn({
|
||||
message: `Failed to inspect container ${info.name}`,
|
||||
service: SERVICE_NAME,
|
||||
|
||||
@@ -19,7 +19,6 @@ import { ILogger } from "@/utils/logger.js";
|
||||
import type { HardwareStatusMetrics } from "@/types/network.js";
|
||||
import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js";
|
||||
import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js";
|
||||
import { findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
||||
|
||||
const SERVICE_NAME = "StatusService";
|
||||
const HARDWARE_ALERT_COUNTER_START = 5;
|
||||
@@ -163,23 +162,6 @@ export class StatusService implements IStatusService {
|
||||
return { nextStatus, transitioned, breaches, nextCounters };
|
||||
};
|
||||
|
||||
private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => {
|
||||
const { currentStatus, reachabilityDown, breaching } = params;
|
||||
let nextStatus: MonitorStatus = currentStatus;
|
||||
let transitioned = false;
|
||||
|
||||
if (!reachabilityDown) {
|
||||
if (breaching && currentStatus !== "breached") {
|
||||
nextStatus = "breached";
|
||||
transitioned = true;
|
||||
} else if (!breaching && currentStatus === "breached") {
|
||||
nextStatus = "up";
|
||||
transitioned = true;
|
||||
}
|
||||
}
|
||||
return { nextStatus, transitioned };
|
||||
};
|
||||
|
||||
updateMonitorStatus = async (
|
||||
statusResponse: MonitorStatusResponse<
|
||||
| PingStatusPayload
|
||||
@@ -286,20 +268,6 @@ export class StatusService implements IStatusService {
|
||||
}
|
||||
}
|
||||
|
||||
const dockerPayload = statusResponse.payload as DockerStatusPayload | undefined;
|
||||
if (monitor.type === "docker" && dockerPayload?.containers) {
|
||||
const breaches = findContainerBreaches(monitor, dockerPayload.containers);
|
||||
const docker = this.computeDockerStatus({
|
||||
currentStatus: newStatus,
|
||||
reachabilityDown: newStatus === "down",
|
||||
breaching: breaches.length > 0,
|
||||
});
|
||||
if (docker.transitioned) {
|
||||
newStatus = docker.nextStatus;
|
||||
statusChanged = true;
|
||||
}
|
||||
}
|
||||
|
||||
patch.status = newStatus;
|
||||
|
||||
// Single atomic write: push arrays + set status/counters
|
||||
|
||||
+15
-12
@@ -60,18 +60,21 @@ class Logger implements ILogger {
|
||||
if (formattedDetails) msg += ` (details: ${formattedDetails})`;
|
||||
|
||||
if (typeof stack === "string") {
|
||||
const stackTrace = stack.split("\n").map((line: string) => {
|
||||
const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/);
|
||||
if (match) {
|
||||
return {
|
||||
function: match[1],
|
||||
file: match[2],
|
||||
line: parseInt(match[3] ?? "0", 10),
|
||||
column: parseInt(match[4] ?? "0", 10),
|
||||
};
|
||||
}
|
||||
return line.trim();
|
||||
});
|
||||
const stackTrace = stack
|
||||
.split("\n")
|
||||
.slice(1) // Remove first line (error message)
|
||||
.map((line: string) => {
|
||||
const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/);
|
||||
if (match) {
|
||||
return {
|
||||
function: match[1],
|
||||
file: match[2],
|
||||
line: parseInt(match[3] ?? "0", 10),
|
||||
column: parseInt(match[4] ?? "0", 10),
|
||||
};
|
||||
}
|
||||
return line.trim();
|
||||
});
|
||||
msg += ` (stack: ${JSON.stringify(stackTrace, null, 2)})`;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ export interface MonitorActionDecision {
|
||||
shouldResolveIncident: boolean;
|
||||
shouldSendNotification: boolean;
|
||||
incidentReason: "status_down" | "threshold_breach" | null;
|
||||
notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null;
|
||||
notificationReason: "status_change" | "threshold_breach" | null;
|
||||
thresholdBreaches?: {
|
||||
cpu?: boolean;
|
||||
memory?: boolean;
|
||||
|
||||
@@ -326,11 +326,7 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "threshold_breach";
|
||||
decision.notificationReason = "threshold_breach";
|
||||
} else if (monitor.status === "up" && prevStatus === "breached") {
|
||||
decision.shouldResolveIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.notificationReason = "threshold_resolved";
|
||||
} else if (monitor.status === "up" && prevStatus === "down") {
|
||||
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
|
||||
decision.shouldResolveIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.notificationReason = "status_change";
|
||||
|
||||
@@ -229,18 +229,6 @@ describe("DockerProvider", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves the exit code undefined for Capture containers", async () => {
|
||||
const response = makeCaptureResponse({
|
||||
payload: { data: [makeCaptureContainer({ status: "exited", running: false, stats: undefined })], errors: [] },
|
||||
});
|
||||
const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) });
|
||||
|
||||
const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" }));
|
||||
|
||||
expect(result.payload?.containers[0]?.state).toBe("exited");
|
||||
expect(result.payload?.containers[0]?.exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps state-derived Capture health separate from Docker healthchecks", async () => {
|
||||
const response = makeCaptureResponse({
|
||||
payload: {
|
||||
@@ -570,16 +558,6 @@ describe("DockerProvider", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("reads the exit code from inspect", async () => {
|
||||
const inspect = jest.fn().mockResolvedValue(makeInspect({ State: { StartedAt: "2026-08-28T10:00:00.000Z", ExitCode: 137 } }));
|
||||
const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status: "Exited (137) 2 minutes ago" })]);
|
||||
const { provider } = setup({ inspect, listContainers });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ state: "exited", exitCode: 137, health: "none" }));
|
||||
});
|
||||
|
||||
it("counts running, stopped, and unhealthy states in the summary", async () => {
|
||||
const containers = [
|
||||
makeContainer({ Id: "a".repeat(64) }),
|
||||
@@ -704,40 +682,11 @@ describe("DockerProvider", () => {
|
||||
|
||||
expect(result.status).toBe(true);
|
||||
expect(result.payload?.containers).toHaveLength(1);
|
||||
expect(result.payload?.containers[0]).toEqual(expect.objectContaining({ health: "none" }));
|
||||
expect(result.payload?.containers[0]?.restartCount).toBeUndefined();
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Failed to inspect") }));
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Up 3 hours (healthy)", "healthy"],
|
||||
["Up 3 hours (unhealthy)", "unhealthy"],
|
||||
["Up 5 seconds (health: starting)", "starting"],
|
||||
["Up 3 hours", "none"],
|
||||
["Exited (1) 2 minutes ago", "none"],
|
||||
])("reads health from the list status %s when inspect rejects", async (Status, health) => {
|
||||
const inspect = jest.fn().mockRejectedValue(new Error("inspect failed"));
|
||||
const listContainers = jest.fn().mockResolvedValue([makeContainer({ Status })]);
|
||||
const { provider } = setup({ inspect, listContainers });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.payload?.containers[0]?.health).toBe(health);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Exited (0) 2 minutes ago", 0],
|
||||
["Exited (137) 2 minutes ago", 137],
|
||||
["Up 3 hours", undefined],
|
||||
])("reads the exit code from the list status %s when inspect rejects", async (Status, exitCode) => {
|
||||
const inspect = jest.fn().mockRejectedValue(new Error("inspect failed"));
|
||||
const listContainers = jest.fn().mockResolvedValue([makeContainer({ State: "exited", Status })]);
|
||||
const { provider } = setup({ inspect, listContainers });
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
expect(result.payload?.containers[0]?.exitCode).toBe(exitCode);
|
||||
});
|
||||
|
||||
it("keeps the container without metrics when stats rejects", async () => {
|
||||
const stats = jest.fn().mockRejectedValue(new Error("stats failed"));
|
||||
const { provider } = setup({ stats });
|
||||
|
||||
@@ -4,7 +4,6 @@ import { createMockLogger } from "../../../helpers/createMockLogger.ts";
|
||||
import { makeNotification, makeMessage } from "../../../helpers/notificationMessage.ts";
|
||||
import { testNotificationProviderContract } from "../../../helpers/notificationProviderContract.ts";
|
||||
import type { IEmailService } from "../../../../src/service/emailService.ts";
|
||||
import { AppError } from "../../../../src/utils/AppError.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -51,29 +50,11 @@ describe("EmailProvider", () => {
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build test email content" }));
|
||||
});
|
||||
|
||||
it("returns false and logs the cause when sendEmail rejects with a plain error", async () => {
|
||||
it("returns false when sendEmail rejects", async () => {
|
||||
const { provider, emailService, logger } = createProvider();
|
||||
(emailService.sendEmail as jest.Mock).mockRejectedValue(new Error("SMTP auth failed"));
|
||||
expect(await provider.sendTestAlert(makeNotification())).toBe(false);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email test alert failed: SMTP auth failed" }));
|
||||
});
|
||||
|
||||
it("rethrows an AppError from sendEmail so the caller can report the reason", async () => {
|
||||
const { provider, emailService, logger } = createProvider();
|
||||
const error = new AppError({
|
||||
message: "Email is not configured. Set the system email host in settings.",
|
||||
status: 400,
|
||||
details: { cause: "no host" },
|
||||
});
|
||||
(emailService.sendEmail as jest.Mock).mockRejectedValue(error);
|
||||
|
||||
await expect(provider.sendTestAlert(makeNotification())).rejects.toBe(error);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: "Email test alert failed: Email is not configured. Set the system email host in settings.",
|
||||
details: { cause: "no host" },
|
||||
})
|
||||
);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email test alert failed" }));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -103,29 +84,11 @@ describe("EmailProvider", () => {
|
||||
|
||||
// NotificationsService fans providers out through Promise.all, so a failed email must
|
||||
// resolve to false rather than reject and take the other channels down with it.
|
||||
it("returns false and logs the cause when sendEmail rejects", async () => {
|
||||
it("returns false when sendEmail rejects", async () => {
|
||||
const { provider, emailService, logger } = createProvider();
|
||||
(emailService.sendEmail as jest.Mock).mockRejectedValue(new Error("SMTP auth failed"));
|
||||
expect(await provider.sendMessage(makeNotification() as any, makeMessage())).toBe(false);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email notification failed: SMTP auth failed" }));
|
||||
});
|
||||
|
||||
it("returns false rather than throwing when sendEmail rejects with an AppError", async () => {
|
||||
const { provider, emailService, logger } = createProvider();
|
||||
(emailService.sendEmail as jest.Mock).mockRejectedValue(
|
||||
new AppError({
|
||||
message: "SMTP verification failed for smtp.example.com:587: 535 Username and Password not accepted",
|
||||
status: 502,
|
||||
details: { cause: "ECONNREFUSED" },
|
||||
})
|
||||
);
|
||||
expect(await provider.sendMessage(makeNotification() as any, makeMessage())).toBe(false);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: "Email notification failed: SMTP verification failed for smtp.example.com:587: 535 Username and Password not accepted",
|
||||
details: { cause: "ECONNREFUSED" },
|
||||
})
|
||||
);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Email notification failed" }));
|
||||
});
|
||||
|
||||
it("builds correct subject for monitor_down", async () => {
|
||||
@@ -152,18 +115,6 @@ describe("EmailProvider", () => {
|
||||
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor thresholds resolved", expect.anything());
|
||||
});
|
||||
|
||||
it("builds correct subject for container_breach", async () => {
|
||||
const { provider, emailService } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" }));
|
||||
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor container alert", expect.anything());
|
||||
});
|
||||
|
||||
it("builds correct subject for container_resolved", async () => {
|
||||
const { provider, emailService } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" }));
|
||||
expect(emailService.sendEmail).toHaveBeenCalledWith(expect.anything(), "Monitor Test Monitor containers recovered", expect.anything());
|
||||
});
|
||||
|
||||
it("builds default subject for unknown type", async () => {
|
||||
const { provider, emailService } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "test" }));
|
||||
|
||||
@@ -105,14 +105,6 @@ describe("MatrixProvider", () => {
|
||||
expect(body.formatted_body).toContain("View Incident");
|
||||
});
|
||||
|
||||
it("links the incident to the monitor's incidents page", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
|
||||
const body = mockGotPut.mock.calls[0][1].json;
|
||||
expect(body.body).toContain("View Incident: https://app.example.com/incidents/mon-1");
|
||||
expect(body.formatted_body).toContain('href="https://app.example.com/incidents/mon-1"');
|
||||
});
|
||||
|
||||
it("omits optional sections when not present", async () => {
|
||||
const { provider } = createProvider();
|
||||
const msg = makeMessage();
|
||||
|
||||
@@ -106,7 +106,7 @@ describe("NtfyProvider", () => {
|
||||
it("includes incident links in text", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNtfyNotification() as any, makeMessageWithIncident());
|
||||
expect(mockGotPost.mock.calls[0][1].body).toContain("https://app.example.com/incidents/mon-1");
|
||||
expect(mockGotPost.mock.calls[0][1].body).toContain("/infrastructure/mon-1");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -89,18 +89,6 @@ describe("PagerDutyProvider", () => {
|
||||
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve");
|
||||
});
|
||||
|
||||
it("uses 'resolve' event_action for container_resolved", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_resolved" }));
|
||||
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("resolve");
|
||||
});
|
||||
|
||||
it("uses 'trigger' event_action for container_breach", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessage({ type: "container_breach" }));
|
||||
expect(mockGotPost.mock.calls[0][1].json.event_action).toBe("trigger");
|
||||
});
|
||||
|
||||
it("includes threshold info in summary and custom_details", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithThresholds());
|
||||
|
||||
@@ -99,12 +99,6 @@ describe("PushoverProvider", () => {
|
||||
expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident");
|
||||
});
|
||||
|
||||
it("links the incident to the monitor's incidents page", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
|
||||
expect(mockGotPost.mock.calls[0][1].form.message).toContain("View Incident: https://app.example.com/incidents/mon-1");
|
||||
});
|
||||
|
||||
it("omits optional sections when not present", async () => {
|
||||
const { provider } = createProvider();
|
||||
const msg = makeMessage();
|
||||
|
||||
@@ -134,7 +134,6 @@ describe("RocketChatProvider", () => {
|
||||
["ping", "https://app.example.com/uptime/mon-1"],
|
||||
["pagespeed", "https://app.example.com/pagespeed/mon-1"],
|
||||
["hardware", "https://app.example.com/infrastructure/mon-1"],
|
||||
["docker", "https://app.example.com/docker/host/mon-1"],
|
||||
])("links %s monitors to %s", async (type, expectedLink) => {
|
||||
const { provider } = createProvider();
|
||||
|
||||
|
||||
@@ -139,12 +139,6 @@ describe("SignalgridProvider", () => {
|
||||
expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident");
|
||||
});
|
||||
|
||||
it("links the incident to the monitor's incidents page", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
|
||||
expect(mockGotPost.mock.calls[0][1].form.body).toContain("View Incident: https://app.example.com/incidents/mon-1");
|
||||
});
|
||||
|
||||
it("omits optional sections when not present", async () => {
|
||||
const { provider } = createProvider();
|
||||
const msg = makeMessage();
|
||||
|
||||
@@ -96,14 +96,6 @@ describe("SlackProvider", () => {
|
||||
expect(text).toContain("Additional Information");
|
||||
});
|
||||
|
||||
it("links the incident button to the monitor's incidents page", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
|
||||
const blocks = mockGotPost.mock.calls[0][1].json.blocks;
|
||||
const actions = blocks.find((b: any) => b.type === "actions");
|
||||
expect(actions.elements[0].url).toBe("https://app.example.com/incidents/mon-1");
|
||||
});
|
||||
|
||||
it("omits threshold, details, and incident sections when not present", async () => {
|
||||
const { provider } = createProvider();
|
||||
const msg = makeMessage();
|
||||
|
||||
@@ -154,7 +154,7 @@ describe("TeamsProvider", () => {
|
||||
expect(card.actions).toHaveLength(1);
|
||||
expect(card.actions[0].type).toBe("Action.OpenUrl");
|
||||
expect(card.actions[0].title).toBe("View Incident");
|
||||
expect(card.actions[0].url).toBe("https://app.example.com/incidents/mon-1");
|
||||
expect(card.actions[0].url).toContain("/incidents/inc-1");
|
||||
});
|
||||
|
||||
it("omits actions when no incident is present", async () => {
|
||||
|
||||
@@ -99,12 +99,6 @@ describe("TelegramProvider", () => {
|
||||
expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident");
|
||||
});
|
||||
|
||||
it("links the incident to the monitor's incidents page", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
|
||||
expect(mockGotPost.mock.calls[0][1].json.text).toContain('<a href="https://app.example.com/incidents/mon-1">View Incident</a>');
|
||||
});
|
||||
|
||||
it("omits optional sections when not present", async () => {
|
||||
const { provider } = createProvider();
|
||||
const msg = makeMessage();
|
||||
|
||||
@@ -86,12 +86,6 @@ describe("WebhookProvider", () => {
|
||||
expect(mockGotPost.mock.calls[0][1].json.text).toContain("View Incident");
|
||||
});
|
||||
|
||||
it("links the incident to the monitor's incidents page", async () => {
|
||||
const { provider } = createProvider();
|
||||
await provider.sendMessage(makeNotification() as any, makeMessageWithIncident());
|
||||
expect(mockGotPost.mock.calls[0][1].json.text).toContain("[View Incident](https://app.example.com/incidents/mon-1)");
|
||||
});
|
||||
|
||||
it("omits threshold and incident sections when not present", async () => {
|
||||
const { provider } = createProvider();
|
||||
const msg = makeMessage();
|
||||
|
||||
@@ -3,7 +3,7 @@ import { CheckService } from "../../../src/domain/checks/check.service.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts";
|
||||
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
|
||||
@@ -357,34 +357,6 @@ describe("CheckService", () => {
|
||||
expect(fromRaw[0].metric).toBe("cpu");
|
||||
expect(fromRebuilt).toEqual(fromRaw);
|
||||
});
|
||||
|
||||
it("docker: preserves payload.containers and payload.summary verbatim", () => {
|
||||
const { service } = createService();
|
||||
const payload: DockerStatusPayload = {
|
||||
containers: [
|
||||
{ id: "1", name: "db", image: "postgres:16", state: "exited", status: "Exited (1) 2 minutes ago", health: "none", exitCode: 1 },
|
||||
{ id: "2", name: "api", image: "api:latest", state: "running", status: "Up 3 hours (unhealthy)", health: "unhealthy", restartCount: 2 },
|
||||
],
|
||||
summary: { total: 2, running: 1, stopped: 1, unhealthy: 1 },
|
||||
};
|
||||
const status = makeStatusResponse({ type: "docker", status: true, payload } as any);
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
const roundtripped = rebuilt.payload as DockerStatusPayload;
|
||||
expect(roundtripped.containers).toEqual(payload.containers);
|
||||
expect(roundtripped.summary).toEqual(payload.summary);
|
||||
expect(roundtripped.containers[0].exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it("docker: leaves payload undefined for a failed check with no containers", () => {
|
||||
const { service } = createService();
|
||||
const status = makeStatusResponse({ type: "docker", status: false, code: 5000, message: "connect ECONNREFUSED", payload: null } as any);
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
expect(rebuilt.payload).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── createChecks ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -335,60 +335,17 @@ describe("EmailService", () => {
|
||||
const { service, logger } = createService({ nodemailer });
|
||||
|
||||
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig())).rejects.toThrow(
|
||||
"SMTP host smtp.example.com:587 failed verification."
|
||||
"Email transporter verification failed"
|
||||
);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
message: "Email transporter verification failed: SMTP auth failed",
|
||||
message: "Email transporter verification failed",
|
||||
service: "EmailService",
|
||||
method: "verifyTransporter",
|
||||
details: { host: "smtp.example.com", port: 587 },
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["EAUTH", "SMTP host smtp.example.com:587 rejected the login. Check the user name and password."],
|
||||
["ETIMEDOUT", "SMTP host smtp.example.com:587 did not respond. Check the port and the secure setting."],
|
||||
["EDNS", "SMTP host smtp.example.com:587 could not be resolved. Check the host name."],
|
||||
["ECONNECTION", "SMTP host smtp.example.com:587 refused the connection. Check the host and port."],
|
||||
["ETLS", "SMTP host smtp.example.com:587 failed the TLS handshake. Check the secure and TLS settings."],
|
||||
])("describes a %s verification failure without the server's reply text", async (code, expected) => {
|
||||
const transporter = createMockTransporter();
|
||||
(transporter.verify as jest.Mock).mockRejectedValue(Object.assign(new Error("535-5.7.8 Username and Password not accepted"), { code }));
|
||||
const nodemailer = createMockNodemailer(transporter);
|
||||
const { service } = createService({ nodemailer });
|
||||
|
||||
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig())).rejects.toMatchObject({
|
||||
message: expected,
|
||||
status: 502,
|
||||
details: { cause: "535-5.7.8 Username and Password not accepted" },
|
||||
});
|
||||
});
|
||||
|
||||
it("throws a 400 without building a transport when the host is not configured", async () => {
|
||||
const nodemailer = createMockNodemailer();
|
||||
const { service } = createService({ nodemailer });
|
||||
|
||||
await expect(
|
||||
service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig({ systemEmailHost: undefined }))
|
||||
).rejects.toMatchObject({
|
||||
status: 400,
|
||||
message: "Email is not configured. Set the system email host in settings.",
|
||||
});
|
||||
expect(nodemailer.createTransport).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("treats an empty host as not configured", async () => {
|
||||
const { service } = createService();
|
||||
|
||||
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig({ systemEmailHost: "" }))).rejects.toMatchObject(
|
||||
{
|
||||
status: 400,
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
it("reports the underlying cause when verification fails", async () => {
|
||||
const transporter = createMockTransporter();
|
||||
(transporter.verify as jest.Mock).mockRejectedValue(new Error("SMTP auth failed"));
|
||||
@@ -396,7 +353,7 @@ describe("EmailService", () => {
|
||||
const { service } = createService({ nodemailer });
|
||||
|
||||
await expect(service.sendEmail("to@example.com", "Subject", "<p>body</p>", makeTransportConfig())).rejects.toMatchObject({
|
||||
status: 502,
|
||||
status: 500,
|
||||
details: { cause: "SMTP auth failed" },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -175,46 +175,6 @@ describe("IncidentService", () => {
|
||||
expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ message: "Threshold breach detected" }));
|
||||
});
|
||||
|
||||
it("builds the container list for docker threshold_breach incidents", async () => {
|
||||
const created = makeIncident();
|
||||
const { service, incidentsRepository, notificationMessageBuilder } = createService();
|
||||
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
|
||||
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
|
||||
|
||||
const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true });
|
||||
const response = {
|
||||
monitorId: "mon-1",
|
||||
payload: {
|
||||
containers: [
|
||||
{ id: "1", name: "db", image: "postgres", state: "exited", status: "Exited (1)", health: "none", exitCode: 1 },
|
||||
{ id: "2", name: "api", image: "api", state: "running", status: "Up", health: "unhealthy" },
|
||||
{ id: "3", name: "web", image: "nginx", state: "running", status: "Up", health: "healthy" },
|
||||
],
|
||||
summary: { total: 3, running: 2, stopped: 1, unhealthy: 1 },
|
||||
},
|
||||
} as any;
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
|
||||
await service.handleIncident(monitor, 200, decision, response);
|
||||
|
||||
expect(incidentsRepository.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ statusCode: 9999, message: "db: stopped (exit code 1), api: unhealthy" })
|
||||
);
|
||||
expect(notificationMessageBuilder.extractThresholdBreaches).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the container fallback message for docker incidents with no response", async () => {
|
||||
const created = makeIncident();
|
||||
const { service, incidentsRepository } = createService();
|
||||
(incidentsRepository.findActiveByMonitorId as jest.Mock).mockResolvedValue(null);
|
||||
(incidentsRepository.create as jest.Mock).mockResolvedValue(created);
|
||||
|
||||
const monitor = makeMonitor({ type: "docker", status: "breached", dockerAlertOnStopped: true });
|
||||
const decision = makeDecision({ shouldCreateIncident: true, incidentReason: "threshold_breach" });
|
||||
await service.handleIncident(monitor, 200, decision, undefined);
|
||||
|
||||
expect(incidentsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 9999, message: "Container alert" }));
|
||||
});
|
||||
|
||||
it("joins multiple threshold breaches with commas", async () => {
|
||||
const created = makeIncident();
|
||||
const { service, incidentsRepository, notificationMessageBuilder } = createService();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
@@ -51,29 +51,6 @@ const makeHardwarePayload = (overrides?: Partial<HardwareStatusPayload["data"]>)
|
||||
},
|
||||
}) as HardwareStatusPayload;
|
||||
|
||||
const makeDockerPayload = (containers: Partial<DockerStatusPayload["containers"][number]>[]): DockerStatusPayload => ({
|
||||
containers: containers.map((container, index) => ({
|
||||
id: `container-${index}`,
|
||||
name: `container-${index}`,
|
||||
image: "nginx:latest",
|
||||
state: "running",
|
||||
status: "Up 3 hours",
|
||||
health: "none",
|
||||
...container,
|
||||
})),
|
||||
summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 },
|
||||
});
|
||||
|
||||
const makeDockerMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
makeMonitor({
|
||||
type: "docker",
|
||||
status: "breached",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: true,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("NotificationMessageBuilder", () => {
|
||||
@@ -146,7 +123,7 @@ describe("NotificationMessageBuilder", () => {
|
||||
|
||||
it("builds a threshold_resolved message for hardware monitor recovering", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const decision = makeDecision({ notificationReason: "threshold_resolved" });
|
||||
const decision = makeDecision({ notificationReason: "status_change" });
|
||||
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "https://app.example.com");
|
||||
|
||||
@@ -186,46 +163,13 @@ describe("NotificationMessageBuilder", () => {
|
||||
expect(msg.type).toBe("threshold_breach");
|
||||
});
|
||||
|
||||
it("returns threshold_resolved for hardware monitor with threshold_resolved reason", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const decision = makeDecision({ notificationReason: "threshold_resolved" });
|
||||
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("threshold_resolved");
|
||||
});
|
||||
|
||||
it("returns monitor_up for hardware monitor recovering from down with status_change", () => {
|
||||
it("returns threshold_resolved for hardware monitor with status_change and up", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const decision = makeDecision({ notificationReason: "status_change" });
|
||||
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("monitor_up");
|
||||
});
|
||||
|
||||
it("returns container_breach for docker monitor with threshold_breach reason", () => {
|
||||
const decision = makeDecision({ notificationReason: "threshold_breach" });
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("container_breach");
|
||||
});
|
||||
|
||||
it("returns container_resolved for docker monitor with threshold_resolved reason", () => {
|
||||
const decision = makeDecision({ notificationReason: "threshold_resolved" });
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor({ status: "up" }), makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("container_resolved");
|
||||
});
|
||||
|
||||
it("returns monitor_down for docker monitor that is down regardless of reason", () => {
|
||||
const decision = makeDecision({ notificationReason: "threshold_breach" });
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor({ status: "down" }), makeStatusResponse(), decision, "");
|
||||
|
||||
expect(msg.type).toBe("monitor_down");
|
||||
expect(msg.type).toBe("threshold_resolved");
|
||||
});
|
||||
|
||||
it("returns monitor_up for non-hardware monitor with status up", () => {
|
||||
@@ -279,22 +223,7 @@ describe("NotificationMessageBuilder", () => {
|
||||
const msg = builder.buildMessage(
|
||||
makeMonitor({ status: "up", type: "hardware" }),
|
||||
makeStatusResponse(),
|
||||
makeDecision({ notificationReason: "threshold_resolved" }),
|
||||
""
|
||||
);
|
||||
expect(msg.severity).toBe("success");
|
||||
});
|
||||
|
||||
it("returns warning for container_breach", () => {
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), makeStatusResponse(), makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
expect(msg.severity).toBe("warning");
|
||||
});
|
||||
|
||||
it("returns success for container_resolved", () => {
|
||||
const msg = builder.buildMessage(
|
||||
makeDockerMonitor({ status: "up" }),
|
||||
makeStatusResponse(),
|
||||
makeDecision({ notificationReason: "threshold_resolved" }),
|
||||
makeDecision({ notificationReason: "status_change" }),
|
||||
""
|
||||
);
|
||||
expect(msg.severity).toBe("success");
|
||||
@@ -382,7 +311,7 @@ describe("NotificationMessageBuilder", () => {
|
||||
describe("threshold_resolved", () => {
|
||||
it("includes resolved details", () => {
|
||||
const monitor = makeMonitor({ status: "up", type: "hardware" });
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision({ notificationReason: "threshold_resolved" }), "");
|
||||
const msg = builder.buildMessage(monitor, makeStatusResponse(), makeDecision(), "");
|
||||
|
||||
expect(msg.content.title).toBe("Thresholds Resolved: Test Monitor");
|
||||
expect(msg.content.summary).toBe('Monitor "Test Monitor" thresholds have returned to normal.');
|
||||
@@ -390,63 +319,6 @@ describe("NotificationMessageBuilder", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("container_breach", () => {
|
||||
it("lists one detail line per breaching container", () => {
|
||||
const response = makeStatusResponse({
|
||||
type: "docker",
|
||||
status: true,
|
||||
payload: makeDockerPayload([{ name: "db", state: "exited", exitCode: 1 }, { name: "api", health: "unhealthy" }, { name: "web" }]),
|
||||
} as any);
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
|
||||
expect(msg.content.title).toBe("Container Alert: Test Monitor");
|
||||
expect(msg.content.summary).toBe('2 container(s) on "Test Monitor" need attention.');
|
||||
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "db: stopped (exit code 1)", "api: unhealthy"]);
|
||||
expect(msg.content.thresholds).toBeUndefined();
|
||||
});
|
||||
|
||||
it("only lists containers matching an enabled switch", () => {
|
||||
const response = makeStatusResponse({
|
||||
type: "docker",
|
||||
status: true,
|
||||
payload: makeDockerPayload([
|
||||
{ name: "db", state: "exited", exitCode: 1 },
|
||||
{ name: "api", health: "unhealthy" },
|
||||
]),
|
||||
} as any);
|
||||
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false });
|
||||
|
||||
const msg = builder.buildMessage(monitor, response, makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
|
||||
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker", "api: unhealthy"]);
|
||||
});
|
||||
|
||||
it("lists no containers when the payload is an error string", () => {
|
||||
const response = makeStatusResponse({ type: "docker", status: false, payload: "connect ECONNREFUSED" } as any);
|
||||
|
||||
const msg = builder.buildMessage(makeDockerMonitor(), response, makeDecision({ notificationReason: "threshold_breach" }), "");
|
||||
|
||||
expect(msg.content.summary).toBe('0 container(s) on "Test Monitor" need attention.');
|
||||
expect(msg.content.details).toEqual(["URL: unix:///var/run/docker.sock", "Type: docker"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("container_resolved", () => {
|
||||
it("includes recovered details", () => {
|
||||
const msg = builder.buildMessage(
|
||||
makeDockerMonitor({ status: "up" }),
|
||||
makeStatusResponse(),
|
||||
makeDecision({ notificationReason: "threshold_resolved" }),
|
||||
""
|
||||
);
|
||||
|
||||
expect(msg.content.title).toBe("Containers Recovered: Test Monitor");
|
||||
expect(msg.content.summary).toBe('All containers on "Test Monitor" are back to normal.');
|
||||
expect(msg.content.details).toContain("Status: Up");
|
||||
});
|
||||
});
|
||||
|
||||
describe("default content", () => {
|
||||
it("builds default content for unhandled notification type", () => {
|
||||
// Force default by creating a scenario where type falls through
|
||||
|
||||
@@ -311,23 +311,6 @@ describe("NotificationsService", () => {
|
||||
const result = await service.testAllNotifications(["notif-1"]);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it("counts a provider that throws as a failed test and still tests the others", async () => {
|
||||
const { service, notificationsRepository, emailProvider, slackProvider, logger } = createService();
|
||||
emailProvider.sendTestAlert.mockRejectedValue(new Error("Email is not configured. Set the system email host in settings."));
|
||||
(notificationsRepository.findNotificationsByIds as jest.Mock).mockResolvedValue([
|
||||
makeNotification({ type: "email" }),
|
||||
makeNotification({ type: "slack" }),
|
||||
]);
|
||||
|
||||
const result = await service.testAllNotifications(["notif-1", "notif-2"]);
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(slackProvider.sendTestAlert).toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: "Test notification failed: Email is not configured. Set the system email host in settings." })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── CRUD operations ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import { MAX_RECENT_CHECKS } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { Monitor, MonitorStatus } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { Check } from "../../../src/domain/checks/check.type.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, DockerStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts";
|
||||
|
||||
@@ -1086,162 +1086,6 @@ describe("StatusService", () => {
|
||||
// they're private implementation details; exposing them publicly would
|
||||
// widen the API surface just for testability.
|
||||
|
||||
describe("docker container alerts", () => {
|
||||
const makeDockerMonitor = (overrides?: Partial<Monitor>) =>
|
||||
makeMonitor({
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
statusWindow: [true, true, true, true],
|
||||
statusWindowSize: 5,
|
||||
status: "up",
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: true,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const makeContainers = (containers: Partial<DockerStatusPayload["containers"][number]>[]): DockerStatusPayload => ({
|
||||
containers: containers.map((container, index) => ({
|
||||
id: `container-${index}`,
|
||||
name: `container-${index}`,
|
||||
image: "nginx:latest",
|
||||
state: "running",
|
||||
status: "Up 3 hours",
|
||||
health: "none",
|
||||
...container,
|
||||
})),
|
||||
summary: { total: containers.length, running: 0, stopped: 0, unhealthy: 0 },
|
||||
});
|
||||
|
||||
const makeDockerResponse = (payload: DockerStatusPayload | null, overrides?: Partial<MonitorStatusResponse<DockerStatusPayload>>) =>
|
||||
makeStatusResponse({
|
||||
type: "docker",
|
||||
status: true,
|
||||
payload,
|
||||
...overrides,
|
||||
} as any) as MonitorStatusResponse<DockerStatusPayload>;
|
||||
|
||||
const run = async (monitor: Monitor, response: MonitorStatusResponse<DockerStatusPayload>, check = makeCheck()) => {
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
const result = await service.updateMonitorStatus(response, check, monitor);
|
||||
return { result, monitorsRepository };
|
||||
};
|
||||
|
||||
const stopped = { name: "db", state: "exited" as const, exitCode: 1 };
|
||||
const unhealthy = { name: "api", health: "unhealthy" as const };
|
||||
|
||||
it("goes to breached on the first check with a stopped container", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("goes to breached on the first check with an unhealthy container", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([unhealthy])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("stays up on a stopped container when both switches are off", async () => {
|
||||
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false, dockerAlertOnUnhealthy: false });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped, unhealthy])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("stays up on a stopped container when only the unhealthy switch is on", async () => {
|
||||
const monitor = makeDockerMonitor({ dockerAlertOnStopped: false });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("stays up on an exited container with code 0", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "job", state: "exited", exitCode: 0 }])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("stays breached without a transition while a container is still down", async () => {
|
||||
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("stays up without a transition while every container is clean", async () => {
|
||||
const { result } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([{ name: "web" }])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("recovers from breached to up on the first clean check", async () => {
|
||||
const { result } = await run(makeDockerMonitor({ status: "breached" }), makeDockerResponse(makeContainers([{ name: "web" }])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.prevStatus).toBe("breached");
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("recovers from breached to up when the breaching switch is turned off", async () => {
|
||||
const monitor = makeDockerMonitor({ status: "breached", dockerAlertOnStopped: false });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("does not override down with breached when the host is unreachable", async () => {
|
||||
const monitor = makeDockerMonitor({ statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
|
||||
const response = makeDockerResponse(null, { status: false, code: 5000 });
|
||||
const { result } = await run(monitor, response, makeCheck({ status: false }));
|
||||
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
|
||||
it("transitions from breached to down when the reachability threshold trips", async () => {
|
||||
const monitor = makeDockerMonitor({ status: "breached", statusWindow: [false, false, false, false], statusWindowThreshold: 80 });
|
||||
const response = makeDockerResponse(null, { status: false, code: 5000 });
|
||||
const { result } = await run(monitor, response, makeCheck({ status: false }));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
|
||||
it("does not evaluate containers while the status window is short", async () => {
|
||||
const monitor = makeDockerMonitor({ statusWindow: [true], statusWindowSize: 5 });
|
||||
const { result } = await run(monitor, makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("up");
|
||||
});
|
||||
|
||||
it("does not evaluate a failed check with a null payload", async () => {
|
||||
const monitor = makeDockerMonitor({ status: "breached" });
|
||||
const response = makeDockerResponse(null, { status: false, code: 5000 });
|
||||
const { result } = await run(monitor, response, makeCheck({ status: false }));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("breached");
|
||||
});
|
||||
|
||||
it("does not touch the hardware counters or thresholdBreaches", async () => {
|
||||
const { result, monitorsRepository } = await run(makeDockerMonitor(), makeDockerResponse(makeContainers([stopped])));
|
||||
|
||||
expect(result.thresholdBreaches).toBeUndefined();
|
||||
const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6];
|
||||
expect(patch.cpuAlertCounter).toBeUndefined();
|
||||
expect(patch.status).toBe("breached");
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeReachability (pure)", () => {
|
||||
const reach = (currentStatus: MonitorStatus, window: boolean[], threshold = 80) => {
|
||||
const { service } = createService();
|
||||
|
||||
@@ -861,14 +861,9 @@ describe("WorkerPipeline", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves incident with threshold_resolved when monitor recovers from breached", async () => {
|
||||
it("resolves incident when monitor recovers from breached", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" }));
|
||||
expect(decision).toMatchObject({
|
||||
shouldResolveIncident: true,
|
||||
shouldSendNotification: true,
|
||||
shouldCreateIncident: false,
|
||||
notificationReason: "threshold_resolved",
|
||||
});
|
||||
expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
|
||||
});
|
||||
|
||||
it("does not create or resolve for unhandled status transitions", async () => {
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { describeContainerBreach, findContainerBreaches, isContainerStopped, isContainerUnhealthy } from "../../../src/domain/docker/docker-alert.ts";
|
||||
import type { DockerContainerInfo } from "../../../src/domain/docker/docker.type.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
|
||||
const makeContainer = (overrides?: Partial<DockerContainerInfo>): DockerContainerInfo => ({
|
||||
id: "abc123",
|
||||
name: "web",
|
||||
image: "nginx:latest",
|
||||
state: "running",
|
||||
status: "Up 3 hours",
|
||||
health: "none",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "mon-1",
|
||||
type: "docker",
|
||||
dockerAlertOnStopped: false,
|
||||
dockerAlertOnUnhealthy: false,
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
describe("isContainerStopped", () => {
|
||||
it("treats dead as stopped", () => {
|
||||
expect(isContainerStopped(makeContainer({ state: "dead" }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each([1, 137, 143])("treats exited with code %s as stopped", (exitCode) => {
|
||||
expect(isContainerStopped(makeContainer({ state: "exited", exitCode }))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat exited with code 0 as stopped", () => {
|
||||
expect(isContainerStopped(makeContainer({ state: "exited", exitCode: 0 }))).toBe(false);
|
||||
});
|
||||
|
||||
it("treats exited with an unknown exit code as stopped", () => {
|
||||
expect(isContainerStopped(makeContainer({ state: "exited" }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["running", "restarting", "paused", "created", "removing"] as const)("does not treat %s as stopped", (state) => {
|
||||
expect(isContainerStopped(makeContainer({ state, exitCode: 1 }))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isContainerUnhealthy", () => {
|
||||
it("treats unhealthy as unhealthy", () => {
|
||||
expect(isContainerUnhealthy(makeContainer({ health: "unhealthy" }))).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["healthy", "starting", "none"] as const)("does not treat %s as unhealthy", (health) => {
|
||||
expect(isContainerUnhealthy(makeContainer({ health }))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findContainerBreaches", () => {
|
||||
const stopped = makeContainer({ name: "db", state: "exited", exitCode: 1 });
|
||||
const unhealthy = makeContainer({ name: "api", health: "unhealthy" });
|
||||
|
||||
it("returns nothing when both switches are off", () => {
|
||||
expect(findContainerBreaches(makeMonitor(), [stopped, unhealthy])).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports only stopped containers when only the stopped switch is on", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true }), [stopped, unhealthy]);
|
||||
expect(breaches).toEqual([{ name: "db", reason: "stopped", exitCode: 1 }]);
|
||||
});
|
||||
|
||||
it("reports only unhealthy containers when only the unhealthy switch is on", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnUnhealthy: true }), [stopped, unhealthy]);
|
||||
expect(breaches).toEqual([{ name: "api", reason: "unhealthy" }]);
|
||||
});
|
||||
|
||||
it("reports both conditions when both switches are on", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [stopped, unhealthy]);
|
||||
expect(breaches.map((b) => b.reason)).toEqual(["stopped", "unhealthy"]);
|
||||
});
|
||||
|
||||
it("reports a container that is both stopped and unhealthy once, as stopped", () => {
|
||||
const both = makeContainer({ name: "worker", state: "dead", health: "unhealthy" });
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [both]);
|
||||
expect(breaches).toEqual([{ name: "worker", reason: "stopped", exitCode: undefined }]);
|
||||
});
|
||||
|
||||
it("ignores healthy running containers", () => {
|
||||
const breaches = findContainerBreaches(makeMonitor({ dockerAlertOnStopped: true, dockerAlertOnUnhealthy: true }), [makeContainer()]);
|
||||
expect(breaches).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("describeContainerBreach", () => {
|
||||
it("describes an unhealthy container", () => {
|
||||
expect(describeContainerBreach({ name: "api", reason: "unhealthy" })).toBe("api: unhealthy");
|
||||
});
|
||||
|
||||
it("describes a stopped container with its exit code", () => {
|
||||
expect(describeContainerBreach({ name: "db", reason: "stopped", exitCode: 137 })).toBe("db: stopped (exit code 137)");
|
||||
});
|
||||
|
||||
it("describes a stopped container without an exit code", () => {
|
||||
expect(describeContainerBreach({ name: "db", reason: "stopped" })).toBe("db: stopped");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import type { SSLDetails, SSLOptions } from "ssl-checker";
|
||||
import { fetchMonitorCertificate } from "../../../src/api/controllers/controllerUtils.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
|
||||
const makeMonitor = (url: string, port?: number): Monitor =>
|
||||
({
|
||||
url,
|
||||
port,
|
||||
type: "http",
|
||||
}) as unknown as Monitor;
|
||||
|
||||
const cert: SSLDetails = {
|
||||
daysRemaining: 30,
|
||||
valid: true,
|
||||
validFrom: "2026-01-01T00:00:00Z",
|
||||
validTo: "2026-12-31T00:00:00Z",
|
||||
validFor: ["example.com"],
|
||||
};
|
||||
|
||||
const setup = () => {
|
||||
const calls: { hostname: string; options?: SSLOptions }[] = [];
|
||||
const checker = async (hostname: string, options?: SSLOptions) => {
|
||||
calls.push({ hostname, options });
|
||||
return cert;
|
||||
};
|
||||
return { calls, checker };
|
||||
};
|
||||
|
||||
describe("fetchMonitorCertificate", () => {
|
||||
it("uses the default port when none is specified", async () => {
|
||||
const { calls, checker } = setup();
|
||||
|
||||
await fetchMonitorCertificate(checker, makeMonitor("https://example.com/health"));
|
||||
|
||||
expect(calls).toEqual([{ hostname: "example.com", options: undefined }]);
|
||||
});
|
||||
|
||||
it("uses the port from the URL", async () => {
|
||||
const { calls, checker } = setup();
|
||||
|
||||
await fetchMonitorCertificate(checker, makeMonitor("https://example.com:54321/health"));
|
||||
|
||||
expect(calls).toEqual([{ hostname: "example.com", options: { port: 54321 } }]);
|
||||
});
|
||||
|
||||
it("falls back to monitor.port when the URL has no port", async () => {
|
||||
const { calls, checker } = setup();
|
||||
|
||||
await fetchMonitorCertificate(checker, makeMonitor("https://example.com/health", 8443));
|
||||
|
||||
expect(calls).toEqual([{ hostname: "example.com", options: { port: 8443 } }]);
|
||||
});
|
||||
|
||||
it("prefers the URL port over monitor.port", async () => {
|
||||
const { calls, checker } = setup();
|
||||
|
||||
await fetchMonitorCertificate(checker, makeMonitor("https://www.myurl.com:52345/health", 443));
|
||||
|
||||
expect(calls).toEqual([{ hostname: "www.myurl.com", options: { port: 52345 } }]);
|
||||
});
|
||||
|
||||
it("throws when the certificate has no expiry", async () => {
|
||||
const checker = async () => ({ ...cert, validTo: undefined }) as unknown as SSLDetails;
|
||||
|
||||
await expect(fetchMonitorCertificate(checker, makeMonitor("https://example.com"))).rejects.toThrow("Certificate not found");
|
||||
});
|
||||
});
|
||||
@@ -247,20 +247,6 @@ describe("monitorValidation — strategy gating", () => {
|
||||
expect(parsed.monitors[0].dockerLogsEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("defaults the docker alert switches to false on imported docker monitors", () => {
|
||||
const parsed = importMonitorsBodyValidation.parse({
|
||||
monitors: [
|
||||
{
|
||||
name: "Imported Docker",
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(parsed.monitors[0].dockerAlertOnStopped).toBe(false);
|
||||
expect(parsed.monitors[0].dockerAlertOnUnhealthy).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects strategy on imported non-pagespeed monitors", () => {
|
||||
expect(() =>
|
||||
importMonitorsBodyValidation.parse({
|
||||
@@ -832,43 +818,3 @@ describe("monitorValidation — Docker TLS credentials", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("monitorValidation — docker alert switches", () => {
|
||||
describe("createMonitorBodyValidation", () => {
|
||||
it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on a docker monitor", () => {
|
||||
const parsed = createMonitorBodyValidation.parse({
|
||||
name: "Docker host",
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: true,
|
||||
});
|
||||
|
||||
expect(parsed.dockerAlertOnStopped).toBe(true);
|
||||
expect(parsed.dockerAlertOnUnhealthy).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves the switches undefined when omitted", () => {
|
||||
const parsed = createMonitorBodyValidation.parse({
|
||||
name: "Docker host",
|
||||
type: "docker",
|
||||
url: "unix:///var/run/docker.sock",
|
||||
});
|
||||
|
||||
expect(parsed.dockerAlertOnStopped).toBeUndefined();
|
||||
expect(parsed.dockerAlertOnUnhealthy).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("editMonitorBodyValidation", () => {
|
||||
it("retains dockerAlertOnStopped and dockerAlertOnUnhealthy on edits", () => {
|
||||
const parsed = editMonitorBodyValidation.parse({
|
||||
dockerAlertOnStopped: true,
|
||||
dockerAlertOnUnhealthy: false,
|
||||
});
|
||||
|
||||
expect(parsed.dockerAlertOnStopped).toBe(true);
|
||||
expect(parsed.dockerAlertOnUnhealthy).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user