Merge latest develop into monitor sorting
# Conflicts: # client/src/Features/UI/uiSlice.ts
This commit is contained in:
commit
e15a85bd61
75 files changed
+1954
-1376
No files matched your search
@@ -35,11 +35,18 @@ class NotificationController implements INotificationController {
|
||||
|
||||
testNotification = catchAsync(async (req: Request, res: Response) => {
|
||||
const notification = testNotificationBodyValidation.parse(req.body);
|
||||
const success = await this.notificationsService.sendTestNotification(notification);
|
||||
let success = false;
|
||||
let reason: string | undefined;
|
||||
try {
|
||||
success = await this.notificationsService.sendTestNotification(notification);
|
||||
} catch (error: unknown) {
|
||||
if (!(error instanceof AppError)) throw error;
|
||||
reason = error.message;
|
||||
}
|
||||
|
||||
return res.status(200).json({
|
||||
success,
|
||||
msg: success ? "Notification sent successfully" : "Notification could not be sent — check the destination details.",
|
||||
msg: success ? "Notification sent successfully" : (reason ?? "Notification could not be sent — check the destination details."),
|
||||
details: { service: SERVICE_NAME },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,8 @@ import { IProxiesService } from "@/domain/proxies/proxy.service.js";
|
||||
import { IEgressStateService } from "@/domain/egress/egress-state.service.js";
|
||||
import { Settings } from "@/domain/app-settings/app-settings.type.js";
|
||||
|
||||
const SERVICE_NAME = "SettingsController";
|
||||
|
||||
export interface ISettingsController {
|
||||
getAppSettings: RequestHandler;
|
||||
updateAppSettings: RequestHandler;
|
||||
@@ -119,7 +121,7 @@ class SettingsController implements ISettingsController {
|
||||
|
||||
const html = await this.emailService.buildEmail("testEmailTemplate", context);
|
||||
if (!html) {
|
||||
throw new AppError({ message: "Failed to build email template.", status: 500 });
|
||||
throw new AppError({ message: "Failed to build email template.", status: 500, service: SERVICE_NAME, method: "sendTestEmail" });
|
||||
}
|
||||
let messageId: string;
|
||||
try {
|
||||
@@ -142,8 +144,11 @@ class SettingsController implements ISettingsController {
|
||||
// Surface the underlying SMTP failure: diagnosing the settings is the whole
|
||||
// point of the test email endpoint.
|
||||
throw new AppError({
|
||||
message: error instanceof Error ? `Failed to send test email. ${error.message}` : "Failed to send test email.",
|
||||
status: 500,
|
||||
message: error instanceof AppError ? error.message : "Failed to send test email.",
|
||||
status: error instanceof AppError ? error.status : 500,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendTestEmail",
|
||||
details: error instanceof AppError ? error.details : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -220,6 +220,8 @@ export const createMonitorBodyValidation = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
|
||||
geoCheckInterval: z.number().min(300000).optional(),
|
||||
dockerLogsEnabled: z.boolean().optional(),
|
||||
dockerAlertOnStopped: z.boolean().optional(),
|
||||
dockerAlertOnUnhealthy: z.boolean().optional(),
|
||||
dockerTlsCa: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsCert: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsKey: z.union([z.string(), z.literal("")]).optional(),
|
||||
@@ -270,6 +272,8 @@ export const editMonitorBodyValidation = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
|
||||
geoCheckInterval: z.number().min(300000).optional(),
|
||||
dockerLogsEnabled: z.boolean().optional(),
|
||||
dockerAlertOnStopped: z.boolean().optional(),
|
||||
dockerAlertOnUnhealthy: z.boolean().optional(),
|
||||
dockerTlsCa: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsCert: z.union([z.string(), z.literal("")]).optional(),
|
||||
dockerTlsKey: z.union([z.string(), z.literal("")]).optional(),
|
||||
@@ -350,6 +354,8 @@ const importedMonitorSchema = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)).default([]),
|
||||
geoCheckInterval: z.number().min(300000).default(300000),
|
||||
dockerLogsEnabled: z.boolean().default(false),
|
||||
dockerAlertOnStopped: z.boolean().default(false),
|
||||
dockerAlertOnUnhealthy: z.boolean().default(false),
|
||||
dnsServer: dnsServerValidation.optional(),
|
||||
dnsRecordType: z.enum(DnsRecordTypes).optional(),
|
||||
createdAt: z.string().optional(),
|
||||
@@ -440,6 +446,8 @@ export const monitorResponseSchema = z
|
||||
geoCheckLocations: z.array(z.enum(GeoContinents)),
|
||||
geoCheckInterval: z.number(),
|
||||
dockerLogsEnabled: z.boolean(),
|
||||
dockerAlertOnStopped: z.boolean(),
|
||||
dockerAlertOnUnhealthy: z.boolean(),
|
||||
dockerTlsCa: z.string().optional(),
|
||||
dockerTlsCert: z.string().optional(),
|
||||
dockerTlsKeySet: z.boolean().optional(),
|
||||
|
||||
@@ -220,6 +220,7 @@ const dockerContainerSchema = new Schema<DockerContainerInfo>(
|
||||
startedAt: { type: String },
|
||||
ports: { type: [dockerContainerPortSchema], default: undefined },
|
||||
mounts: { type: [dockerContainerMountSchema], default: undefined },
|
||||
exitCode: { type: Number, default: undefined },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
@@ -13,7 +13,6 @@ const SERVICE_NAME = "checkService";
|
||||
export interface ICheckService {
|
||||
createChecks(checks: Check[]): Promise<Check[]>;
|
||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||
getChecksByMonitor(params: {
|
||||
monitorId: string;
|
||||
teamId: string;
|
||||
@@ -135,24 +134,6 @@ export class CheckService implements ICheckService {
|
||||
return check;
|
||||
};
|
||||
|
||||
toStatusResponse = (check: Check) => {
|
||||
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
|
||||
monitorId: check.metadata.monitorId,
|
||||
teamId: check.metadata.teamId,
|
||||
type: check.metadata.type,
|
||||
status: check.status,
|
||||
code: check.statusCode,
|
||||
message: check.message,
|
||||
responseTime: check.responseTime,
|
||||
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
|
||||
payload:
|
||||
check.metadata.type === "hardware"
|
||||
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
|
||||
: undefined,
|
||||
};
|
||||
return statusResponse;
|
||||
};
|
||||
|
||||
getChecksByMonitor = async ({
|
||||
monitorId,
|
||||
teamId,
|
||||
|
||||
@@ -45,12 +45,14 @@ const mapMemory = (memory?: CheckMemoryInfo): SnapshotMemoryInfo | undefined =>
|
||||
};
|
||||
|
||||
const mapDisk = (disk?: CheckDiskInfo[]): SnapshotDiskInfo[] | undefined =>
|
||||
disk?.map((entry) => ({
|
||||
device: entry.device,
|
||||
total_bytes: entry.total_bytes,
|
||||
used_bytes: entry.used_bytes,
|
||||
usage_percent: entry.usage_percent,
|
||||
}));
|
||||
disk
|
||||
?.filter((entry) => entry != null)
|
||||
.map((entry) => ({
|
||||
device: entry.device,
|
||||
total_bytes: entry.total_bytes,
|
||||
used_bytes: entry.used_bytes,
|
||||
usage_percent: entry.usage_percent,
|
||||
}));
|
||||
|
||||
const mapHost = (host?: CheckHostInfo): SnapshotHostInfo | undefined =>
|
||||
host && {
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { DockerContainerInfo } from "@/domain/docker/docker.type.js";
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
|
||||
export const DockerAlertReasons = ["stopped", "unhealthy"] as const;
|
||||
export type DockerAlertReason = (typeof DockerAlertReasons)[number];
|
||||
|
||||
export interface DockerContainerBreach {
|
||||
name: string;
|
||||
reason: DockerAlertReason;
|
||||
exitCode?: number;
|
||||
}
|
||||
|
||||
export const isContainerStopped = (container: DockerContainerInfo): boolean => {
|
||||
// Container with exit code 0 is probably a one shot container don't report stopped
|
||||
return container.state === "dead" || (container.state === "exited" && container.exitCode !== 0);
|
||||
};
|
||||
|
||||
export const isContainerUnhealthy = (container: DockerContainerInfo): boolean => {
|
||||
return container.health === "unhealthy";
|
||||
};
|
||||
|
||||
export const findContainerBreaches = (monitor: Monitor, containers: DockerContainerInfo[]): DockerContainerBreach[] => {
|
||||
const breaches: DockerContainerBreach[] = [];
|
||||
for (const container of containers) {
|
||||
if (monitor.dockerAlertOnStopped && isContainerStopped(container)) {
|
||||
breaches.push({
|
||||
name: container.name,
|
||||
reason: "stopped",
|
||||
exitCode: container.exitCode,
|
||||
});
|
||||
} else if (monitor.dockerAlertOnUnhealthy && isContainerUnhealthy(container)) {
|
||||
breaches.push({
|
||||
name: container.name,
|
||||
reason: "unhealthy",
|
||||
});
|
||||
}
|
||||
}
|
||||
return breaches;
|
||||
};
|
||||
|
||||
export const describeContainerBreach = (breach: DockerContainerBreach): string => {
|
||||
if (breach.reason === "unhealthy") return `${breach.name}: unhealthy`;
|
||||
if (breach.exitCode === undefined) {
|
||||
return `${breach.name}: stopped`;
|
||||
}
|
||||
return `${breach.name}: stopped (exit code ${breach.exitCode})`;
|
||||
};
|
||||
@@ -55,6 +55,7 @@ export interface DockerContainerInfo {
|
||||
startedAt?: string; // ISO date
|
||||
ports?: DockerContainerPort[];
|
||||
mounts?: DockerContainerMount[];
|
||||
exitCode?: number;
|
||||
}
|
||||
|
||||
export interface DockerContainerSummary {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
const SERVICE_NAME = "incidentService";
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { Check } from "@/domain/checks/check.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { getDateForRange } from "@/utils/dataUtils.js";
|
||||
import type { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
|
||||
@@ -13,13 +12,9 @@ import type { INotificationMessageBuilder } from "@/domain/notifications/notific
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import { DateRange } from "@/types/query.js";
|
||||
|
||||
const SERVICE_NAME = "incidentService";
|
||||
export interface IIncidentService {
|
||||
handleIncident(
|
||||
monitor: Monitor,
|
||||
code: number,
|
||||
decision: MonitorActionDecision,
|
||||
monitorStatusResponse?: MonitorStatusResponse
|
||||
): Promise<Incident | null>;
|
||||
handleIncident(monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise<Incident | null>;
|
||||
resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise<Incident>;
|
||||
getIncidentsByTeam(
|
||||
teamId: string,
|
||||
@@ -58,12 +53,7 @@ export class IncidentService implements IIncidentService {
|
||||
this.notificationMessageBuilder = notificationMessageBuilder;
|
||||
}
|
||||
|
||||
handleIncident = async (
|
||||
monitor: Monitor,
|
||||
code: number,
|
||||
decision: MonitorActionDecision,
|
||||
monitorStatusResponse?: MonitorStatusResponse
|
||||
): Promise<Incident | null> => {
|
||||
handleIncident = async (monitor: Monitor, decision: MonitorActionDecision, check: Check): Promise<Incident | null> => {
|
||||
if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) {
|
||||
return null;
|
||||
}
|
||||
@@ -74,13 +64,13 @@ export class IncidentService implements IIncidentService {
|
||||
if (activeIncident) {
|
||||
return activeIncident;
|
||||
} else {
|
||||
let statusCode = code;
|
||||
let statusCode = check.statusCode;
|
||||
let message: string | undefined;
|
||||
|
||||
// For threshold breaches, use 9999 status code and build descriptive message
|
||||
if (decision.incidentReason === "threshold_breach") {
|
||||
statusCode = 9999;
|
||||
message = this.buildThresholdBreachMessage(monitor, monitorStatusResponse);
|
||||
message = this.notificationMessageBuilder.buildThresholdBreachMessage(monitor, check, decision.thresholdBreaches);
|
||||
}
|
||||
|
||||
const incident = {
|
||||
@@ -105,20 +95,6 @@ export class IncidentService implements IIncidentService {
|
||||
return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
|
||||
};
|
||||
|
||||
private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string {
|
||||
if (!monitorStatusResponse) {
|
||||
return "Threshold breach detected";
|
||||
}
|
||||
|
||||
const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse);
|
||||
|
||||
if (breaches.length === 0) {
|
||||
return "Threshold breach detected";
|
||||
}
|
||||
|
||||
return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", ");
|
||||
}
|
||||
|
||||
resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => {
|
||||
try {
|
||||
if (!incidentId) {
|
||||
|
||||
@@ -307,6 +307,14 @@ const MonitorSchema = new Schema<MonitorDocument>(
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
dockerAlertOnStopped: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
dockerAlertOnUnhealthy: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
dockerTlsCa: {
|
||||
type: String,
|
||||
},
|
||||
|
||||
@@ -473,6 +473,8 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
geoCheckLocations: doc.geoCheckLocations ?? [],
|
||||
geoCheckInterval: doc.geoCheckInterval ?? 300000,
|
||||
dockerLogsEnabled: doc.dockerLogsEnabled ?? false,
|
||||
dockerAlertOnStopped: doc.dockerAlertOnStopped ?? false,
|
||||
dockerAlertOnUnhealthy: doc.dockerAlertOnUnhealthy ?? false,
|
||||
dockerTlsCa: doc.dockerTlsCa ?? undefined,
|
||||
dockerTlsCert: doc.dockerTlsCert ?? undefined,
|
||||
dockerTlsKeySet: doc.dockerTlsKeySet ?? false,
|
||||
|
||||
@@ -51,6 +51,22 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
|
||||
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
|
||||
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
|
||||
|
||||
const MonitorPaths: Record<MonitorType, string> = {
|
||||
http: "uptime",
|
||||
port: "uptime",
|
||||
ping: "uptime",
|
||||
game: "uptime",
|
||||
grpc: "uptime",
|
||||
websocket: "uptime",
|
||||
dns: "uptime",
|
||||
unknown: "uptime",
|
||||
docker: "docker/host",
|
||||
hardware: "infrastructure",
|
||||
pagespeed: "pagespeed",
|
||||
};
|
||||
|
||||
export const getMonitorPath = (type: MonitorType): string => MonitorPaths[type];
|
||||
|
||||
// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
|
||||
// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
|
||||
// however far away the agent is. `unknown` is excluded because no request is made for it.
|
||||
@@ -134,6 +150,8 @@ export interface Monitor {
|
||||
geoCheckLocations?: GeoContinent[];
|
||||
geoCheckInterval?: number;
|
||||
dockerLogsEnabled?: boolean;
|
||||
dockerAlertOnStopped?: boolean;
|
||||
dockerAlertOnUnhealthy?: boolean;
|
||||
dockerTlsCa?: string;
|
||||
dockerTlsCert?: string;
|
||||
// EncryptionService ciphertext. Normally excluded
|
||||
@@ -222,3 +240,8 @@ export type GamesMap = Record<string, Game>;
|
||||
export type MonitorScheduleFields = Pick<Monitor, "id" | "type" | "isActive" | "interval" | "geoCheckEnabled" | "geoCheckInterval">;
|
||||
|
||||
export type DockerContainerLogsResult = DockerLogPage;
|
||||
|
||||
export const HardwareMetricKeys = ["cpu", "memory", "disk", "temp"] as const;
|
||||
export type HardwareMetricKey = (typeof HardwareMetricKeys)[number];
|
||||
export type HardwareBreaches = Record<HardwareMetricKey, boolean>;
|
||||
export type HardwareCounters = Record<HardwareMetricKey, number>;
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { HardwareMetricKeys, type HardwareBreaches, type HardwareMetricKey, type Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
import type {
|
||||
NotificationMessage,
|
||||
@@ -8,15 +7,12 @@ import type {
|
||||
ThresholdBreach,
|
||||
NotificationContent,
|
||||
} from "@/domain/notifications/notification.type.js";
|
||||
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
||||
import type { Check } from "@/domain/checks/check.type.js";
|
||||
|
||||
export interface INotificationMessageBuilder {
|
||||
buildMessage(
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
decision: MonitorActionDecision,
|
||||
clientHost: string
|
||||
): NotificationMessage;
|
||||
extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[];
|
||||
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage;
|
||||
buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "NotificationMessageBuilder";
|
||||
@@ -24,15 +20,10 @@ const SERVICE_NAME = "NotificationMessageBuilder";
|
||||
export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
buildMessage(
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
decision: MonitorActionDecision,
|
||||
clientHost: string
|
||||
): NotificationMessage {
|
||||
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage {
|
||||
const type = this.determineNotificationType(decision, monitor);
|
||||
const severity = this.determineSeverity(type);
|
||||
const content = this.buildContent(type, monitor, monitorStatusResponse);
|
||||
const content = this.buildContent(type, monitor, check, decision);
|
||||
|
||||
return {
|
||||
type,
|
||||
@@ -48,7 +39,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
clientHost,
|
||||
metadata: {
|
||||
teamId: monitor.teamId,
|
||||
notificationReason: decision.notificationReason || "status_change",
|
||||
notificationReason: decision.notificationReason,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -59,32 +50,29 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
return "monitor_down";
|
||||
}
|
||||
|
||||
// Threshold breach (only if not down)
|
||||
if (decision.notificationReason === "threshold_breach") {
|
||||
return "threshold_breach";
|
||||
const isDocker = monitor.type === "docker";
|
||||
switch (decision.notificationReason) {
|
||||
case "threshold_breach":
|
||||
return isDocker ? "container_breach" : "threshold_breach";
|
||||
case "threshold_resolved":
|
||||
return isDocker ? "container_resolved" : "threshold_resolved";
|
||||
default:
|
||||
return "monitor_up";
|
||||
}
|
||||
|
||||
// Recovery from threshold breach (only for hardware monitors)
|
||||
if (decision.notificationReason === "status_change" && monitor.status === "up" && monitor.type === "hardware") {
|
||||
return "threshold_resolved";
|
||||
}
|
||||
|
||||
// Standard recovery (up)
|
||||
if (monitor.status === "up") {
|
||||
return "monitor_up";
|
||||
}
|
||||
|
||||
// Default to monitor_up for any other case
|
||||
return "monitor_up";
|
||||
}
|
||||
|
||||
private determineSeverity(type: NotificationType): NotificationSeverity {
|
||||
switch (type) {
|
||||
case "monitor_up":
|
||||
return "success";
|
||||
case "monitor_down":
|
||||
return "critical";
|
||||
case "container_breach":
|
||||
return "warning";
|
||||
case "threshold_breach":
|
||||
return "warning";
|
||||
case "monitor_up":
|
||||
case "container_resolved":
|
||||
return "success";
|
||||
case "threshold_resolved":
|
||||
return "success";
|
||||
case "test":
|
||||
@@ -94,34 +82,38 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
private buildContent(type: NotificationType, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent {
|
||||
private buildContent(type: NotificationType, monitor: Monitor, check: Check, decision: MonitorActionDecision): NotificationContent {
|
||||
switch (type) {
|
||||
case "monitor_down":
|
||||
return this.buildMonitorDownContent(monitor, monitorStatusResponse);
|
||||
return this.buildMonitorDownContent(monitor, check);
|
||||
case "monitor_up":
|
||||
return this.buildMonitorUpContent(monitor);
|
||||
case "threshold_breach":
|
||||
return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<HardwareStatusPayload>);
|
||||
return this.buildThresholdBreachContent(monitor, check, decision.thresholdBreaches);
|
||||
case "threshold_resolved":
|
||||
return this.buildThresholdResolvedContent(monitor);
|
||||
case "container_breach":
|
||||
return this.buildContainerBreachContent(monitor, check);
|
||||
case "container_resolved":
|
||||
return this.buildContainersRecoveredContent(monitor);
|
||||
default:
|
||||
return this.buildDefaultContent(monitor);
|
||||
}
|
||||
}
|
||||
|
||||
private buildMonitorDownContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent {
|
||||
private buildMonitorDownContent(monitor: Monitor, check: Check): NotificationContent {
|
||||
const title = `Monitor Down: ${monitor.name}`;
|
||||
const summary = `Monitor "${monitor.name}" is currently down and unreachable.`;
|
||||
const details = [`URL: ${monitor.url}`, `Status: Down`, `Type: ${monitor.type}`];
|
||||
|
||||
// Add response code if available
|
||||
if (monitorStatusResponse.code) {
|
||||
details.push(`Response Code: ${monitorStatusResponse.code}`);
|
||||
if (check.statusCode) {
|
||||
details.push(`Response Code: ${check.statusCode}`);
|
||||
}
|
||||
|
||||
// Add error message if available
|
||||
if (monitorStatusResponse.message) {
|
||||
details.push(`Error: ${monitorStatusResponse.message}`);
|
||||
if (check.message) {
|
||||
details.push(`Error: ${check.message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -145,12 +137,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
};
|
||||
}
|
||||
|
||||
private buildThresholdBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<HardwareStatusPayload>): NotificationContent {
|
||||
private buildThresholdBreachContent(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): NotificationContent {
|
||||
const title = `Threshold Exceeded: ${monitor.name}`;
|
||||
const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`;
|
||||
const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`];
|
||||
|
||||
const thresholds = this.extractThresholdBreaches(monitor, monitorStatusResponse);
|
||||
const thresholds = this.extractThresholdBreaches(monitor, check, thresholdBreaches);
|
||||
|
||||
return {
|
||||
title,
|
||||
@@ -174,6 +166,21 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
};
|
||||
}
|
||||
|
||||
private buildContainerBreachContent(monitor: Monitor, check: Check): NotificationContent {
|
||||
const lines = this.describeContainerBreaches(monitor, check);
|
||||
const title = `Container Alert: ${monitor.name}`;
|
||||
const summary = `${lines.length} container(s) on "${monitor.name}" need attention.`;
|
||||
const details = [`URL: ${monitor.url}`, `Type: ${monitor.type}`, ...lines];
|
||||
return { title, summary, details, timestamp: new Date() };
|
||||
}
|
||||
|
||||
private buildContainersRecoveredContent(monitor: Monitor): NotificationContent {
|
||||
const title = `Containers Recovered: ${monitor.name}`;
|
||||
const summary = `All containers on "${monitor.name}" are back to normal.`;
|
||||
const details = [`URL: ${monitor.url}`, `Status: Up`, `Type: ${monitor.type}`];
|
||||
return { title, summary, details, timestamp: new Date() };
|
||||
}
|
||||
|
||||
private buildDefaultContent(monitor: Monitor): NotificationContent {
|
||||
return {
|
||||
title: `Monitor: ${monitor.name}`,
|
||||
@@ -183,93 +190,57 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
};
|
||||
}
|
||||
|
||||
public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<HardwareStatusPayload>): ThresholdBreach[] {
|
||||
private describeBreach(metric: HardwareMetricKey, monitor: Monitor, check: Check): ThresholdBreach {
|
||||
switch (metric) {
|
||||
case "cpu": {
|
||||
const currentValue = (check.cpu?.usage_percent ?? 0) * 100;
|
||||
return { metric, currentValue, threshold: monitor.cpuAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
|
||||
}
|
||||
case "memory": {
|
||||
const currentValue = (check.memory?.usage_percent ?? 0) * 100;
|
||||
return { metric, currentValue, threshold: monitor.memoryAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
|
||||
}
|
||||
case "disk": {
|
||||
const currentValue = Math.max(0, ...(check.disk ?? []).map((disk) => disk?.usage_percent ?? 0)) * 100;
|
||||
return { metric, currentValue, threshold: monitor.diskAlertThreshold, unit: "%", formattedValue: `${currentValue.toFixed(1)}%` };
|
||||
}
|
||||
case "temp": {
|
||||
const currentValue = Math.max(0, ...(check.cpu?.temperature ?? []));
|
||||
return { metric, currentValue, threshold: monitor.tempAlertThreshold, unit: "°C", formattedValue: `${currentValue.toFixed(1)}°C` };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private extractThresholdBreaches(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): ThresholdBreach[] {
|
||||
const breaches: ThresholdBreach[] = [];
|
||||
|
||||
// Check if this is a hardware monitor with threshold data
|
||||
if (monitor.type !== "hardware" || !monitorStatusResponse.payload || typeof monitorStatusResponse.payload === "string") {
|
||||
if (monitor.type !== "hardware" || thresholdBreaches === undefined) {
|
||||
return breaches;
|
||||
}
|
||||
|
||||
// Cast to HardwareStatusPayload type
|
||||
const payload = monitorStatusResponse.payload;
|
||||
const hardware = payload.data;
|
||||
|
||||
if (!hardware) {
|
||||
return breaches;
|
||||
}
|
||||
|
||||
// Note: usage_percent values in hardware payload are decimals (0-1)
|
||||
if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && hardware.cpu?.usage_percent !== undefined) {
|
||||
const cpuUsageDecimal = hardware.cpu.usage_percent;
|
||||
const cpuPercent = cpuUsageDecimal * 100;
|
||||
const threshold = monitor.cpuAlertThreshold;
|
||||
if (cpuPercent > threshold) {
|
||||
breaches.push({
|
||||
metric: "cpu",
|
||||
currentValue: cpuPercent,
|
||||
threshold,
|
||||
unit: "%",
|
||||
formattedValue: `${cpuPercent.toFixed(1)}%`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Memory threshold breach
|
||||
if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && hardware.memory?.usage_percent !== undefined) {
|
||||
const memoryUsageDecimal = hardware.memory.usage_percent;
|
||||
const memoryPercent = memoryUsageDecimal * 100;
|
||||
const threshold = monitor.memoryAlertThreshold;
|
||||
if (memoryPercent > threshold) {
|
||||
breaches.push({
|
||||
metric: "memory",
|
||||
currentValue: memoryPercent,
|
||||
threshold,
|
||||
unit: "%",
|
||||
formattedValue: `${memoryPercent.toFixed(1)}%`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Disk threshold breach
|
||||
if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(hardware.disk)) {
|
||||
// Find the highest disk usage
|
||||
let maxDiskUsageDecimal = 0;
|
||||
for (const disk of hardware.disk) {
|
||||
if (disk.usage_percent !== undefined && disk.usage_percent > maxDiskUsageDecimal) {
|
||||
maxDiskUsageDecimal = disk.usage_percent;
|
||||
}
|
||||
}
|
||||
const maxDiskPercent = maxDiskUsageDecimal * 100;
|
||||
const threshold = monitor.diskAlertThreshold;
|
||||
if (maxDiskPercent > threshold) {
|
||||
breaches.push({
|
||||
metric: "disk",
|
||||
currentValue: maxDiskPercent,
|
||||
threshold,
|
||||
unit: "%",
|
||||
formattedValue: `${maxDiskPercent.toFixed(1)}%`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Temperature threshold breach
|
||||
if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && hardware.cpu?.temperature) {
|
||||
// Temperature is an array in cpu.temperature
|
||||
const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature];
|
||||
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
|
||||
const threshold = monitor.tempAlertThreshold;
|
||||
if (maxTemp > threshold) {
|
||||
breaches.push({
|
||||
metric: "temp",
|
||||
currentValue: maxTemp,
|
||||
threshold,
|
||||
unit: "°C",
|
||||
formattedValue: `${maxTemp.toFixed(1)}°C`,
|
||||
});
|
||||
}
|
||||
for (const metric of HardwareMetricKeys) {
|
||||
if (!thresholdBreaches[metric]) continue;
|
||||
breaches.push(this.describeBreach(metric, monitor, check));
|
||||
}
|
||||
|
||||
return breaches;
|
||||
}
|
||||
|
||||
private describeContainerBreaches(monitor: Monitor, check: Check): string[] {
|
||||
return findContainerBreaches(monitor, check.containers ?? []).map(describeContainerBreach);
|
||||
}
|
||||
|
||||
public buildThresholdBreachMessage(monitor: Monitor, check: Check, thresholdBreaches: HardwareBreaches | undefined): string {
|
||||
if (monitor.type === "docker") {
|
||||
const lines = this.describeContainerBreaches(monitor, check);
|
||||
return lines.length > 0 ? lines.join(", ") : "Container alert";
|
||||
}
|
||||
|
||||
const breaches = this.extractThresholdBreaches(monitor, check, thresholdBreaches);
|
||||
if (breaches.length === 0) {
|
||||
return "Threshold breach detected";
|
||||
}
|
||||
return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", ");
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import type { MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
|
||||
@@ -10,6 +9,7 @@ import type { ISettingsService } from "@/domain/app-settings/app-settings.servic
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
|
||||
import type { NotificationChannel } from "@/domain/notifications/notification.type.js";
|
||||
import type { Check } from "@/domain/checks/check.type.js";
|
||||
|
||||
export type NotificationProviderRegistry = Record<NotificationChannel, INotificationProvider>;
|
||||
|
||||
@@ -19,7 +19,7 @@ export interface INotificationsService {
|
||||
findNotificationsByTeamId: (teamId: string) => Promise<Notification[]>;
|
||||
updateById(id: string, teamId: string, updateData: Partial<Notification>): Promise<Notification>;
|
||||
deleteById: (id: string, teamId: string) => Promise<Notification>;
|
||||
handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise<boolean>;
|
||||
handleNotifications: (monitor: Monitor, check: Check, decision: MonitorActionDecision) => Promise<boolean>;
|
||||
|
||||
sendTestNotification: (notification: Partial<Notification>) => Promise<boolean>;
|
||||
testAllNotifications: (notificationIds: string[]) => Promise<boolean>;
|
||||
@@ -63,7 +63,7 @@ export class NotificationsService implements INotificationsService {
|
||||
private send = async (
|
||||
notification: Notification,
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
check: Check,
|
||||
decision: MonitorActionDecision,
|
||||
notificationMessage: NotificationMessage | undefined
|
||||
): Promise<boolean> => {
|
||||
@@ -89,16 +89,16 @@ export class NotificationsService implements INotificationsService {
|
||||
return await provider.sendMessage(notification, notificationMessage);
|
||||
};
|
||||
|
||||
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
|
||||
private sendNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => {
|
||||
const notificationIds = monitor.notifications ?? [];
|
||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||
|
||||
// Build notification message once for all notifications
|
||||
const settings = this.settingsService.getSettings();
|
||||
const clientHost = settings.clientHost || "Host not defined";
|
||||
const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost);
|
||||
const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, check, decision, clientHost);
|
||||
|
||||
const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage));
|
||||
const tasks = notifications.map((notification) => this.send(notification, monitor, check, decision, notificationMessage));
|
||||
|
||||
const outcomes = await Promise.all(tasks);
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
@@ -114,13 +114,13 @@ export class NotificationsService implements INotificationsService {
|
||||
return succeeded === notifications.length;
|
||||
};
|
||||
|
||||
handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
|
||||
handleNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => {
|
||||
if (!decision.shouldSendNotification) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Send notifications based on decision
|
||||
return await this.sendNotifications(monitor, monitorStatusResponse, decision);
|
||||
return await this.sendNotifications(monitor, check, decision);
|
||||
};
|
||||
|
||||
sendTestNotification = async (notification: Partial<Notification>) => {
|
||||
@@ -148,7 +148,17 @@ export class NotificationsService implements INotificationsService {
|
||||
|
||||
testAllNotifications = async (notificationIds: string[]) => {
|
||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||
const tasks = notifications.map((notification) => this.sendTestNotification(notification));
|
||||
const tasks = notifications.map((notification) =>
|
||||
this.sendTestNotification(notification).catch((error: unknown) => {
|
||||
this.logger.warn({
|
||||
message: `Test notification failed: ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "testAllNotifications",
|
||||
details: { notificationId: notification.id, type: notification.type },
|
||||
});
|
||||
return false;
|
||||
})
|
||||
);
|
||||
const outcomes = await Promise.all(tasks);
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
const failed = outcomes.length - succeeded;
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import type { HardwareMetricKey } from "@/domain/monitors/monitor.type.js";
|
||||
import { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
|
||||
export const NotificationChannels = [
|
||||
"email",
|
||||
"slack",
|
||||
@@ -71,7 +74,8 @@ export interface AlertDiscordPayload {
|
||||
* Part of notification system unification effort
|
||||
*/
|
||||
|
||||
export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test";
|
||||
export type NotificationType =
|
||||
"monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "container_breach" | "container_resolved" | "test";
|
||||
|
||||
export type NotificationSeverity = "critical" | "warning" | "info" | "success";
|
||||
|
||||
@@ -84,7 +88,7 @@ export interface MonitorInfo {
|
||||
}
|
||||
|
||||
export interface ThresholdBreach {
|
||||
metric: "cpu" | "memory" | "disk" | "temp";
|
||||
metric: HardwareMetricKey;
|
||||
currentValue: number;
|
||||
threshold: number;
|
||||
unit: string;
|
||||
@@ -116,6 +120,6 @@ export interface NotificationMessage {
|
||||
clientHost: string;
|
||||
metadata: {
|
||||
teamId: string;
|
||||
notificationReason: string;
|
||||
notificationReason: MonitorActionDecision["notificationReason"];
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
const SERVICE_NAME = "EmailProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { buildTestEmail } from "@/domain/notifications/providers/utils.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
@@ -40,11 +41,13 @@ export class EmailProvider extends NotificationProvider {
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: "Email test alert failed",
|
||||
message: `Email test alert failed: ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendTestAlert",
|
||||
details: error instanceof AppError ? error.details : undefined,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
if (error instanceof AppError) throw error;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -73,9 +76,10 @@ export class EmailProvider extends NotificationProvider {
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.warn({
|
||||
message: "Email notification failed",
|
||||
message: `Email notification failed: ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendMessage",
|
||||
details: error instanceof AppError ? error.details : undefined,
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
return false;
|
||||
@@ -92,6 +96,10 @@ export class EmailProvider extends NotificationProvider {
|
||||
return `Monitor ${message.monitor.name} threshold exceeded`;
|
||||
case "threshold_resolved":
|
||||
return `Monitor ${message.monitor.name} thresholds resolved`;
|
||||
case "container_breach":
|
||||
return `Monitor ${message.monitor.name} container alert`;
|
||||
case "container_resolved":
|
||||
return `Monitor ${message.monitor.name} containers recovered`;
|
||||
default:
|
||||
return `Alert: ${message.monitor.name}`;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import got from "got";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { AlertMatrixPayload, Notification } from "@/domain/notifications/notification.type.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
export class MatrixProvider extends NotificationProvider {
|
||||
@@ -172,10 +172,10 @@ export class MatrixProvider extends NotificationProvider {
|
||||
|
||||
// Incident link (if incident exists)
|
||||
if (message.content.incident) {
|
||||
const incidentUrl = `${message.clientHost}/infrastructure/${message.monitor.id}`;
|
||||
const url = incidentUrl(message);
|
||||
plainLines.push("");
|
||||
plainLines.push(`View Incident: ${incidentUrl}`);
|
||||
htmlLines.push(`<p><a href="${this.escapeHtml(incidentUrl)}">View Incident</a></p>`);
|
||||
plainLines.push(`View Incident: ${url}`);
|
||||
htmlLines.push(`<p><a href="${this.escapeHtml(url)}">View Incident</a></p>`);
|
||||
}
|
||||
|
||||
// Footer with timestamp
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "NtfyProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class NtfyProvider extends NotificationProvider {
|
||||
@@ -142,7 +142,7 @@ export class NtfyProvider extends NotificationProvider {
|
||||
}
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("", `${message.clientHost}/infrastructure/${message.monitor.id}`);
|
||||
lines.push("", incidentUrl(message));
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -75,7 +75,8 @@ export class PagerDutyProvider extends NotificationProvider {
|
||||
|
||||
private buildPagerDutyPayload(notification: Notification, message: NotificationMessage): AlertPagerDutyPayload {
|
||||
// Map our notification type to PagerDuty event_action
|
||||
const eventAction = message.type === "monitor_up" || message.type === "threshold_resolved" ? "resolve" : "trigger";
|
||||
const eventAction =
|
||||
message.type === "monitor_up" || message.type === "threshold_resolved" || message.type === "container_resolved" ? "resolve" : "trigger";
|
||||
|
||||
// Map severity to PagerDuty severity levels
|
||||
const severityMap: Record<string, string> = {
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "PushoverProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class PushoverProvider extends NotificationProvider {
|
||||
@@ -101,7 +101,7 @@ export class PushoverProvider extends NotificationProvider {
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("");
|
||||
lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`);
|
||||
lines.push(`View Incident: ${incidentUrl(message)}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const SERVICE_NAME = "RocketChatProvider";
|
||||
import { supportsUptimeDetails, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { getMonitorPath, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import type { Notification, NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
@@ -139,19 +139,7 @@ export class RocketChatProvider extends NotificationProvider {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (message.monitor.type === "hardware") {
|
||||
return `${clientHost}/infrastructure/${message.monitor.id}`;
|
||||
}
|
||||
|
||||
if (message.monitor.type === "pagespeed") {
|
||||
return `${clientHost}/pagespeed/${message.monitor.id}`;
|
||||
}
|
||||
|
||||
if (supportsUptimeDetails(message.monitor.type as MonitorType)) {
|
||||
return `${clientHost}/uptime/${message.monitor.id}`;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
return `${clientHost}/${getMonitorPath(message.monitor.type as MonitorType)}/${message.monitor.id}`;
|
||||
}
|
||||
|
||||
private severityColor(severity: NotificationMessage["severity"]): string {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const SERVICE_NAME = "SignalgridProvider";
|
||||
import type { Notification, NotificationMessage, NotificationSeverity } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class SignalgridProvider extends NotificationProvider {
|
||||
@@ -115,7 +115,7 @@ export class SignalgridProvider extends NotificationProvider {
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("");
|
||||
lines.push(`View Incident: ${message.clientHost}/incidents/${message.monitor.id}`);
|
||||
lines.push(`View Incident: ${incidentUrl(message)}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "SlackProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import got, { HTTPError } from "got";
|
||||
|
||||
export class SlackProvider extends NotificationProvider {
|
||||
@@ -161,7 +161,7 @@ export class SlackProvider extends NotificationProvider {
|
||||
text: "View Incident",
|
||||
emoji: true,
|
||||
},
|
||||
url: `${message.clientHost}/infrastructure/${message.monitor.id}`,
|
||||
url: incidentUrl(message),
|
||||
style: "primary",
|
||||
},
|
||||
],
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "TeamsProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import got, { HTTPError } from "got";
|
||||
|
||||
// Types for Adaptive Card elements
|
||||
@@ -254,7 +254,7 @@ export class TeamsProvider extends NotificationProvider {
|
||||
actions.push({
|
||||
type: "Action.OpenUrl",
|
||||
title: "View Incident",
|
||||
url: `${message.clientHost}/incidents/${message.content.incident.id}`,
|
||||
url: incidentUrl(message),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "TelegramProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class TelegramProvider extends NotificationProvider {
|
||||
@@ -101,7 +101,7 @@ export class TelegramProvider extends NotificationProvider {
|
||||
|
||||
if (message.content.incident) {
|
||||
lines.push("");
|
||||
lines.push(`<a href="${message.clientHost}/incidents/${message.monitor.id}">View Incident</a>`);
|
||||
lines.push(`<a href="${incidentUrl(message)}">View Incident</a>`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { IEmailService } from "@/service/emailService.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
|
||||
export const incidentUrl = (message: NotificationMessage): string => `${message.clientHost}/incidents/${message.monitor.id}`;
|
||||
|
||||
export const buildTestEmail = async (emailService: IEmailService) => {
|
||||
const context = { testName: "Monitoring System" };
|
||||
const html = await emailService.buildEmail("testEmailTemplate", context);
|
||||
|
||||
@@ -2,7 +2,7 @@ const SERVICE_NAME = "WebhookProvider";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import { NotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { getTestMessage } from "@/domain/notifications/providers/utils.js";
|
||||
import { getTestMessage, incidentUrl } from "@/domain/notifications/providers/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class WebhookProvider extends NotificationProvider {
|
||||
@@ -74,7 +74,7 @@ export class WebhookProvider extends NotificationProvider {
|
||||
|
||||
// Incident link
|
||||
if (message.content.incident) {
|
||||
lines.push(`[View Incident](${message.clientHost}/infrastructure/${message.monitor.id})`);
|
||||
lines.push(`[View Incident](${incidentUrl(message)})`);
|
||||
}
|
||||
|
||||
// Return webhook payload with both text and structured data
|
||||
|
||||
@@ -12,6 +12,23 @@ const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const SERVICE_NAME = "EmailService";
|
||||
|
||||
const SMTP_FAILURE_REASONS: Record<string, string> = {
|
||||
EAUTH: "rejected the login. Check the user name and password",
|
||||
ENOAUTH: "requires a login. Set the user name and password",
|
||||
EDNS: "could not be resolved. Check the host name",
|
||||
ECONNECTION: "refused the connection. Check the host and port",
|
||||
ESOCKET: "refused the connection. Check the host and port",
|
||||
ETIMEDOUT: "did not respond. Check the port and the secure setting",
|
||||
ETLS: "failed the TLS handshake. Check the secure and TLS settings",
|
||||
EPROTOCOL: "sent an invalid response. Check the port and the secure setting",
|
||||
};
|
||||
|
||||
const describeSmtpFailure = (error: unknown, host: string, port: number): string => {
|
||||
const code = typeof error === "object" && error !== null && "code" in error ? String((error as { code: unknown }).code) : undefined;
|
||||
const reason = (code !== undefined && SMTP_FAILURE_REASONS[code]) || "failed verification";
|
||||
return `SMTP host ${host}:${port} ${reason}.`;
|
||||
};
|
||||
type MjmlFn = typeof mjml2html;
|
||||
type FileSystem = typeof fs;
|
||||
type PathModule = typeof path;
|
||||
@@ -123,6 +140,15 @@ export class EmailService implements IEmailService {
|
||||
systemEmailRejectUnauthorized,
|
||||
} = config;
|
||||
|
||||
if (!systemEmailHost) {
|
||||
throw new AppError({
|
||||
message: "Email is not configured. Set the system email host in settings.",
|
||||
status: 400,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendEmail",
|
||||
});
|
||||
}
|
||||
|
||||
const emailConfig = {
|
||||
host: systemEmailHost,
|
||||
port: Number(systemEmailPort),
|
||||
@@ -146,17 +172,20 @@ export class EmailService implements IEmailService {
|
||||
try {
|
||||
await this.transporter.verify();
|
||||
} catch (error: unknown) {
|
||||
const cause = error instanceof Error ? error.message : "Unknown error";
|
||||
this.logger.warn({
|
||||
message: "Email transporter verification failed",
|
||||
message: `Email transporter verification failed: ${cause}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "verifyTransporter",
|
||||
details: { host: systemEmailHost, port: emailConfig.port },
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
throw new AppError({
|
||||
message: "Email transporter verification failed",
|
||||
message: describeSmtpFailure(error, systemEmailHost, emailConfig.port),
|
||||
status: 502,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendEmail",
|
||||
details: { cause: error instanceof Error ? error.message : "Unknown error" },
|
||||
details: { cause },
|
||||
});
|
||||
}
|
||||
|
||||
@@ -180,6 +209,7 @@ export class EmailService implements IEmailService {
|
||||
});
|
||||
throw new AppError({
|
||||
message: "Failed to send email",
|
||||
status: 502,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendEmail",
|
||||
details: { cause: error instanceof Error ? error.message : "Unknown error" },
|
||||
|
||||
@@ -42,6 +42,8 @@ const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
|
||||
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
|
||||
const DOCKER_TLS_DEFAULT_PORT = 2376;
|
||||
const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]);
|
||||
const EXITED_STATUS = /^Exited \((\d+)\)/;
|
||||
const HEALTH_STATUS = /\((healthy|unhealthy|health: starting)\)$/;
|
||||
|
||||
export interface DockerError extends Error {
|
||||
statusCode?: number;
|
||||
@@ -140,6 +142,17 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none";
|
||||
}
|
||||
|
||||
private exitCodeFromStatus = (status: string): number | undefined => {
|
||||
const match = EXITED_STATUS.exec(status);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
};
|
||||
|
||||
private healthFromStatus = (status: string): DockerHealthStatus => {
|
||||
const match = HEALTH_STATUS.exec(status);
|
||||
if (!match) return "none";
|
||||
return match[1] === "health: starting" ? "starting" : (match[1] as DockerHealthStatus);
|
||||
};
|
||||
|
||||
private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary {
|
||||
return {
|
||||
total: containers.length,
|
||||
@@ -273,7 +286,10 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
info.health = this.toHealthStatus(inspectResult.value.State?.Health?.Status);
|
||||
info.ports = this.toPorts(inspectResult.value.NetworkSettings?.Ports);
|
||||
info.mounts = this.toMounts(inspectResult.value.Mounts);
|
||||
info.exitCode = inspectResult.value.State?.ExitCode;
|
||||
} else {
|
||||
info.exitCode = this.exitCodeFromStatus(summary.Status);
|
||||
info.health = this.healthFromStatus(summary.Status);
|
||||
this.logger.warn({
|
||||
message: `Failed to inspect container ${info.name}`,
|
||||
service: SERVICE_NAME,
|
||||
|
||||
@@ -1,49 +1,27 @@
|
||||
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js";
|
||||
import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js";
|
||||
import type {
|
||||
DockerStatusPayload,
|
||||
GameStatusPayload,
|
||||
GrpcStatusPayload,
|
||||
HardwareStatusPayload,
|
||||
HttpStatusPayload,
|
||||
MonitorStatusResponse,
|
||||
PageSpeedStatusPayload,
|
||||
PingStatusPayload,
|
||||
PortStatusPayload,
|
||||
StatusChangeResult,
|
||||
} from "@/types/network.js";
|
||||
import {
|
||||
HardwareMetricKeys,
|
||||
MonitorStatuses,
|
||||
type HardwareBreaches,
|
||||
type HardwareCounters,
|
||||
type Monitor,
|
||||
type MonitorStatus,
|
||||
} from "@/domain/monitors/monitor.type.js";
|
||||
import type { StatusChangeResult } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import type { HardwareStatusMetrics } from "@/types/network.js";
|
||||
import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js";
|
||||
import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js";
|
||||
import { findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
||||
|
||||
const SERVICE_NAME = "StatusService";
|
||||
const HARDWARE_ALERT_COUNTER_START = 5;
|
||||
const HARDWARE_METRIC_KEYS = ["cpu", "memory", "disk", "temp"] as const;
|
||||
type HardwareMetricKey = (typeof HARDWARE_METRIC_KEYS)[number];
|
||||
type HardwareBreaches = Record<HardwareMetricKey, boolean>;
|
||||
type HardwareCounters = Record<HardwareMetricKey, number>;
|
||||
|
||||
export interface IStatusService {
|
||||
updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise<boolean>;
|
||||
updateMonitorStatus(
|
||||
statusResponse: MonitorStatusResponse<
|
||||
| PingStatusPayload
|
||||
| HttpStatusPayload
|
||||
| PageSpeedStatusPayload
|
||||
| HardwareStatusPayload
|
||||
| DockerStatusPayload
|
||||
| PortStatusPayload
|
||||
| GameStatusPayload
|
||||
| GrpcStatusPayload
|
||||
| undefined
|
||||
>,
|
||||
check: Check,
|
||||
monitor: Monitor
|
||||
): Promise<StatusChangeResult>;
|
||||
updateMonitorStatus(check: Check, monitor: Monitor): Promise<StatusChangeResult>;
|
||||
}
|
||||
|
||||
export class StatusService implements IStatusService {
|
||||
@@ -58,32 +36,19 @@ export class StatusService implements IStatusService {
|
||||
this.monitorStatsRepository = monitorStatsRepository;
|
||||
}
|
||||
|
||||
async updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse) {
|
||||
private tryUpdateRunningStats = async (check: Check, monitor: Monitor) => {
|
||||
try {
|
||||
await this.monitorStatsRepository.updateByMonitorId(monitor.id, {
|
||||
status: networkResponse.status === true,
|
||||
responseTime: networkResponse.responseTime ?? 0,
|
||||
status: check.status === true,
|
||||
responseTime: check.responseTime ?? 0,
|
||||
now: Date.now(),
|
||||
});
|
||||
return true;
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
service: SERVICE_NAME,
|
||||
message: error instanceof Error ? error.message : "Unknown error",
|
||||
method: "updateRunningStats",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private tryUpdateRunningStats = async (monitor: Monitor, statusResponse: MonitorStatusResponse) => {
|
||||
const statsOk = await this.updateRunningStats(monitor, statusResponse);
|
||||
if (!statsOk) {
|
||||
this.logger.warn({
|
||||
service: SERVICE_NAME,
|
||||
method: "updateMonitorStatus",
|
||||
message: `Stats update failed for monitor ${monitor.id}`,
|
||||
message: error instanceof Error ? error.message : "Unknown error",
|
||||
method: "tryUpdateRunningStats",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -135,7 +100,7 @@ export class StatusService implements IStatusService {
|
||||
|
||||
// Update counters: decrement (floored at 0) if breached, reset to start otherwise.
|
||||
const nextCounters = { ...counters };
|
||||
for (const key of HARDWARE_METRIC_KEYS) {
|
||||
for (const key of HardwareMetricKeys) {
|
||||
nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START;
|
||||
}
|
||||
|
||||
@@ -147,8 +112,8 @@ export class StatusService implements IStatusService {
|
||||
if (!reachabilityDown) {
|
||||
// A counter can only reach zero via the decrement path, which only runs when that
|
||||
// metric is currently breaching — so anyCounterZero already implies anyBreached.
|
||||
const anyCounterZero = HARDWARE_METRIC_KEYS.some((k) => nextCounters[k] === 0);
|
||||
const allNormal = HARDWARE_METRIC_KEYS.every((k) => !breaches[k]);
|
||||
const anyCounterZero = HardwareMetricKeys.some((k) => nextCounters[k] === 0);
|
||||
const allNormal = HardwareMetricKeys.every((k) => !breaches[k]);
|
||||
|
||||
if (anyCounterZero && currentStatus !== "breached") {
|
||||
nextStatus = "breached";
|
||||
@@ -162,26 +127,29 @@ export class StatusService implements IStatusService {
|
||||
return { nextStatus, transitioned, breaches, nextCounters };
|
||||
};
|
||||
|
||||
updateMonitorStatus = async (
|
||||
statusResponse: MonitorStatusResponse<
|
||||
| PingStatusPayload
|
||||
| HttpStatusPayload
|
||||
| PageSpeedStatusPayload
|
||||
| HardwareStatusPayload
|
||||
| DockerStatusPayload
|
||||
| PortStatusPayload
|
||||
| GameStatusPayload
|
||||
| GrpcStatusPayload
|
||||
| undefined
|
||||
>,
|
||||
check: Check,
|
||||
monitor: Monitor
|
||||
): Promise<StatusChangeResult> => {
|
||||
private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => {
|
||||
const { currentStatus, reachabilityDown, breaching } = params;
|
||||
let nextStatus: MonitorStatus = currentStatus;
|
||||
let transitioned = false;
|
||||
|
||||
if (!reachabilityDown) {
|
||||
if (breaching && currentStatus !== "breached") {
|
||||
nextStatus = "breached";
|
||||
transitioned = true;
|
||||
} else if (!breaching && currentStatus === "breached") {
|
||||
nextStatus = "up";
|
||||
transitioned = true;
|
||||
}
|
||||
}
|
||||
return { nextStatus, transitioned };
|
||||
};
|
||||
|
||||
updateMonitorStatus = async (check: Check, monitor: Monitor): Promise<StatusChangeResult> => {
|
||||
try {
|
||||
const { status, code } = statusResponse;
|
||||
const { status } = check;
|
||||
|
||||
// Update running stats
|
||||
await this.tryUpdateRunningStats(monitor, statusResponse);
|
||||
await this.tryUpdateRunningStats(check, monitor);
|
||||
|
||||
const prevStatus = monitor.status;
|
||||
const checkSnapshot = toCheckSnapshot(check);
|
||||
@@ -206,69 +174,62 @@ export class StatusService implements IStatusService {
|
||||
statusChanged = newStatus === "down";
|
||||
}
|
||||
|
||||
// Not enough data points yet — record the check and return
|
||||
if (projectedWindow.length < monitor.statusWindowSize) {
|
||||
const updated = await this.monitorsRepository.updateStatusWindowAndChecks(
|
||||
monitor.id,
|
||||
monitor.teamId,
|
||||
check.status,
|
||||
checkSnapshot,
|
||||
monitor.statusWindowSize,
|
||||
MAX_RECENT_CHECKS,
|
||||
patch
|
||||
);
|
||||
|
||||
return {
|
||||
monitor: updated,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
code,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// First evaluate reachability status changes, which apply to all monitor types
|
||||
// and take precedence over hardware breaches.
|
||||
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
|
||||
if (reachabilityResult.transitioned) {
|
||||
newStatus = reachabilityResult.nextStatus;
|
||||
statusChanged = true;
|
||||
}
|
||||
|
||||
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
|
||||
let thresholdBreaches: HardwareBreaches | undefined;
|
||||
const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined;
|
||||
if (monitor.type === "hardware" && hardwarePayload?.data) {
|
||||
const hardware = this.computeHardwareStatus({
|
||||
currentStatus: newStatus,
|
||||
reachabilityDown: newStatus === "down",
|
||||
metrics: hardwarePayload.data,
|
||||
thresholds: {
|
||||
cpu: monitor.cpuAlertThreshold,
|
||||
memory: monitor.memoryAlertThreshold,
|
||||
disk: monitor.diskAlertThreshold,
|
||||
temp: monitor.tempAlertThreshold,
|
||||
},
|
||||
counters: {
|
||||
cpu: monitor.cpuAlertCounter,
|
||||
memory: monitor.memoryAlertCounter,
|
||||
disk: monitor.diskAlertCounter,
|
||||
temp: monitor.tempAlertCounter,
|
||||
},
|
||||
});
|
||||
|
||||
patch.cpuAlertCounter = hardware.nextCounters.cpu;
|
||||
patch.memoryAlertCounter = hardware.nextCounters.memory;
|
||||
patch.diskAlertCounter = hardware.nextCounters.disk;
|
||||
patch.tempAlertCounter = hardware.nextCounters.temp;
|
||||
thresholdBreaches = hardware.breaches;
|
||||
if (hardware.transitioned) {
|
||||
newStatus = hardware.nextStatus;
|
||||
if (projectedWindow.length >= monitor.statusWindowSize) {
|
||||
// First evaluate reachability status changes, which apply to all monitor types
|
||||
// and take precedence over hardware breaches.
|
||||
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
|
||||
if (reachabilityResult.transitioned) {
|
||||
newStatus = reachabilityResult.nextStatus;
|
||||
statusChanged = true;
|
||||
}
|
||||
}
|
||||
|
||||
patch.status = newStatus;
|
||||
// Evaluate hardware threshold breaches
|
||||
if (monitor.type === "hardware") {
|
||||
const hardware = this.computeHardwareStatus({
|
||||
currentStatus: newStatus,
|
||||
reachabilityDown: newStatus === "down",
|
||||
metrics: check,
|
||||
thresholds: {
|
||||
cpu: monitor.cpuAlertThreshold,
|
||||
memory: monitor.memoryAlertThreshold,
|
||||
disk: monitor.diskAlertThreshold,
|
||||
temp: monitor.tempAlertThreshold,
|
||||
},
|
||||
counters: {
|
||||
cpu: monitor.cpuAlertCounter,
|
||||
memory: monitor.memoryAlertCounter,
|
||||
disk: monitor.diskAlertCounter,
|
||||
temp: monitor.tempAlertCounter,
|
||||
},
|
||||
});
|
||||
|
||||
patch.cpuAlertCounter = hardware.nextCounters.cpu;
|
||||
patch.memoryAlertCounter = hardware.nextCounters.memory;
|
||||
patch.diskAlertCounter = hardware.nextCounters.disk;
|
||||
patch.tempAlertCounter = hardware.nextCounters.temp;
|
||||
thresholdBreaches = hardware.breaches;
|
||||
if (hardware.transitioned) {
|
||||
newStatus = hardware.nextStatus;
|
||||
statusChanged = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (monitor.type === "docker" && check.containers) {
|
||||
const breaches = findContainerBreaches(monitor, check.containers);
|
||||
const docker = this.computeDockerStatus({
|
||||
currentStatus: newStatus,
|
||||
reachabilityDown: newStatus === "down",
|
||||
breaching: breaches.length > 0,
|
||||
});
|
||||
if (docker.transitioned) {
|
||||
newStatus = docker.nextStatus;
|
||||
statusChanged = true;
|
||||
}
|
||||
}
|
||||
patch.status = newStatus;
|
||||
}
|
||||
|
||||
// Single atomic write: push arrays + set status/counters
|
||||
const updated = await this.monitorsRepository.updateStatusWindowAndChecks(
|
||||
@@ -285,8 +246,6 @@ export class StatusService implements IStatusService {
|
||||
monitor: updated,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
code,
|
||||
timestamp: new Date().getTime(),
|
||||
thresholdBreaches,
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
|
||||
@@ -10,7 +10,7 @@ import type {
|
||||
ILighthouseAudit,
|
||||
} from "@/domain/checks/check.type.js";
|
||||
import { DockerContainerInfo, DockerContainerLogs, DockerContainerSummary } from "@/domain/docker/docker.type.js";
|
||||
import type { DnsRecordType, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import type { DnsRecordType, HardwareBreaches, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
|
||||
import type { QueryResult } from "gamedig";
|
||||
|
||||
@@ -182,14 +182,7 @@ export type StatusChangeResult = {
|
||||
monitor: Monitor;
|
||||
statusChanged: boolean;
|
||||
prevStatus: MonitorStatus;
|
||||
code: number;
|
||||
timestamp: number;
|
||||
thresholdBreaches?: {
|
||||
cpu: boolean;
|
||||
memory: boolean;
|
||||
disk: boolean;
|
||||
temp: boolean;
|
||||
};
|
||||
thresholdBreaches?: HardwareBreaches;
|
||||
};
|
||||
|
||||
export type MonitorStatusResponseOverrides<T> = Partial<Omit<MonitorStatusResponse<T>, "monitorId" | "teamId" | "type">>;
|
||||
|
||||
+12
-15
@@ -60,21 +60,18 @@ class Logger implements ILogger {
|
||||
if (formattedDetails) msg += ` (details: ${formattedDetails})`;
|
||||
|
||||
if (typeof stack === "string") {
|
||||
const stackTrace = stack
|
||||
.split("\n")
|
||||
.slice(1) // Remove first line (error message)
|
||||
.map((line: string) => {
|
||||
const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/);
|
||||
if (match) {
|
||||
return {
|
||||
function: match[1],
|
||||
file: match[2],
|
||||
line: parseInt(match[3] ?? "0", 10),
|
||||
column: parseInt(match[4] ?? "0", 10),
|
||||
};
|
||||
}
|
||||
return line.trim();
|
||||
});
|
||||
const stackTrace = stack.split("\n").map((line: string) => {
|
||||
const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/);
|
||||
if (match) {
|
||||
return {
|
||||
function: match[1],
|
||||
file: match[2],
|
||||
line: parseInt(match[3] ?? "0", 10),
|
||||
column: parseInt(match[4] ?? "0", 10),
|
||||
};
|
||||
}
|
||||
return line.trim();
|
||||
});
|
||||
msg += ` (stack: ${JSON.stringify(stackTrace, null, 2)})`;
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,6 @@ export class IncidentReactor implements IMonitorReactor {
|
||||
constructor(private incidentService: IIncidentService) {}
|
||||
|
||||
react = async (evaluation: MonitorEvaluation) => {
|
||||
await this.incidentService.handleIncident(evaluation.monitor, evaluation.statusChange.code, evaluation.decision, evaluation.status);
|
||||
await this.incidentService.handleIncident(evaluation.monitor, evaluation.decision, evaluation.check);
|
||||
};
|
||||
}
|
||||
@@ -7,7 +7,6 @@ export class NotificationReactor implements IMonitorReactor {
|
||||
constructor(private notificationService: INotificationsService) {}
|
||||
|
||||
react = async (evaluation: MonitorEvaluation) => {
|
||||
if (!evaluation.decision.shouldSendNotification) return;
|
||||
await this.notificationService.handleNotifications(evaluation.monitor, evaluation.status, evaluation.decision);
|
||||
await this.notificationService.handleNotifications(evaluation.monitor, evaluation.check, evaluation.decision);
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { HardwareBreaches, Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { Job, JobType } from "@/domain/jobs/job.type.js";
|
||||
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
import type { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
|
||||
@@ -12,20 +11,13 @@ export interface MonitorActionDecision {
|
||||
shouldResolveIncident: boolean;
|
||||
shouldSendNotification: boolean;
|
||||
incidentReason: "status_down" | "threshold_breach" | null;
|
||||
notificationReason: "status_change" | "threshold_breach" | null;
|
||||
thresholdBreaches?: {
|
||||
cpu?: boolean;
|
||||
memory?: boolean;
|
||||
disk?: boolean;
|
||||
temp?: boolean;
|
||||
};
|
||||
notificationReason: "status_change" | "threshold_breach" | "threshold_resolved" | null;
|
||||
thresholdBreaches?: HardwareBreaches;
|
||||
}
|
||||
|
||||
export type MonitorEvaluation = {
|
||||
monitor: Monitor;
|
||||
status: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>; // raw result from networkService.requestStatus
|
||||
check: Check;
|
||||
statusChange: StatusChangeResult; // from statusService.updateMonitorStatus
|
||||
decision: MonitorActionDecision;
|
||||
};
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
@@ -21,7 +20,13 @@ import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
|
||||
const SERVICE_NAME = "WorkerPipeline";
|
||||
|
||||
const NO_ACTION: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
// The three entry points: the queue calls the two handlers, the buffer calls ingestChecks.
|
||||
// produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer.
|
||||
export interface IWorkerPipeline {
|
||||
@@ -128,7 +133,7 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
|
||||
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
|
||||
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
|
||||
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
|
||||
const egressStatus = this.isTransportFailure(monitor, status) ? await this.deps.egressService.assessAfterFailure() : null;
|
||||
if (egressStatus !== null) {
|
||||
check.egressStatus = egressStatus;
|
||||
}
|
||||
@@ -171,20 +176,6 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
|
||||
}
|
||||
|
||||
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
|
||||
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
|
||||
try {
|
||||
return await this.deps.egressService.assessAfterFailure();
|
||||
} catch (error: unknown) {
|
||||
this.deps.logger.warn({
|
||||
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "assessEgress",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ****************************************************************
|
||||
// Stage 2: Ingestion
|
||||
// Called by the buffer, checks are inserted and evaluation jobs created
|
||||
@@ -240,8 +231,7 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
|
||||
let current = monitor;
|
||||
for (const check of checks) {
|
||||
const status = this.deps.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.evaluateCheck(status, check, current);
|
||||
const evaluation = await this.evaluateCheck(check, current);
|
||||
await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
if (!leaseHeld) {
|
||||
@@ -253,7 +243,7 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
});
|
||||
return;
|
||||
}
|
||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||
current = evaluation.monitor; // fresh statusWindow/status/counters for the next check
|
||||
}
|
||||
|
||||
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
|
||||
@@ -272,48 +262,34 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
// A check taken while the instance had no outbound connectivity says nothing about the target.
|
||||
// It is left out of the status window, running stats and monitor status entirely, so that no incident
|
||||
// opens for it and no spurious "resolved"/"up" fires once egress returns.
|
||||
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
|
||||
private skipDegradedEgressCheck = (check: Check, monitor: Monitor): MonitorEvaluation => {
|
||||
this.deps.logger.debug({
|
||||
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
|
||||
service: SERVICE_NAME,
|
||||
method: "evaluateCheck",
|
||||
});
|
||||
const decision: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
|
||||
return {
|
||||
monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
|
||||
decision,
|
||||
decision: { ...NO_ACTION },
|
||||
};
|
||||
};
|
||||
|
||||
evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
|
||||
evaluateCheck = async (check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
|
||||
if (check.egressStatus === "degraded") {
|
||||
return this.skipDegradedEgressCheck(status, check, monitor);
|
||||
return this.skipDegradedEgressCheck(check, monitor);
|
||||
}
|
||||
|
||||
const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor);
|
||||
const statusChange = await this.deps.statusService.updateMonitorStatus(check, monitor);
|
||||
|
||||
const decision = this.decide(statusChange);
|
||||
return { monitor: statusChange.monitor, status, check, statusChange, decision };
|
||||
return { monitor: statusChange.monitor, check, decision };
|
||||
};
|
||||
|
||||
private decide = (statusChange: StatusChangeResult): MonitorActionDecision => {
|
||||
const { monitor, statusChanged, prevStatus } = statusChange;
|
||||
const decision: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
const decision: MonitorActionDecision = { ...NO_ACTION };
|
||||
if (!statusChanged) return decision;
|
||||
|
||||
if (monitor.status === "down") {
|
||||
@@ -322,11 +298,16 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
decision.incidentReason = "status_down";
|
||||
decision.notificationReason = "status_change";
|
||||
} else if (monitor.status === "breached") {
|
||||
if (statusChange.thresholdBreaches) decision.thresholdBreaches = statusChange.thresholdBreaches;
|
||||
decision.shouldCreateIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "threshold_breach";
|
||||
decision.notificationReason = "threshold_breach";
|
||||
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
|
||||
} else if (monitor.status === "up" && prevStatus === "breached") {
|
||||
decision.shouldResolveIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.notificationReason = "threshold_resolved";
|
||||
} else if (monitor.status === "up" && prevStatus === "down") {
|
||||
decision.shouldResolveIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.notificationReason = "status_change";
|
||||
|
||||
Reference in new issue
Block a user