diff --git a/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts b/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts index 2d363d6b3..0a1dbf876 100644 --- a/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts +++ b/server/src/service/infrastructure/notificationProviders/INotificationProvider.ts @@ -1,6 +1,12 @@ import type { Monitor, Notification, Alert, MonitorStatusResponse } from "@/types/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; export interface INotificationProvider { - sendAlert: (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => Promise; + sendAlert: ( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => Promise; sendTestAlert(notification: Notification): Promise; } diff --git a/server/src/service/infrastructure/notificationProviders/discord.ts b/server/src/service/infrastructure/notificationProviders/discord.ts index ad1f2f28a..200b8af2c 100644 --- a/server/src/service/infrastructure/notificationProviders/discord.ts +++ b/server/src/service/infrastructure/notificationProviders/discord.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "DiscordProvider"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, buildDiscordBody, getTestMessage } from "@/service/infrastructure/notificationProviders/utils.js"; import got from "got"; @@ -10,15 +11,20 @@ export class DiscordProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildDiscordBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format const { discordPayload } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); return discordPayload; }; - sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent(monitor, monitorStatusResponse, decision); } else { body = buildDiscordBody(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/email.ts b/server/src/service/infrastructure/notificationProviders/email.ts index 32ccf1859..802c44b80 100644 --- a/server/src/service/infrastructure/notificationProviders/email.ts +++ b/server/src/service/infrastructure/notificationProviders/email.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "EmailProvider"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, buildHardwareEmail, buildEmail, buildTestEmail } from "@/service/infrastructure/notificationProviders/utils.js"; export class EmailProvider implements INotificationProvider { @@ -12,21 +13,42 @@ export class EmailProvider implements INotificationProvider { this.logger = logger; } - private buildHardwareEmail = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + private buildHardwareEmail = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + // For status changes (recovery), use standard email format + if (decision.notificationReason === "status_change") { + return await buildEmail(this.emailService, monitor); + } + // For threshold breaches, use hardware alert format const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); const html = buildHardwareEmail(this.emailService, monitor, alertsToSend); return html; }; - async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise { + async sendAlert( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ): Promise { // For grouped notifications (identified by ":" in name), customize subject to indicate multiple services. // Example: "2 services: Service A, Service B" becomes "Alert: 2 services are down" const isGroupedNotification = monitor.name.includes(":"); - const subject = isGroupedNotification ? `Alert: ${monitor.name} are down` : `Monitor ${monitor.name} is down`; + + // Build subject based on notification reason and monitor status + let subject: string; + if (isGroupedNotification) { + subject = `Alert: ${monitor.name} are down`; + } else if (decision.notificationReason === "threshold_breach") { + subject = `Monitor ${monitor.name} threshold breached`; + } else if (monitor.status === "up") { + subject = `Monitor ${monitor.name} is back up`; + } else { + subject = `Monitor ${monitor.name} is down`; + } let html; if (monitor.type === "hardware") { - html = this.buildHardwareEmail(monitor, monitorStatusResponse); + html = await this.buildHardwareEmail(monitor, monitorStatusResponse, decision); } else { html = await buildEmail(this.emailService, monitor); } diff --git a/server/src/service/infrastructure/notificationProviders/matrix.ts b/server/src/service/infrastructure/notificationProviders/matrix.ts index 8914dbf80..5a65a7909 100644 --- a/server/src/service/infrastructure/notificationProviders/matrix.ts +++ b/server/src/service/infrastructure/notificationProviders/matrix.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "MatrixProvider"; import got from "got"; import type { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import type { Notification, Monitor, MonitorStatusResponse } from "@/types/index.js"; import { buildHardwareAlerts, @@ -15,9 +16,19 @@ export class MatrixProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + const body = buildHardwareWebhookBody(clientHost, alertsToSend, monitor); return body; }; @@ -26,12 +37,12 @@ export class MatrixProvider implements INotificationProvider { return body; }; - sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { const { homeserverUrl, accessToken, roomId } = notification; let content; if (monitor.type === "hardware") { - content = this.getHardwareContent(monitor, monitorStatusResponse); + content = this.getHardwareContent("HOST_PLACEHOLDER", monitor, monitorStatusResponse, decision); } else { content = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/pagerduty.ts b/server/src/service/infrastructure/notificationProviders/pagerduty.ts index f65e50c68..8ed4ccc68 100644 --- a/server/src/service/infrastructure/notificationProviders/pagerduty.ts +++ b/server/src/service/infrastructure/notificationProviders/pagerduty.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "PagerDutyProvider"; import got from "got"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { INotificationProvider } from "@/service/index.js"; import { buildHardwareAlerts, @@ -15,9 +16,19 @@ export class PagerDutyProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + const body = buildHardwareWebhookBody(clientHost, alertsToSend, monitor); return body; }; @@ -26,10 +37,15 @@ export class PagerDutyProvider implements INotificationProvider { return body; }; - async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise { + async sendAlert( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ): Promise { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent("HOST_PLACEHOLDER", monitor, monitorStatusResponse, decision); } else { body = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/slack.ts b/server/src/service/infrastructure/notificationProviders/slack.ts index 2aeb442eb..0e7710b3f 100644 --- a/server/src/service/infrastructure/notificationProviders/slack.ts +++ b/server/src/service/infrastructure/notificationProviders/slack.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "SlackProvider"; import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, buildHardwareWebhookBody, @@ -15,9 +16,19 @@ export class SlackProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + const body = buildHardwareWebhookBody(clientHost, alertsToSend, monitor); return body; }; @@ -26,10 +37,15 @@ export class SlackProvider implements INotificationProvider { return body; }; - async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise { + async sendAlert( + notification: Notification, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ): Promise { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent("HOST_PLACEHOLDER", monitor, monitorStatusResponse, decision); } else { body = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationProviders/utils.ts b/server/src/service/infrastructure/notificationProviders/utils.ts index b258ccd52..bb1c928ca 100644 --- a/server/src/service/infrastructure/notificationProviders/utils.ts +++ b/server/src/service/infrastructure/notificationProviders/utils.ts @@ -123,8 +123,8 @@ export const buildHardwareNotificationMessage = (clientHost: string, alerts: any return alertText.map((alert) => alert).join("\n"); }; -export const buildHardwareWebhookBody = (alerts: string[], monitor: Monitor): string => { - const content = alerts.map((alert) => alert).join("\n"); +export const buildHardwareWebhookBody = (clientHost: string, alerts: string[], monitor: Monitor): string => { + const content = buildHardwareNotificationMessage(clientHost, alerts, monitor); return content; }; diff --git a/server/src/service/infrastructure/notificationProviders/webhook.ts b/server/src/service/infrastructure/notificationProviders/webhook.ts index 9e743ae53..dfc5f4763 100644 --- a/server/src/service/infrastructure/notificationProviders/webhook.ts +++ b/server/src/service/infrastructure/notificationProviders/webhook.ts @@ -1,6 +1,7 @@ const SERVICE_NAME = "WebhookProvider"; import type { Monitor, Alert, Notification, MonitorStatusResponse } from "@/types/index.js"; import { INotificationProvider } from "@/service/index.js"; +import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js"; import { buildHardwareAlerts, buildHardwareWebhookBody, @@ -15,9 +16,19 @@ export class WebhookProvider implements INotificationProvider { constructor(logger: any) { this.logger = logger; } - private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + private getHardwareContent = ( + clientHost: string, + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision + ) => { + // For status changes (recovery), use standard format + if (decision.notificationReason === "status_change") { + return buildWebhookBody(monitor, monitorStatusResponse); + } + // For threshold breaches, use hardware alert format const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse); - const body = buildHardwareWebhookBody(alertsToSend, monitor); + const body = buildHardwareWebhookBody(clientHost, alertsToSend, monitor); return body; }; @@ -26,10 +37,10 @@ export class WebhookProvider implements INotificationProvider { return body; }; - sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => { + sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { let body; if (monitor.type === "hardware") { - body = this.getHardwareContent(monitor, monitorStatusResponse); + body = this.getHardwareContent("HOST_PLACEHOLDER", monitor, monitorStatusResponse, decision); } else { body = this.getContent(monitor, monitorStatusResponse); } diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index 6174dceb3..f43402f30 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -41,6 +41,7 @@ export class NotificationsService implements INotificationsService { createdAt: number; } >; + private currentDecision?: MonitorActionDecision; constructor( notificationsRepository: INotificationsRepository, @@ -83,17 +84,17 @@ export class NotificationsService implements INotificationsService { private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise => { switch (notification.type) { case "email": - return await this.emailProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.emailProvider.sendAlert(notification, monitor, monitorStatusResponse, this.currentDecision!); case "slack": - return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse, this.currentDecision!); case "discord": - return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse, this.currentDecision!); case "pager_duty": - return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse, this.currentDecision!); case "matrix": - return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse, this.currentDecision!); case "webhook": - return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse); + return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse, this.currentDecision!); default: return false; } @@ -157,7 +158,15 @@ export class NotificationsService implements INotificationsService { this.pendingEmailGroups.delete(key); try { - await this.flushEmailGroup(notification, group.monitors, group.statusResponses); + // For grouped notifications (always DOWN), create a decision + const groupedDecision: MonitorActionDecision = { + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: true, + incidentReason: "status_down", + notificationReason: "status_change", + }; + await this.flushEmailGroup(notification, group.monitors, group.statusResponses, groupedDecision); } catch (error: any) { this.logger.error({ message: error?.message, @@ -196,7 +205,12 @@ export class NotificationsService implements INotificationsService { * @param statusResponses Array of status responses (parallel to monitors) * @returns true if email was sent successfully, false otherwise */ - private flushEmailGroup = async (notification: Notification, monitors: Monitor[], statusResponses: MonitorStatusResponse[]): Promise => { + private flushEmailGroup = async ( + notification: Notification, + monitors: Monitor[], + statusResponses: MonitorStatusResponse[], + decision: MonitorActionDecision + ): Promise => { if (!monitors.length || !statusResponses.length) { return false; } @@ -221,7 +235,7 @@ export class NotificationsService implements INotificationsService { }; // Reuse existing email provider to send grouped notification. - return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus); + return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus, decision); }; handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { @@ -230,6 +244,9 @@ export class NotificationsService implements INotificationsService { return false; } + // Store decision for use in send method + this.currentDecision = decision; + // Send notifications based on decision return await this.sendNotifications(monitor, monitorStatusResponse); };