3 files changed
+142
-3
No files matched your search
Generated
+1
-1
@@ -6203,4 +6203,4 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -19,7 +19,11 @@ export class EmailProvider implements INotificationProvider {
|
|||||||
};
|
};
|
||||||
|
|
||||||
async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
|
async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
|
||||||
const subject = `Monitor ${monitor.name} infrastructure alerts`;
|
// For grouped notifications (identified by ":" in name), customize subject to indicate multiple services.
|
||||||
|
// Example: "2 services: Service A, Service B" becomes "Alert: 2 services are down"
|
||||||
|
const isGroupedNotification = monitor.name.includes(":");
|
||||||
|
const subject = isGroupedNotification ? `Alert: ${monitor.name} are down` : `Monitor ${monitor.name} is down`;
|
||||||
|
|
||||||
let html;
|
let html;
|
||||||
if (monitor.type === "hardware") {
|
if (monitor.type === "hardware") {
|
||||||
html = this.buildHardwareEmail(monitor, monitorStatusResponse);
|
html = this.buildHardwareEmail(monitor, monitorStatusResponse);
|
||||||
|
|||||||
@@ -34,6 +34,18 @@ export class NotificationsService implements INotificationsService {
|
|||||||
private matrixProvider: INotificationProvider;
|
private matrixProvider: INotificationProvider;
|
||||||
private logger: any;
|
private logger: any;
|
||||||
|
|
||||||
|
// Email grouping (batching) configuration
|
||||||
|
private emailGroupingWindowMs: number;
|
||||||
|
private pendingEmailGroups: Map<
|
||||||
|
string,
|
||||||
|
{
|
||||||
|
monitors: Monitor[];
|
||||||
|
statusResponses: MonitorStatusResponse[];
|
||||||
|
timer: ReturnType<typeof setTimeout>;
|
||||||
|
createdAt: number;
|
||||||
|
}
|
||||||
|
>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
notificationsRepository: INotificationsRepository,
|
notificationsRepository: INotificationsRepository,
|
||||||
monitorsRepository: IMonitorsRepository,
|
monitorsRepository: IMonitorsRepository,
|
||||||
@@ -54,6 +66,22 @@ export class NotificationsService implements INotificationsService {
|
|||||||
this.pagerDutyProvider = pagerDutyProvider;
|
this.pagerDutyProvider = pagerDutyProvider;
|
||||||
this.matrixProvider = matrixProvider;
|
this.matrixProvider = matrixProvider;
|
||||||
this.logger = logger;
|
this.logger = logger;
|
||||||
|
|
||||||
|
// Configure email grouping window (in milliseconds).
|
||||||
|
// When > 0, multiple DOWN events for monitors that share the same
|
||||||
|
// email notification within this window will be batched into a single email.
|
||||||
|
const rawGroupingWindow = process.env.NOTIFICATION_GROUP_WINDOW_MS ?? process.env.NOTIFICATION_GROUP_WINDOW_SECONDS;
|
||||||
|
let groupingWindowMs = 0;
|
||||||
|
if (rawGroupingWindow) {
|
||||||
|
const parsed = Number(rawGroupingWindow);
|
||||||
|
if (!Number.isNaN(parsed) && parsed > 0) {
|
||||||
|
// If value looks like seconds (small number), convert to ms.
|
||||||
|
// This allows either milliseconds (e.g. 60000) or seconds (e.g. 60).
|
||||||
|
groupingWindowMs = parsed <= 300 ? parsed * 1000 : parsed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.emailGroupingWindowMs = groupingWindowMs;
|
||||||
|
this.pendingEmailGroups = new Map();
|
||||||
}
|
}
|
||||||
|
|
||||||
private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
|
private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
|
||||||
@@ -78,7 +106,17 @@ export class NotificationsService implements INotificationsService {
|
|||||||
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||||
const notificationIds = monitor.notifications ?? [];
|
const notificationIds = monitor.notifications ?? [];
|
||||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||||
const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse));
|
|
||||||
|
const tasks = notifications.map((notification) => {
|
||||||
|
// Only group emails, only for DOWN transitions, and only if a window is configured.
|
||||||
|
if (notification.type === "email" && this.emailGroupingWindowMs > 0 && monitorStatusResponse.status === false) {
|
||||||
|
return this.queueGroupedEmailNotification(notification, monitor, monitorStatusResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
// For all other cases (UP notifications or non-email channels), send immediately.
|
||||||
|
return this.send(notification, monitor, monitorStatusResponse);
|
||||||
|
});
|
||||||
|
|
||||||
const outcomes = await Promise.all(tasks);
|
const outcomes = await Promise.all(tasks);
|
||||||
const succeeded = outcomes.filter(Boolean).length;
|
const succeeded = outcomes.filter(Boolean).length;
|
||||||
const failed = outcomes.length - succeeded;
|
const failed = outcomes.length - succeeded;
|
||||||
@@ -93,6 +131,103 @@ export class NotificationsService implements INotificationsService {
|
|||||||
return succeeded === notifications.length;
|
return succeeded === notifications.length;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Queue a DOWN email notification to be potentially grouped with other
|
||||||
|
* DOWN events for the same email notification within the configured window.
|
||||||
|
*
|
||||||
|
* This method returns immediately; the actual email is sent asynchronously
|
||||||
|
* when the grouping window expires.
|
||||||
|
*/
|
||||||
|
private queueGroupedEmailNotification = async (
|
||||||
|
notification: Notification,
|
||||||
|
monitor: Monitor,
|
||||||
|
monitorStatusResponse: MonitorStatusResponse
|
||||||
|
): Promise<boolean> => {
|
||||||
|
// If grouping is disabled, fallback to immediate send.
|
||||||
|
if (this.emailGroupingWindowMs <= 0) {
|
||||||
|
return await this.send(notification, monitor, monitorStatusResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
const key = notification.id;
|
||||||
|
const now = Date.now();
|
||||||
|
const existingGroup = this.pendingEmailGroups.get(key);
|
||||||
|
|
||||||
|
if (!existingGroup) {
|
||||||
|
// Create a new group and schedule a flush after the window expires.
|
||||||
|
const timer = setTimeout(async () => {
|
||||||
|
const group = this.pendingEmailGroups.get(key);
|
||||||
|
if (!group) return;
|
||||||
|
|
||||||
|
this.pendingEmailGroups.delete(key);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.flushEmailGroup(notification, group.monitors, group.statusResponses);
|
||||||
|
} catch (error: any) {
|
||||||
|
this.logger.error({
|
||||||
|
message: error?.message,
|
||||||
|
service: SERVICE_NAME,
|
||||||
|
method: "flushEmailGroup",
|
||||||
|
stack: error?.stack,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}, this.emailGroupingWindowMs);
|
||||||
|
|
||||||
|
this.pendingEmailGroups.set(key, {
|
||||||
|
monitors: [monitor],
|
||||||
|
statusResponses: [monitorStatusResponse],
|
||||||
|
timer,
|
||||||
|
createdAt: now,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Append to existing group.
|
||||||
|
existingGroup.monitors.push(monitor);
|
||||||
|
existingGroup.statusResponses.push(monitorStatusResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consider queueing as "succeeded" from the caller's perspective.
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flush a grouped set of DOWN events into a single email.
|
||||||
|
*
|
||||||
|
* To avoid changing email templates, we construct a synthetic Monitor
|
||||||
|
* whose name concisely lists all affected services. The existing
|
||||||
|
* `serverIsDownTemplate` is then reused.
|
||||||
|
*
|
||||||
|
* @param notification The email notification to send to
|
||||||
|
* @param monitors Array of monitors that went down
|
||||||
|
* @param statusResponses Array of status responses (parallel to monitors)
|
||||||
|
* @returns true if email was sent successfully, false otherwise
|
||||||
|
*/
|
||||||
|
private flushEmailGroup = async (notification: Notification, monitors: Monitor[], statusResponses: MonitorStatusResponse[]): Promise<boolean> => {
|
||||||
|
if (!monitors.length || !statusResponses.length) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build a combined monitor name listing all affected services.
|
||||||
|
// Example: "Service A, Service B" (2 services) or "Service A" (1 service)
|
||||||
|
const uniqueNames = Array.from(new Set(monitors.map((m) => m.name)));
|
||||||
|
const servicesCount = uniqueNames.length;
|
||||||
|
const servicesList = uniqueNames.join(", ");
|
||||||
|
|
||||||
|
const combinedName = servicesCount === 1 ? servicesList : `${servicesCount} services: ${servicesList}`;
|
||||||
|
|
||||||
|
// Use the first monitor as a base for URL and other fields.
|
||||||
|
const baseMonitor = monitors[0]!;
|
||||||
|
const baseStatus = statusResponses[0]!;
|
||||||
|
|
||||||
|
// Create a shallow clone so we don't mutate the original entity.
|
||||||
|
// This preserves monitor properties while overriding the name for grouped display.
|
||||||
|
const syntheticMonitor: Monitor = {
|
||||||
|
...baseMonitor,
|
||||||
|
name: combinedName,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Reuse existing email provider to send grouped notification.
|
||||||
|
return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus);
|
||||||
|
};
|
||||||
|
|
||||||
handleNotifications = async (
|
handleNotifications = async (
|
||||||
monitor: Monitor,
|
monitor: Monitor,
|
||||||
monitorStatusResponse: MonitorStatusResponse,
|
monitorStatusResponse: MonitorStatusResponse,
|
||||||
|
|||||||
Reference in new issue
Block a user