fixed notificationsService.ts
This commit is contained in:
1 parent
53f420cf56
commit
f9e366e181
1 file changed
+135
-1
@@ -34,6 +34,18 @@ export class NotificationsService implements INotificationsService {
|
||||
private matrixProvider: INotificationProvider;
|
||||
private logger: any;
|
||||
|
||||
// Email grouping (batching) configuration
|
||||
private emailGroupingWindowMs: number;
|
||||
private pendingEmailGroups: Map<
|
||||
string,
|
||||
{
|
||||
monitors: Monitor[];
|
||||
statusResponses: MonitorStatusResponse[];
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
createdAt: number;
|
||||
}
|
||||
>;
|
||||
|
||||
constructor(
|
||||
notificationsRepository: INotificationsRepository,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
@@ -54,6 +66,22 @@ export class NotificationsService implements INotificationsService {
|
||||
this.pagerDutyProvider = pagerDutyProvider;
|
||||
this.matrixProvider = matrixProvider;
|
||||
this.logger = logger;
|
||||
|
||||
// Configure email grouping window (in milliseconds).
|
||||
// When > 0, multiple DOWN events for monitors that share the same
|
||||
// email notification within this window will be batched into a single email.
|
||||
const rawGroupingWindow = process.env.NOTIFICATION_GROUP_WINDOW_MS ?? process.env.NOTIFICATION_GROUP_WINDOW_SECONDS;
|
||||
let groupingWindowMs = 0;
|
||||
if (rawGroupingWindow) {
|
||||
const parsed = Number(rawGroupingWindow);
|
||||
if (!Number.isNaN(parsed) && parsed > 0) {
|
||||
// If value looks like seconds (small number), convert to ms.
|
||||
// This allows either milliseconds (e.g. 60000) or seconds (e.g. 60).
|
||||
groupingWindowMs = parsed <= 300 ? parsed * 1000 : parsed;
|
||||
}
|
||||
}
|
||||
this.emailGroupingWindowMs = groupingWindowMs;
|
||||
this.pendingEmailGroups = new Map();
|
||||
}
|
||||
|
||||
private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
|
||||
@@ -78,7 +106,17 @@ export class NotificationsService implements INotificationsService {
|
||||
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||
const notificationIds = monitor.notifications ?? [];
|
||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||
const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse));
|
||||
|
||||
const tasks = notifications.map((notification) => {
|
||||
// Only group emails, only for DOWN transitions, and only if a window is configured.
|
||||
if (notification.type === "email" && this.emailGroupingWindowMs > 0 && monitorStatusResponse.status === false) {
|
||||
return this.queueGroupedEmailNotification(notification, monitor, monitorStatusResponse);
|
||||
}
|
||||
|
||||
// For all other cases (UP notifications or non-email channels), send immediately.
|
||||
return this.send(notification, monitor, monitorStatusResponse);
|
||||
});
|
||||
|
||||
const outcomes = await Promise.all(tasks);
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
const failed = outcomes.length - succeeded;
|
||||
@@ -93,6 +131,102 @@ export class NotificationsService implements INotificationsService {
|
||||
return succeeded === notifications.length;
|
||||
};
|
||||
|
||||
/**
|
||||
* Queue a DOWN email notification to be potentially grouped with other
|
||||
* DOWN events for the same email notification within the configured window.
|
||||
*
|
||||
* This method returns immediately; the actual email is sent asynchronously
|
||||
* when the grouping window expires.
|
||||
*/
|
||||
private queueGroupedEmailNotification = async (
|
||||
notification: Notification,
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse
|
||||
): Promise<boolean> => {
|
||||
// If grouping is disabled, fallback to immediate send.
|
||||
if (this.emailGroupingWindowMs <= 0) {
|
||||
return await this.send(notification, monitor, monitorStatusResponse);
|
||||
}
|
||||
|
||||
const key = notification.id;
|
||||
const now = Date.now();
|
||||
const existingGroup = this.pendingEmailGroups.get(key);
|
||||
|
||||
if (!existingGroup) {
|
||||
// Create a new group and schedule a flush after the window expires.
|
||||
const timer = setTimeout(async () => {
|
||||
const group = this.pendingEmailGroups.get(key);
|
||||
if (!group) return;
|
||||
|
||||
this.pendingEmailGroups.delete(key);
|
||||
|
||||
try {
|
||||
await this.flushEmailGroup(notification, group.monitors, group.statusResponses);
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
message: error?.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "flushEmailGroup",
|
||||
stack: error?.stack,
|
||||
});
|
||||
}
|
||||
}, this.emailGroupingWindowMs);
|
||||
|
||||
this.pendingEmailGroups.set(key, {
|
||||
monitors: [monitor],
|
||||
statusResponses: [monitorStatusResponse],
|
||||
timer,
|
||||
createdAt: now,
|
||||
});
|
||||
} else {
|
||||
// Append to existing group.
|
||||
existingGroup.monitors.push(monitor);
|
||||
existingGroup.statusResponses.push(monitorStatusResponse);
|
||||
}
|
||||
|
||||
// Consider queueing as "succeeded" from the caller's perspective.
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Flush a grouped set of DOWN events into a single email.
|
||||
*
|
||||
* To avoid changing email templates, we construct a synthetic Monitor
|
||||
* whose name concisely lists all affected services. The existing
|
||||
* `serverIsDownTemplate` is then reused.
|
||||
*/
|
||||
private flushEmailGroup = async (
|
||||
notification: Notification,
|
||||
monitors: Monitor[],
|
||||
statusResponses: MonitorStatusResponse[]
|
||||
): Promise<boolean> => {
|
||||
if (!monitors.length || !statusResponses.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Build a combined monitor name, e.g.:
|
||||
// "2 services are down: A, B"
|
||||
const uniqueNames = Array.from(new Set(monitors.map((m) => m.name)));
|
||||
const servicesCount = uniqueNames.length;
|
||||
const combinedName =
|
||||
servicesCount === 1
|
||||
? `${uniqueNames[0]}`
|
||||
: `${servicesCount} services are down: ${uniqueNames.join(", ")}`;
|
||||
|
||||
// Use the first monitor as a base for URL and other fields.
|
||||
const baseMonitor = monitors[0];
|
||||
const baseStatus = statusResponses[0];
|
||||
|
||||
// Create a shallow clone so we don't mutate the original entity.
|
||||
const syntheticMonitor: Monitor = {
|
||||
...baseMonitor,
|
||||
name: combinedName,
|
||||
};
|
||||
|
||||
// Reuse existing email provider to build and send the email.
|
||||
return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus);
|
||||
};
|
||||
|
||||
handleNotifications = async (
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
|
||||
Reference in new issue
Block a user