test alerts
This commit is contained in:
1 parent
1d36f40caf
commit
7c9410f9d7
12 files changed
+258
-107
No files matched your search
@@ -3,8 +3,11 @@ export * from "@/service/infrastructure/networkService.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/discord.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/email.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/INotificationProvider.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/pagerduty.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/matrix.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/slack.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/webhook.js";
|
||||
export * from "@/service/infrastructure/notificationsService.js";
|
||||
export * from "@/service/infrastructure/statusService.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/webhook.js";
|
||||
export * from "@/service/infrastructure/notificationProviders/matrix.js";
|
||||
@@ -9,11 +9,9 @@ class SuperSimpleQueueHelper {
|
||||
private logger: any;
|
||||
private networkService: INetworkService;
|
||||
private statusService: IStatusService;
|
||||
private notificationService: any;
|
||||
private notificationsService: INotificationsService;
|
||||
private checkService: any;
|
||||
private buffer: any;
|
||||
private incidentService: any;
|
||||
|
||||
constructor({
|
||||
db,
|
||||
@@ -29,7 +27,6 @@ class SuperSimpleQueueHelper {
|
||||
logger: any;
|
||||
networkService: INetworkService;
|
||||
statusService: IStatusService;
|
||||
notificationService: any;
|
||||
notificationsService: INotificationsService;
|
||||
checkService: any;
|
||||
buffer: any;
|
||||
@@ -38,7 +35,6 @@ class SuperSimpleQueueHelper {
|
||||
this.logger = logger;
|
||||
this.networkService = networkService;
|
||||
this.statusService = statusService;
|
||||
this.notificationService = notificationService;
|
||||
this.checkService = checkService;
|
||||
this.buffer = buffer;
|
||||
this.notificationsService = notificationsService;
|
||||
@@ -84,24 +80,17 @@ class SuperSimpleQueueHelper {
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
|
||||
|
||||
// Step 5 handle notifications (best effort, continue even in event of failure, don't wait)
|
||||
this.notificationsService.handleNotifications(monitor, status, statusChangeResult.prevStatus, statusChangeResult.statusChanged);
|
||||
|
||||
// this.notificationService
|
||||
// .handleNotifications({
|
||||
// ...status,
|
||||
// monitor: statusChangeResult.monitor,
|
||||
// prevStatus: statusChangeResult.prevStatus,
|
||||
// statusChanged: statusChangeResult.statusChanged,
|
||||
// })
|
||||
// .catch((error: any) => {
|
||||
// this.logger.error({
|
||||
// message: error.message,
|
||||
// service: SERVICE_NAME,
|
||||
// method: "getMonitorJob",
|
||||
// details: `Error sending notifications for job ${monitor.id}: ${error.message}`,
|
||||
// stack: error.stack,
|
||||
// });
|
||||
// });
|
||||
this.notificationsService
|
||||
.handleNotifications(monitor, status, statusChangeResult.prevStatus, statusChangeResult.statusChanged)
|
||||
.catch((error: any) => {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
details: `Error sending notifications for job ${monitor.id}: ${error.message}`,
|
||||
stack: error.stack,
|
||||
});
|
||||
});
|
||||
} catch (error: any) {
|
||||
this.logger.warn({
|
||||
message: error.message,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
|
||||
import { INotificationProvider } from "@/service/index.js";
|
||||
import { buildHardwareAlerts, buildHardwareEmail, buildEmail } from "@/service/infrastructure/notificationProviders/utils.js";
|
||||
import { buildHardwareAlerts, buildHardwareEmail, buildEmail, buildTestEmail } from "@/service/infrastructure/notificationProviders/utils.js";
|
||||
|
||||
export class EmailProvider implements INotificationProvider {
|
||||
private emailService: any;
|
||||
@@ -33,6 +33,10 @@ export class EmailProvider implements INotificationProvider {
|
||||
}
|
||||
|
||||
async sendTestAlert(notification: Notification): Promise<boolean> {
|
||||
return false;
|
||||
const subject = "Test notification";
|
||||
const html = await buildTestEmail(this.emailService);
|
||||
const messageId = await this.emailService.sendEmail(notification.address, subject, html);
|
||||
if (!messageId) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
const SERVICE_NAME = "Matrix";
|
||||
|
||||
class Matrix {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ networkService, logger }) {
|
||||
this.networkService = networkService;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return Matrix.SERVICE_NAME;
|
||||
}
|
||||
|
||||
async send({ friendlyName, homeserverUrl, accessToken, roomId, message, monitorName }) {
|
||||
const title = `Checkmate status for ${monitorName}`;
|
||||
const formattedMessage = `## ${title}\n${message}`;
|
||||
try {
|
||||
await this.networkService.requestMatrix({
|
||||
homeserverUrl,
|
||||
accessToken,
|
||||
roomId,
|
||||
message: formattedMessage,
|
||||
});
|
||||
this.logger.info(`Successfully sent Matrix notification for ${friendlyName}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to send Matrix notification for ${friendlyName}: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default Matrix;
|
||||
@@ -0,0 +1,56 @@
|
||||
import got from "got";
|
||||
import type { INotificationProvider } from "@/service/index.js";
|
||||
import type { Notification, Monitor, MonitorStatusResponse } from "@/types/index.js";
|
||||
import { buildHardwareAlerts, buildHardwareWebhookBody, buildWebhookBody } from "@/service/infrastructure/notificationProviders/utils.js";
|
||||
|
||||
export class MatrixProvider implements INotificationProvider {
|
||||
private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||
const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
|
||||
const body = buildHardwareWebhookBody(alertsToSend, monitor);
|
||||
return body;
|
||||
};
|
||||
|
||||
private getContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||
const body = buildWebhookBody(monitor, monitorStatusResponse);
|
||||
return body;
|
||||
};
|
||||
|
||||
sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||
const { homeserverUrl, accessToken, roomId } = notification;
|
||||
|
||||
let content;
|
||||
if (monitor.type === "hardware") {
|
||||
content = this.getHardwareContent(monitor, monitorStatusResponse);
|
||||
} else {
|
||||
content = this.getContent(monitor, monitorStatusResponse);
|
||||
}
|
||||
|
||||
const title = `Checkmate status for ${monitor.name}`;
|
||||
|
||||
const formattedMessage = `## ${title}\n${content}`;
|
||||
|
||||
const url = `${homeserverUrl}/_matrix/client/v3/rooms/${roomId}/send/m.room.message?access_token=${accessToken}`;
|
||||
const body = {
|
||||
msgtype: "m.text",
|
||||
body: formattedMessage,
|
||||
format: "org.matrix.custom.html",
|
||||
formatted_body: formattedMessage,
|
||||
};
|
||||
try {
|
||||
await got.post(url, {
|
||||
json: body,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
responseType: "json",
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
sendTestAlert = async (notification: Notification) => {
|
||||
return false;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import got from "got";
|
||||
import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
|
||||
import { INotificationProvider } from "@/service/index.js";
|
||||
import { buildHardwareAlerts, buildHardwareWebhookBody, buildWebhookBody } from "@/service/infrastructure/notificationProviders/utils.js";
|
||||
|
||||
export class PagerDutyProvider implements INotificationProvider {
|
||||
private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||
const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
|
||||
const body = buildHardwareWebhookBody(alertsToSend, monitor);
|
||||
return body;
|
||||
};
|
||||
|
||||
private getContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||
const body = buildWebhookBody(monitor, monitorStatusResponse);
|
||||
return body;
|
||||
};
|
||||
|
||||
async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
|
||||
let body;
|
||||
if (monitor.type === "hardware") {
|
||||
body = this.getHardwareContent(monitor, monitorStatusResponse);
|
||||
} else {
|
||||
body = this.getContent(monitor, monitorStatusResponse);
|
||||
}
|
||||
|
||||
try {
|
||||
await got.post("https://events.pagerduty.com/v2/enqueue", {
|
||||
json: {
|
||||
routing_key: notification.address,
|
||||
event_action: "trigger",
|
||||
payload: {
|
||||
summary: body,
|
||||
severity: "critical",
|
||||
source: monitor.url,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
},
|
||||
responseType: "json",
|
||||
});
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async sendTestAlert(notification: Notification): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
|
||||
import { INotificationProvider } from "@/service/index.js";
|
||||
import { buildHardwareAlerts, buildHardwareWebhookBody, buildWebhookBody } from "@/service/infrastructure/notificationProviders/utils.js";
|
||||
import {
|
||||
buildHardwareAlerts,
|
||||
buildHardwareWebhookBody,
|
||||
buildWebhookBody,
|
||||
getTestMessage,
|
||||
} from "@/service/infrastructure/notificationProviders/utils.js";
|
||||
import got from "got";
|
||||
|
||||
export class SlackProvider implements INotificationProvider {
|
||||
@@ -42,6 +47,21 @@ export class SlackProvider implements INotificationProvider {
|
||||
}
|
||||
|
||||
async sendTestAlert(notification: Notification): Promise<boolean> {
|
||||
return false;
|
||||
if (!notification.address) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
await got.post(notification.address, {
|
||||
json: { text: getTestMessage() },
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
responseType: "json",
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -190,3 +190,13 @@ export const buildDiscordBody = (monitor: Monitor, monitorStatusResponse: Monito
|
||||
};
|
||||
return body;
|
||||
};
|
||||
|
||||
export const buildTestEmail = async (emailService: any) => {
|
||||
const context = { testName: "Monitoring System" };
|
||||
const html = await emailService.buildEmail("testEmailTemplate", context);
|
||||
return html;
|
||||
};
|
||||
|
||||
export const getTestMessage = () => {
|
||||
return "This is a test notification from Checkmate";
|
||||
};
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { HardwareStatusPayload, Monitor, MonitorStatusResponse, Notification } from "@/types/index.js";
|
||||
import { shouldSendHardwareAlert } from "@/service/infrastructure/notificationProviders/utils.js";
|
||||
import { INotificationsRepository } from "@/repositories/index.js";
|
||||
import { DiscordProvider, EmailProvider, SlackProvider, WebhookProvider } from "@/service/index.js";
|
||||
import { INotificationProvider } from "./notificationProviders/INotificationProvider.js";
|
||||
export interface INotificationsService {
|
||||
handleNotifications: (
|
||||
monitor: Monitor,
|
||||
@@ -9,29 +9,42 @@ export interface INotificationsService {
|
||||
prevStatus: boolean | undefined,
|
||||
statusChanged: boolean
|
||||
) => Promise<boolean>;
|
||||
sendTestNotification: (notification: Notification) => Promise<boolean>;
|
||||
testAllNotifications: (notificationIds: string[]) => Promise<boolean>;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "NotificationsService";
|
||||
|
||||
export class NotificationsService implements INotificationsService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private notificationsRepository: INotificationsRepository;
|
||||
private webhookProvider: WebhookProvider;
|
||||
private emailProvider: EmailProvider;
|
||||
private slackProvider: SlackProvider;
|
||||
private discordProvider: DiscordProvider;
|
||||
private webhookProvider: INotificationProvider;
|
||||
private emailProvider: INotificationProvider;
|
||||
private slackProvider: INotificationProvider;
|
||||
private discordProvider: INotificationProvider;
|
||||
private pagerDutyProvider: INotificationProvider;
|
||||
private matrixProvider: INotificationProvider;
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
notificationsRepository: INotificationsRepository,
|
||||
providers: {
|
||||
webhookProvider: WebhookProvider;
|
||||
emailProvider: EmailProvider;
|
||||
slackProvider: SlackProvider;
|
||||
discordProvider: DiscordProvider;
|
||||
}
|
||||
webhookProvider: INotificationProvider,
|
||||
emailProvider: INotificationProvider,
|
||||
slackProvider: INotificationProvider,
|
||||
discordProvider: INotificationProvider,
|
||||
pagerDutyProvider: INotificationProvider,
|
||||
matrixProvider: INotificationProvider,
|
||||
logger: any
|
||||
) {
|
||||
this.notificationsRepository = notificationsRepository;
|
||||
this.webhookProvider = providers.webhookProvider;
|
||||
this.emailProvider = providers.emailProvider;
|
||||
this.slackProvider = providers.slackProvider;
|
||||
this.discordProvider = providers.discordProvider;
|
||||
this.webhookProvider = webhookProvider;
|
||||
this.emailProvider = emailProvider;
|
||||
this.slackProvider = slackProvider;
|
||||
this.discordProvider = discordProvider;
|
||||
this.pagerDutyProvider = pagerDutyProvider;
|
||||
this.matrixProvider = matrixProvider;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
|
||||
@@ -42,6 +55,10 @@ export class NotificationsService implements INotificationsService {
|
||||
return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse);
|
||||
case "discord":
|
||||
return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse);
|
||||
case "pager_duty":
|
||||
return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse);
|
||||
case "matrix":
|
||||
return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse);
|
||||
case "webhook":
|
||||
return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse);
|
||||
default:
|
||||
@@ -57,7 +74,11 @@ export class NotificationsService implements INotificationsService {
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
const failed = outcomes.length - succeeded;
|
||||
if (failed > 0) {
|
||||
// logger.warn(`Notification send completed with ${succeeded} success, ${failed} failure(s)`);
|
||||
this.logger.warn({
|
||||
message: `Notification send completed with ${succeeded} success, ${failed} failure(s)`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
});
|
||||
}
|
||||
// Return true if all notificaitons succeeded
|
||||
return succeeded === notifications.length;
|
||||
@@ -88,15 +109,42 @@ export class NotificationsService implements INotificationsService {
|
||||
|
||||
const shouldSend = shouldSendHardwareAlert(monitor, monitorStatusResponse);
|
||||
if (shouldSend === false) return false;
|
||||
console.log(JSON.stringify(monitor, null, 2));
|
||||
// const { subject, html } = await this.notificationUtils.buildHardwareEmail(networkResponse, alerts);
|
||||
// const content = await this.notificationUtils.buildHardwareNotificationMessage(alerts, monitor);
|
||||
// const webhookBody = await this.notificationUtils.buildHardwareWebhookBody(alerts, monitor);
|
||||
// const success = await this.notifyAll({ notificationIDs, subject, html, content, discordContent, webhookBody });
|
||||
|
||||
return await this.sendNotifications(monitor, monitorStatusResponse);
|
||||
}
|
||||
|
||||
// We should send a notification for non-hardware monitor status change
|
||||
return await this.sendNotifications(monitor, monitorStatusResponse);
|
||||
};
|
||||
|
||||
sendTestNotification = async (notification: Notification) => {
|
||||
switch (notification.type) {
|
||||
case "email":
|
||||
return await this.emailProvider.sendTestAlert(notification);
|
||||
case "slack":
|
||||
return await this.slackProvider.sendTestAlert(notification);
|
||||
case "discord":
|
||||
return await this.discordProvider.sendTestAlert(notification);
|
||||
case "pager_duty":
|
||||
return await this.pagerDutyProvider.sendTestAlert(notification);
|
||||
case "matrix":
|
||||
return await this.matrixProvider.sendTestAlert(notification);
|
||||
case "webhook":
|
||||
return await this.webhookProvider.sendTestAlert(notification);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
testAllNotifications = async (notificationIds: string[]) => {
|
||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||
const tasks = notifications.map((notification) => this.sendTestNotification(notification));
|
||||
const outcomes = await Promise.all(tasks);
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
const failed = outcomes.length - succeeded;
|
||||
if (failed > 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user