test alerts

This commit is contained in:
Alex Holliday committed 2026-01-19 21:39:02 +00:00
1 parent 1d36f40caf
commit 7c9410f9d7
12 files changed
+258 -107

No files matched your search

+1 -1
View File
@@ -26,7 +26,7 @@ export const initializeControllers = (services: InitializedSerivces) => {
controllers.queueController = new QueueController(services.jobQueue);
controllers.logController = new LogController(services.logger);
controllers.statusPageController = new StatusPageController(services.db);
controllers.notificationController = new NotificationController(services.notificationService, services.db, services.monitorsRepository);
controllers.notificationController = new NotificationController(services.notificationsService, services.db, services.monitorsRepository);
controllers.diagnosticController = new DiagnosticController(services.diagnosticService);
controllers.incidentController = new IncidentController(services.incidentService);
+22 -18
View File
@@ -6,8 +6,17 @@ import NetworkService from "../service/infrastructure/networkService.js";
import EmailService from "../service/infrastructure/emailService.js";
import BufferService from "../service/infrastructure/bufferService.js";
import NotificationUtils from "../service/infrastructure/notificationUtils.js";
import NotificationService from "../service/infrastructure/notificationService.js";
import { NotificationsService, StatusService, WebhookProvider, SlackProvider, EmailProvider, DiscordProvider } from "@/service/index.js";
import {
NotificationsService,
StatusService,
WebhookProvider,
SlackProvider,
EmailProvider,
DiscordProvider,
PagerDutyProvider,
MatrixProvider,
INotificationsService,
} from "@/service/index.js";
import ErrorService from "../service/infrastructure/errorService.js";
import SuperSimpleQueueHelper from "../service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
import SuperSimpleQueue from "../service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.js";
@@ -97,7 +106,6 @@ export type InitializedSerivces = {
emailService: any;
bufferService: any;
statusService: any;
notificationService: any;
jobQueue: any;
userService: any;
checkService: any;
@@ -108,6 +116,7 @@ export type InitializedSerivces = {
incidentService: any;
errorService: any;
logger: any;
notificationsService: INotificationsService;
// Repositories
monitorsRepository: IMonitorsRepository;
@@ -219,33 +228,29 @@ export const initializeServices = async ({
settingsService,
});
const notificationService = new NotificationService({
emailService,
db,
logger,
networkService,
stringService,
notificationUtils,
});
const webhookProvider = new WebhookProvider();
const slackProvider = new SlackProvider();
const emailProvider = new EmailProvider(emailService);
const discordProvider = new DiscordProvider();
const pagerDutyProvider = new PagerDutyProvider();
const matrixProvider = new MatrixProvider();
const notificationsService = new NotificationsService(notificationsRepository, {
const notificationsService = new NotificationsService(
notificationsRepository,
webhookProvider,
slackProvider,
emailProvider,
slackProvider,
discordProvider,
});
pagerDutyProvider,
matrixProvider,
logger
);
const superSimpleQueueHelper = new SuperSimpleQueueHelper({
db,
logger,
networkService,
statusService,
notificationService,
notificationsService,
checkService,
buffer: bufferService,
@@ -312,7 +317,6 @@ export const initializeServices = async ({
emailService,
bufferService,
statusService,
notificationService,
jobQueue: superSimpleQueue,
userService,
checkService,
@@ -323,6 +327,7 @@ export const initializeServices = async ({
incidentService,
errorService,
logger,
notificationsService,
// Repositories
monitorsRepository,
@@ -334,7 +339,6 @@ export const initializeServices = async ({
recoveryTokensRepository,
settingsRepository,
notificationsRepository,
notificationsService,
};
Object.values(services).forEach((service) => {
@@ -1,18 +1,20 @@
import { Request, Response, NextFunction } from "express";
import { Notification } from "@/types/index.js";
import { createNotificationBodyValidation } from "@/validation/joi.js";
import { AppError } from "@/utils/AppError.js";
import { IMonitorsRepository } from "@/repositories/index.js";
import { INotificationsService } from "@/service/index.js";
const SERVICE_NAME = "NotificationController";
class NotificationController {
static SERVICE_NAME = SERVICE_NAME;
private db: any;
private notificationService: any;
private notificationsService: INotificationsService;
private monitorsRepository: IMonitorsRepository;
constructor(notificationService: any, db: any, monitorsRepository: IMonitorsRepository) {
this.notificationService = notificationService;
constructor(notificationsService: INotificationsService, db: any, monitorsRepository: IMonitorsRepository) {
this.notificationsService = notificationsService;
this.db = db;
this.monitorsRepository = monitorsRepository;
}
@@ -23,9 +25,8 @@ class NotificationController {
testNotification = async (req: Request, res: Response, next: NextFunction) => {
try {
const notification = req.body;
const success = await this.notificationService.sendTestNotification(notification);
const notification: Notification = req.body;
const success = await this.notificationsService.sendTestNotification(notification);
if (!success) {
throw new AppError({ message: "Sending notification failed", status: 500 });
@@ -177,7 +178,7 @@ class NotificationController {
throw new AppError({ message: "No notifications", status: 400 });
}
const result = await this.notificationService.testAllNotifications(notifications);
const result = await this.notificationsService.testAllNotifications(notifications);
if (!result) {
throw new AppError({ message: "Failed to send all notifications", status: 500 });
+3
View File
@@ -3,8 +3,11 @@ export * from "@/service/infrastructure/networkService.js";
export * from "@/service/infrastructure/notificationProviders/discord.js";
export * from "@/service/infrastructure/notificationProviders/email.js";
export * from "@/service/infrastructure/notificationProviders/INotificationProvider.js";
export * from "@/service/infrastructure/notificationProviders/pagerduty.js";
export * from "@/service/infrastructure/notificationProviders/matrix.js";
export * from "@/service/infrastructure/notificationProviders/slack.js";
export * from "@/service/infrastructure/notificationProviders/webhook.js";
export * from "@/service/infrastructure/notificationsService.js";
export * from "@/service/infrastructure/statusService.js";
export * from "@/service/infrastructure/notificationProviders/webhook.js";
export * from "@/service/infrastructure/notificationProviders/matrix.js";
@@ -9,11 +9,9 @@ class SuperSimpleQueueHelper {
private logger: any;
private networkService: INetworkService;
private statusService: IStatusService;
private notificationService: any;
private notificationsService: INotificationsService;
private checkService: any;
private buffer: any;
private incidentService: any;
constructor({
db,
@@ -29,7 +27,6 @@ class SuperSimpleQueueHelper {
logger: any;
networkService: INetworkService;
statusService: IStatusService;
notificationService: any;
notificationsService: INotificationsService;
checkService: any;
buffer: any;
@@ -38,7 +35,6 @@ class SuperSimpleQueueHelper {
this.logger = logger;
this.networkService = networkService;
this.statusService = statusService;
this.notificationService = notificationService;
this.checkService = checkService;
this.buffer = buffer;
this.notificationsService = notificationsService;
@@ -84,24 +80,17 @@ class SuperSimpleQueueHelper {
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
// Step 5 handle notifications (best effort, continue even in event of failure, don't wait)
this.notificationsService.handleNotifications(monitor, status, statusChangeResult.prevStatus, statusChangeResult.statusChanged);
// this.notificationService
// .handleNotifications({
// ...status,
// monitor: statusChangeResult.monitor,
// prevStatus: statusChangeResult.prevStatus,
// statusChanged: statusChangeResult.statusChanged,
// })
// .catch((error: any) => {
// this.logger.error({
// message: error.message,
// service: SERVICE_NAME,
// method: "getMonitorJob",
// details: `Error sending notifications for job ${monitor.id}: ${error.message}`,
// stack: error.stack,
// });
// });
this.notificationsService
.handleNotifications(monitor, status, statusChangeResult.prevStatus, statusChangeResult.statusChanged)
.catch((error: any) => {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "getMonitorJob",
details: `Error sending notifications for job ${monitor.id}: ${error.message}`,
stack: error.stack,
});
});
} catch (error: any) {
this.logger.warn({
message: error.message,
@@ -1,6 +1,6 @@
import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
import { INotificationProvider } from "@/service/index.js";
import { buildHardwareAlerts, buildHardwareEmail, buildEmail } from "@/service/infrastructure/notificationProviders/utils.js";
import { buildHardwareAlerts, buildHardwareEmail, buildEmail, buildTestEmail } from "@/service/infrastructure/notificationProviders/utils.js";
export class EmailProvider implements INotificationProvider {
private emailService: any;
@@ -33,6 +33,10 @@ export class EmailProvider implements INotificationProvider {
}
async sendTestAlert(notification: Notification): Promise<boolean> {
return false;
const subject = "Test notification";
const html = await buildTestEmail(this.emailService);
const messageId = await this.emailService.sendEmail(notification.address, subject, html);
if (!messageId) return false;
return true;
}
}
@@ -1,34 +0,0 @@
const SERVICE_NAME = "Matrix";
class Matrix {
static SERVICE_NAME = SERVICE_NAME;
constructor({ networkService, logger }) {
this.networkService = networkService;
this.logger = logger;
}
get serviceName() {
return Matrix.SERVICE_NAME;
}
async send({ friendlyName, homeserverUrl, accessToken, roomId, message, monitorName }) {
const title = `Checkmate status for ${monitorName}`;
const formattedMessage = `## ${title}\n${message}`;
try {
await this.networkService.requestMatrix({
homeserverUrl,
accessToken,
roomId,
message: formattedMessage,
});
this.logger.info(`Successfully sent Matrix notification for ${friendlyName}`);
return true;
} catch (error) {
this.logger.error(`Failed to send Matrix notification for ${friendlyName}: ${error.message}`);
return false;
}
}
}
export default Matrix;
@@ -0,0 +1,56 @@
import got from "got";
import type { INotificationProvider } from "@/service/index.js";
import type { Notification, Monitor, MonitorStatusResponse } from "@/types/index.js";
import { buildHardwareAlerts, buildHardwareWebhookBody, buildWebhookBody } from "@/service/infrastructure/notificationProviders/utils.js";
export class MatrixProvider implements INotificationProvider {
private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
const body = buildHardwareWebhookBody(alertsToSend, monitor);
return body;
};
private getContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
const body = buildWebhookBody(monitor, monitorStatusResponse);
return body;
};
sendAlert = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
const { homeserverUrl, accessToken, roomId } = notification;
let content;
if (monitor.type === "hardware") {
content = this.getHardwareContent(monitor, monitorStatusResponse);
} else {
content = this.getContent(monitor, monitorStatusResponse);
}
const title = `Checkmate status for ${monitor.name}`;
const formattedMessage = `## ${title}\n${content}`;
const url = `${homeserverUrl}/_matrix/client/v3/rooms/${roomId}/send/m.room.message?access_token=${accessToken}`;
const body = {
msgtype: "m.text",
body: formattedMessage,
format: "org.matrix.custom.html",
formatted_body: formattedMessage,
};
try {
await got.post(url, {
json: body,
headers: {
"Content-Type": "application/json",
},
responseType: "json",
});
return true;
} catch (error) {
return false;
}
};
sendTestAlert = async (notification: Notification) => {
return false;
};
}
@@ -0,0 +1,50 @@
import got from "got";
import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
import { INotificationProvider } from "@/service/index.js";
import { buildHardwareAlerts, buildHardwareWebhookBody, buildWebhookBody } from "@/service/infrastructure/notificationProviders/utils.js";
export class PagerDutyProvider implements INotificationProvider {
private getHardwareContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
const { alertsToSend } = buildHardwareAlerts("HOST_PLACEHOLDER", monitor, monitorStatusResponse);
const body = buildHardwareWebhookBody(alertsToSend, monitor);
return body;
};
private getContent = (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
const body = buildWebhookBody(monitor, monitorStatusResponse);
return body;
};
async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
let body;
if (monitor.type === "hardware") {
body = this.getHardwareContent(monitor, monitorStatusResponse);
} else {
body = this.getContent(monitor, monitorStatusResponse);
}
try {
await got.post("https://events.pagerduty.com/v2/enqueue", {
json: {
routing_key: notification.address,
event_action: "trigger",
payload: {
summary: body,
severity: "critical",
source: monitor.url,
timestamp: new Date().toISOString(),
},
},
responseType: "json",
});
return true;
} catch (error) {
return false;
}
}
async sendTestAlert(notification: Notification): Promise<boolean> {
return false;
}
}
@@ -1,6 +1,11 @@
import type { Monitor, Notification, MonitorStatusResponse } from "@/types/index.js";
import { INotificationProvider } from "@/service/index.js";
import { buildHardwareAlerts, buildHardwareWebhookBody, buildWebhookBody } from "@/service/infrastructure/notificationProviders/utils.js";
import {
buildHardwareAlerts,
buildHardwareWebhookBody,
buildWebhookBody,
getTestMessage,
} from "@/service/infrastructure/notificationProviders/utils.js";
import got from "got";
export class SlackProvider implements INotificationProvider {
@@ -42,6 +47,21 @@ export class SlackProvider implements INotificationProvider {
}
async sendTestAlert(notification: Notification): Promise<boolean> {
return false;
if (!notification.address) {
return false;
}
try {
await got.post(notification.address, {
json: { text: getTestMessage() },
headers: {
"Content-Type": "application/json",
},
responseType: "json",
});
return true;
} catch (error) {
return false;
}
}
}
@@ -190,3 +190,13 @@ export const buildDiscordBody = (monitor: Monitor, monitorStatusResponse: Monito
};
return body;
};
export const buildTestEmail = async (emailService: any) => {
const context = { testName: "Monitoring System" };
const html = await emailService.buildEmail("testEmailTemplate", context);
return html;
};
export const getTestMessage = () => {
return "This is a test notification from Checkmate";
};
@@ -1,7 +1,7 @@
import type { HardwareStatusPayload, Monitor, MonitorStatusResponse, Notification } from "@/types/index.js";
import { shouldSendHardwareAlert } from "@/service/infrastructure/notificationProviders/utils.js";
import { INotificationsRepository } from "@/repositories/index.js";
import { DiscordProvider, EmailProvider, SlackProvider, WebhookProvider } from "@/service/index.js";
import { INotificationProvider } from "./notificationProviders/INotificationProvider.js";
export interface INotificationsService {
handleNotifications: (
monitor: Monitor,
@@ -9,29 +9,42 @@ export interface INotificationsService {
prevStatus: boolean | undefined,
statusChanged: boolean
) => Promise<boolean>;
sendTestNotification: (notification: Notification) => Promise<boolean>;
testAllNotifications: (notificationIds: string[]) => Promise<boolean>;
}
const SERVICE_NAME = "NotificationsService";
export class NotificationsService implements INotificationsService {
static SERVICE_NAME = SERVICE_NAME;
private notificationsRepository: INotificationsRepository;
private webhookProvider: WebhookProvider;
private emailProvider: EmailProvider;
private slackProvider: SlackProvider;
private discordProvider: DiscordProvider;
private webhookProvider: INotificationProvider;
private emailProvider: INotificationProvider;
private slackProvider: INotificationProvider;
private discordProvider: INotificationProvider;
private pagerDutyProvider: INotificationProvider;
private matrixProvider: INotificationProvider;
private logger: any;
constructor(
notificationsRepository: INotificationsRepository,
providers: {
webhookProvider: WebhookProvider;
emailProvider: EmailProvider;
slackProvider: SlackProvider;
discordProvider: DiscordProvider;
}
webhookProvider: INotificationProvider,
emailProvider: INotificationProvider,
slackProvider: INotificationProvider,
discordProvider: INotificationProvider,
pagerDutyProvider: INotificationProvider,
matrixProvider: INotificationProvider,
logger: any
) {
this.notificationsRepository = notificationsRepository;
this.webhookProvider = providers.webhookProvider;
this.emailProvider = providers.emailProvider;
this.slackProvider = providers.slackProvider;
this.discordProvider = providers.discordProvider;
this.webhookProvider = webhookProvider;
this.emailProvider = emailProvider;
this.slackProvider = slackProvider;
this.discordProvider = discordProvider;
this.pagerDutyProvider = pagerDutyProvider;
this.matrixProvider = matrixProvider;
this.logger = logger;
}
private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
@@ -42,6 +55,10 @@ export class NotificationsService implements INotificationsService {
return await this.slackProvider.sendAlert(notification, monitor, monitorStatusResponse);
case "discord":
return await this.discordProvider.sendAlert(notification, monitor, monitorStatusResponse);
case "pager_duty":
return await this.pagerDutyProvider.sendAlert(notification, monitor, monitorStatusResponse);
case "matrix":
return await this.matrixProvider.sendAlert(notification, monitor, monitorStatusResponse);
case "webhook":
return await this.webhookProvider.sendAlert(notification, monitor, monitorStatusResponse);
default:
@@ -57,7 +74,11 @@ export class NotificationsService implements INotificationsService {
const succeeded = outcomes.filter(Boolean).length;
const failed = outcomes.length - succeeded;
if (failed > 0) {
// logger.warn(`Notification send completed with ${succeeded} success, ${failed} failure(s)`);
this.logger.warn({
message: `Notification send completed with ${succeeded} success, ${failed} failure(s)`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
}
// Return true if all notificaitons succeeded
return succeeded === notifications.length;
@@ -88,15 +109,42 @@ export class NotificationsService implements INotificationsService {
const shouldSend = shouldSendHardwareAlert(monitor, monitorStatusResponse);
if (shouldSend === false) return false;
console.log(JSON.stringify(monitor, null, 2));
// const { subject, html } = await this.notificationUtils.buildHardwareEmail(networkResponse, alerts);
// const content = await this.notificationUtils.buildHardwareNotificationMessage(alerts, monitor);
// const webhookBody = await this.notificationUtils.buildHardwareWebhookBody(alerts, monitor);
// const success = await this.notifyAll({ notificationIDs, subject, html, content, discordContent, webhookBody });
return await this.sendNotifications(monitor, monitorStatusResponse);
}
// We should send a notification for non-hardware monitor status change
return await this.sendNotifications(monitor, monitorStatusResponse);
};
sendTestNotification = async (notification: Notification) => {
switch (notification.type) {
case "email":
return await this.emailProvider.sendTestAlert(notification);
case "slack":
return await this.slackProvider.sendTestAlert(notification);
case "discord":
return await this.discordProvider.sendTestAlert(notification);
case "pager_duty":
return await this.pagerDutyProvider.sendTestAlert(notification);
case "matrix":
return await this.matrixProvider.sendTestAlert(notification);
case "webhook":
return await this.webhookProvider.sendTestAlert(notification);
default:
return false;
}
};
testAllNotifications = async (notificationIds: string[]) => {
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
const tasks = notifications.map((notification) => this.sendTestNotification(notification));
const outcomes = await Promise.all(tasks);
const succeeded = outcomes.filter(Boolean).length;
const failed = outcomes.length - succeeded;
if (failed > 0) {
return false;
}
return true;
};
}