-> timeseries
This commit is contained in:
1 parent
5c14c68899
commit
e090d3cfd3
91 files changed
+941
-987
No files matched your search
@@ -0,0 +1,176 @@
|
||||
import Scheduler from "super-simple-scheduler";
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
|
||||
class SuperSimpleQueue {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ envSettings, db, logger, helper }) {
|
||||
this.envSettings = envSettings;
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.helper = helper;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return SuperSimpleQueue.SERVICE_NAME;
|
||||
}
|
||||
|
||||
static async create({ envSettings, db, logger, helper }) {
|
||||
const instance = new SuperSimpleQueue({ envSettings, db, logger, helper });
|
||||
await instance.init();
|
||||
return instance;
|
||||
}
|
||||
|
||||
init = async () => {
|
||||
try {
|
||||
this.scheduler = new Scheduler({
|
||||
// storeType: "mongo",
|
||||
// storeType: "redis",
|
||||
logLevel: "debug",
|
||||
debug: true,
|
||||
// dbUri: this.envSettings.dbConnectionString,
|
||||
});
|
||||
this.scheduler.start();
|
||||
|
||||
this.scheduler.addTemplate("monitor-job", this.helper.getMonitorJob());
|
||||
const monitors = await this.db.monitorModule.getAllMonitors();
|
||||
for (const monitor of monitors) {
|
||||
const randomOffset = Math.floor(Math.random() * 100);
|
||||
setTimeout(() => {
|
||||
this.addJob(monitor._id, monitor);
|
||||
}, randomOffset);
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: "Failed to initialize SuperSimpleQueue",
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
details: error,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
addJob = async (monitorId, monitor) => {
|
||||
this.scheduler.addJob({
|
||||
id: monitorId.toString(),
|
||||
template: "monitor-job",
|
||||
repeat: monitor.interval,
|
||||
active: monitor.isActive,
|
||||
data: monitor.toObject(),
|
||||
});
|
||||
};
|
||||
|
||||
deleteJob = async (monitor) => {
|
||||
this.scheduler.removeJob(monitor._id.toString());
|
||||
};
|
||||
|
||||
pauseJob = async (monitor) => {
|
||||
const result = this.scheduler.pauseJob(monitor._id.toString());
|
||||
if (result === false) {
|
||||
throw new Error("Failed to resume monitor");
|
||||
}
|
||||
this.logger.debug({
|
||||
message: `Paused monitor ${monitor._id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
};
|
||||
|
||||
resumeJob = async (monitor) => {
|
||||
const result = this.scheduler.resumeJob(monitor._id.toString());
|
||||
if (result === false) {
|
||||
throw new Error("Failed to resume monitor");
|
||||
}
|
||||
this.logger.debug({
|
||||
message: `Resumed monitor ${monitor._id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
};
|
||||
|
||||
updateJob = async (monitor) => {
|
||||
this.scheduler.updateJob(monitor._id.toString(), { repeat: monitor.interval, data: monitor.toObject() });
|
||||
};
|
||||
|
||||
shutdown = async () => {
|
||||
this.scheduler.stop();
|
||||
};
|
||||
|
||||
getMetrics = async () => {
|
||||
const jobs = await this.scheduler.getJobs();
|
||||
const metrics = jobs.reduce(
|
||||
(acc, job) => {
|
||||
acc.totalRuns += job.runCount || 0;
|
||||
acc.totalFailures += job.failCount || 0;
|
||||
acc.jobs++;
|
||||
if (job.failCount > 0 && job.lastFailedAt >= job.lsatRunAt) {
|
||||
acc.failingJobs++;
|
||||
}
|
||||
|
||||
if (job.lockedAt) {
|
||||
acc.activeJobs++;
|
||||
}
|
||||
|
||||
if (job.failCount > 0) {
|
||||
acc.jobsWithFailures.push({
|
||||
monitorId: job.id,
|
||||
monitorUrl: job?.data?.url || null,
|
||||
monitorType: job?.data?.type || null,
|
||||
failedAt: job.lastFailedAt,
|
||||
failCount: job.failCount,
|
||||
failReason: job.lastFailReason,
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{
|
||||
jobs: 0,
|
||||
activeJobs: 0,
|
||||
failingJobs: 0,
|
||||
jobsWithFailures: [],
|
||||
totalRuns: 0,
|
||||
totalFailures: 0,
|
||||
}
|
||||
);
|
||||
return metrics;
|
||||
};
|
||||
|
||||
getJobs = async () => {
|
||||
const jobs = await this.scheduler.getJobs();
|
||||
return jobs.map((job) => {
|
||||
return {
|
||||
monitorId: job.id,
|
||||
monitorUrl: job?.data?.url || null,
|
||||
monitorType: job?.data?.type || null,
|
||||
monitorInterval: job?.data?.interval || null,
|
||||
active: job.active,
|
||||
lockedAt: job.lockedAt,
|
||||
runCount: job.runCount || 0,
|
||||
failCount: job.failCount || 0,
|
||||
failReason: job.lastFailReason,
|
||||
lastRunAt: job.lastRunAt,
|
||||
lastFinishedAt: job.lastFinishedAt,
|
||||
lastRunTook: job.lockedAt ? null : job.lastFinishedAt - job.lastRunAt,
|
||||
lastFailedAt: job.lastFailedAt,
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
flushQueues = async () => {
|
||||
const stopRes = this.scheduler.stop();
|
||||
const flushRes = this.scheduler.flushJobs();
|
||||
const initRes = await this.init();
|
||||
return {
|
||||
success: stopRes && flushRes && initRes,
|
||||
};
|
||||
};
|
||||
|
||||
obliterate = async () => {
|
||||
console.log("obliterate not implemented");
|
||||
};
|
||||
}
|
||||
|
||||
export default SuperSimpleQueue;
|
||||
@@ -0,0 +1,115 @@
|
||||
const SERVICE_NAME = "JobQueueHelper";
|
||||
|
||||
class SuperSimpleQueueHelper {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
/**
|
||||
* @param {{
|
||||
* db: import("../database.js").Database,
|
||||
* logger: import("../logger.js").Logger,
|
||||
* networkService: import("../networkService.js").NetworkService,
|
||||
* statusService: import("../statusService.js").StatusService,
|
||||
* notificationService: import("../notificationService.js").NotificationService
|
||||
* }}
|
||||
*/
|
||||
constructor({ db, logger, networkService, statusService, notificationService }) {
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.networkService = networkService;
|
||||
this.statusService = statusService;
|
||||
this.notificationService = notificationService;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return SuperSimpleQueueHelper.SERVICE_NAME;
|
||||
}
|
||||
|
||||
getMonitorJob = () => {
|
||||
return async (monitor) => {
|
||||
try {
|
||||
const monitorId = monitor._id;
|
||||
const teamId = monitor.teamId;
|
||||
if (!monitorId) {
|
||||
throw new Error("No monitor id");
|
||||
}
|
||||
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId, teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.logger.info({
|
||||
message: `Monitor ${monitorId} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const networkResponse = await this.networkService.requestStatus(monitor);
|
||||
|
||||
if (!networkResponse) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
const { monitor: updatedMonitor, statusChanged, prevStatus } = await this.statusService.updateStatus(networkResponse);
|
||||
|
||||
this.notificationService
|
||||
.handleNotifications({
|
||||
...networkResponse,
|
||||
monitor: updatedMonitor,
|
||||
prevStatus,
|
||||
statusChanged,
|
||||
})
|
||||
.catch((error) => {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
details: `Error sending notifications for job ${monitor._id}: ${error.message}`,
|
||||
stack: error.stack,
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn({
|
||||
message: error.message,
|
||||
service: error.service || SERVICE_NAME,
|
||||
method: error.method || "getMonitorJob",
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
async isInMaintenanceWindow(monitorId, teamId) {
|
||||
const maintenanceWindows = await this.db.maintenanceWindowModule.getMaintenanceWindowsByMonitorId({
|
||||
monitorId: monitorId.toString(),
|
||||
teamId: teamId.toString(),
|
||||
});
|
||||
// Check for active maintenance window:
|
||||
const maintenanceWindowIsActive = maintenanceWindows.reduce((acc, window) => {
|
||||
if (window.active) {
|
||||
const start = new Date(window.start);
|
||||
const end = new Date(window.end);
|
||||
const now = new Date();
|
||||
const repeatInterval = window.repeat || 0;
|
||||
|
||||
// If start is < now and end > now, we're in maintenance
|
||||
if (start <= now && end >= now) return true;
|
||||
|
||||
// If maintenance window was set in the past with a repeat,
|
||||
// we need to advance start and end to see if we are in range
|
||||
|
||||
while (start < now && repeatInterval !== 0) {
|
||||
start.setTime(start.getTime() + repeatInterval);
|
||||
end.setTime(end.getTime() + repeatInterval);
|
||||
if (start <= now && end >= now) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return acc;
|
||||
}, false);
|
||||
return maintenanceWindowIsActive;
|
||||
}
|
||||
}
|
||||
|
||||
export default SuperSimpleQueueHelper;
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
import { config } from "@/config/index.js";
|
||||
|
||||
const SERVICE_NAME = "BufferService";
|
||||
|
||||
class BufferService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
constructor({ db, logger, envSettings, incidentService }) {
|
||||
console.log(envSettings);
|
||||
this.BUFFER_TIMEOUT = config.NODE_ENV === "development" ? 10 : 1000 * 60 * 1; // 1 minute
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.incidentService = incidentService;
|
||||
this.SERVICE_NAME = SERVICE_NAME;
|
||||
this.buffer = [];
|
||||
this.incidentBuffer = [];
|
||||
this.scheduleNextFlush();
|
||||
this.logger.info({
|
||||
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "constructor",
|
||||
});
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return BufferService.SERVICE_NAME;
|
||||
}
|
||||
|
||||
addToBuffer({ check }) {
|
||||
try {
|
||||
this.buffer.push(check);
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "addToBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
addIncidentToBuffer({ monitor, check, action = "create" }) {
|
||||
try {
|
||||
if (!monitor || !check) {
|
||||
this.logger.warn({
|
||||
message: "Skipping incident buffer item: missing monitor or check",
|
||||
service: this.SERVICE_NAME,
|
||||
method: "addIncidentToBuffer",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
this.incidentBuffer.push({ monitor, check, action });
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "addIncidentToBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
removeCheckFromBuffer(checkToRemove) {
|
||||
try {
|
||||
if (!checkToRemove) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const index = this.buffer.findIndex((check) => {
|
||||
if (checkToRemove._id && check._id) {
|
||||
return check._id.toString() === checkToRemove._id.toString();
|
||||
}
|
||||
return (
|
||||
check.monitorId?.toString() === checkToRemove.monitorId?.toString() &&
|
||||
check.teamId?.toString() === checkToRemove.teamId?.toString() &&
|
||||
check.type === checkToRemove.type &&
|
||||
check.status === checkToRemove.status &&
|
||||
check.statusCode === checkToRemove.statusCode &&
|
||||
check.responseTime === checkToRemove.responseTime &&
|
||||
check.message === checkToRemove.message
|
||||
);
|
||||
});
|
||||
|
||||
if (index !== -1) {
|
||||
this.buffer.splice(index, 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "removeCheckFromBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
scheduleNextFlush() {
|
||||
this.bufferTimer = setTimeout(async () => {
|
||||
try {
|
||||
await this.flushBuffer();
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: `Error in flush cycle: ${error.message}`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "scheduleNextFlush",
|
||||
stack: error.stack,
|
||||
});
|
||||
} finally {
|
||||
// Schedule the next flush only after the current one completes
|
||||
this.scheduleNextFlush();
|
||||
}
|
||||
}, this.BUFFER_TIMEOUT);
|
||||
}
|
||||
async flushBuffer() {
|
||||
try {
|
||||
if (this.buffer.length > 0) {
|
||||
await this.db.checkModule.createChecks(this.buffer);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.incidentBuffer.length > 0 && this.incidentService) {
|
||||
await this.flushIncidentBuffer();
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
|
||||
this.buffer = [];
|
||||
this.incidentBuffer = [];
|
||||
}
|
||||
|
||||
async flushIncidentBuffer() {
|
||||
if (!this.incidentService || this.incidentBuffer.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const itemsToProcess = [...this.incidentBuffer];
|
||||
await this.incidentService.processIncidentsFromBuffer(itemsToProcess);
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: `Error flushing incident buffer: ${error.message}`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushIncidentBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default BufferService;
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
import { fileURLToPath } from "url";
|
||||
import path from "path";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const SERVICE_NAME = "EmailService";
|
||||
|
||||
/**
|
||||
* Represents an email service that can load templates, build, and send emails.
|
||||
*/
|
||||
class EmailService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
/**
|
||||
* Constructs an instance of the EmailService, initializing template loaders and the email transporter.
|
||||
* @param {Object} settingsService - The settings service to get email configuration.
|
||||
* @param {Object} fs - The file system module.
|
||||
* @param {Object} path - The path module.
|
||||
* @param {Function} compile - The Handlebars compile function.
|
||||
* @param {Function} mjml2html - The MJML to HTML conversion function.
|
||||
* @param {Object} nodemailer - The nodemailer module.
|
||||
* @param {Object} logger - The logger module.
|
||||
*/
|
||||
constructor(settingsService, fs, path, compile, mjml2html, nodemailer, logger) {
|
||||
this.settingsService = settingsService;
|
||||
this.fs = fs;
|
||||
this.path = path;
|
||||
this.compile = compile;
|
||||
this.mjml2html = mjml2html;
|
||||
this.nodemailer = nodemailer;
|
||||
this.logger = logger;
|
||||
this.init();
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return EmailService.SERVICE_NAME;
|
||||
}
|
||||
|
||||
init = async () => {
|
||||
/**
|
||||
* Loads an email template from the filesystem.
|
||||
*
|
||||
* @param {string} templateName - The name of the template to load.
|
||||
* @returns {Function} A compiled template function that can be used to generate HTML email content.
|
||||
*/
|
||||
this.loadTemplate = (templateName) => {
|
||||
try {
|
||||
const templatePath = this.path.join(__dirname, `../../../templates/${templateName}.mjml`);
|
||||
const templateContent = this.fs.readFileSync(templatePath, "utf8");
|
||||
return this.compile(templateContent);
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "loadTemplate",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* A lookup object to access preloaded email templates.
|
||||
* @type {Object.<string, Function>}
|
||||
* TODO Load less used templates in their respective functions
|
||||
*/
|
||||
this.templateLookup = {
|
||||
welcomeEmailTemplate: this.loadTemplate("welcomeEmail"),
|
||||
employeeActivationTemplate: this.loadTemplate("employeeActivation"),
|
||||
noIncidentsThisWeekTemplate: this.loadTemplate("noIncidentsThisWeek"),
|
||||
serverIsDownTemplate: this.loadTemplate("serverIsDown"),
|
||||
serverIsUpTemplate: this.loadTemplate("serverIsUp"),
|
||||
passwordResetTemplate: this.loadTemplate("passwordReset"),
|
||||
hardwareIncidentTemplate: this.loadTemplate("hardwareIncident"),
|
||||
testEmailTemplate: this.loadTemplate("testEmailTemplate"),
|
||||
};
|
||||
|
||||
/**
|
||||
* The email transporter used to send emails.
|
||||
* @type {Object}
|
||||
*/
|
||||
};
|
||||
|
||||
buildEmail = async (template, context) => {
|
||||
try {
|
||||
const mjml = this.templateLookup[template](context);
|
||||
const html = await this.mjml2html(mjml);
|
||||
return html.html;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "buildEmail",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
sendEmail = async (to, subject, html, transportConfig) => {
|
||||
let config;
|
||||
if (typeof transportConfig !== "undefined") {
|
||||
config = transportConfig;
|
||||
} else {
|
||||
config = await this.settingsService.getDBSettings();
|
||||
}
|
||||
const {
|
||||
systemEmailHost,
|
||||
systemEmailPort,
|
||||
systemEmailSecure,
|
||||
systemEmailPool,
|
||||
systemEmailUser,
|
||||
systemEmailAddress,
|
||||
systemEmailPassword,
|
||||
systemEmailConnectionHost,
|
||||
systemEmailTLSServername,
|
||||
systemEmailIgnoreTLS,
|
||||
systemEmailRequireTLS,
|
||||
systemEmailRejectUnauthorized,
|
||||
} = config;
|
||||
|
||||
const emailConfig = {
|
||||
host: systemEmailHost,
|
||||
port: Number(systemEmailPort),
|
||||
secure: systemEmailSecure,
|
||||
auth: {
|
||||
user: systemEmailUser || systemEmailAddress,
|
||||
pass: systemEmailPassword,
|
||||
},
|
||||
name: systemEmailConnectionHost || "localhost",
|
||||
connectionTimeout: 5000,
|
||||
pool: systemEmailPool,
|
||||
tls: {
|
||||
rejectUnauthorized: systemEmailRejectUnauthorized,
|
||||
ignoreTLS: systemEmailIgnoreTLS,
|
||||
requireTLS: systemEmailRequireTLS,
|
||||
servername: systemEmailTLSServername,
|
||||
},
|
||||
};
|
||||
this.transporter = this.nodemailer.createTransport(emailConfig);
|
||||
|
||||
try {
|
||||
await this.transporter.verify();
|
||||
} catch (error) {
|
||||
this.logger.warn({
|
||||
message: "Email transporter verification failed",
|
||||
service: SERVICE_NAME,
|
||||
method: "verifyTransporter",
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const info = await this.transporter.sendMail({
|
||||
to: to,
|
||||
from: systemEmailAddress,
|
||||
subject: subject,
|
||||
html: html,
|
||||
});
|
||||
return info?.messageId;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "sendEmail",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default EmailService;
|
||||
@@ -0,0 +1,101 @@
|
||||
export class AppError extends Error {
|
||||
constructor(message, status = 500, service = null, method = null, details = null) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
this.service = service;
|
||||
this.method = method;
|
||||
this.details = details;
|
||||
|
||||
Error.captureStackTrace(this, this.constructor);
|
||||
}
|
||||
}
|
||||
|
||||
class ValidationError extends AppError {
|
||||
constructor(message, details = null, service = null, method = null) {
|
||||
super(message, 422, service, method, details);
|
||||
}
|
||||
}
|
||||
|
||||
class AuthenticationError extends AppError {
|
||||
constructor(message, details = null, service = null, method = null) {
|
||||
super(message, 401, service, method, details);
|
||||
}
|
||||
}
|
||||
|
||||
class AuthorizationError extends AppError {
|
||||
constructor(message, details = null, service = null, method = null) {
|
||||
super(message, 403, service, method, details);
|
||||
}
|
||||
}
|
||||
|
||||
class NotFoundError extends AppError {
|
||||
constructor(message, details = null, service = null, method = null) {
|
||||
super(message, 404, service, method, details);
|
||||
}
|
||||
}
|
||||
|
||||
class ConflictError extends AppError {
|
||||
constructor(message, details = null, service = null, method = null) {
|
||||
super(message, 409, service, method, details);
|
||||
}
|
||||
}
|
||||
|
||||
class DatabaseError extends AppError {
|
||||
constructor(message, details = null, service = null, method = null) {
|
||||
super(message, 500, service, method, details);
|
||||
}
|
||||
}
|
||||
|
||||
class BadRequestError extends AppError {
|
||||
constructor(message, details = null, service = null, method = null) {
|
||||
super(message, 400, service, method, details);
|
||||
}
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "ErrorService";
|
||||
class ErrorService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
constructor() {}
|
||||
|
||||
get serviceName() {
|
||||
return ErrorService.SERVICE_NAME;
|
||||
}
|
||||
|
||||
createError = (message, status = 500, service = null, method = null, details = null) => {
|
||||
return new AppError(message, status, service, method, details);
|
||||
};
|
||||
|
||||
createValidationError = (message, details = null, service = null, method = null) => {
|
||||
return new ValidationError(message, details, service, method);
|
||||
};
|
||||
|
||||
createAuthenticationError = (message = "Unauthorized", details = null, service = null, method = null) => {
|
||||
return new AuthenticationError(message, details, service, method);
|
||||
};
|
||||
|
||||
createAuthorizationError = (message, details = null, service = null, method = null) => {
|
||||
return new AuthorizationError(message, details, service, method);
|
||||
};
|
||||
|
||||
createNotFoundError = (message, details = null, service = null, method = null) => {
|
||||
return new NotFoundError(message, details, service, method);
|
||||
};
|
||||
|
||||
createConflictError = (message, details = null, service = null, method = null) => {
|
||||
return new ConflictError(message, details, service, method);
|
||||
};
|
||||
|
||||
createDatabaseError = (message, details = null, service = null, method = null) => {
|
||||
return new DatabaseError(message, details, service, method);
|
||||
};
|
||||
|
||||
createServerError = (message, details = null, service = null, method = null) => {
|
||||
return this.createError(message, 500, service, method, details);
|
||||
};
|
||||
|
||||
createBadRequestError = (message = "BadRequest", details = null, service = null, method = null) => {
|
||||
return new BadRequestError(message, details, service, method);
|
||||
};
|
||||
}
|
||||
|
||||
export default ErrorService;
|
||||
@@ -0,0 +1,580 @@
|
||||
import CacheableLookup from "cacheable-lookup";
|
||||
const SERVICE_NAME = "NetworkService";
|
||||
|
||||
class NetworkService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ axios, got, https, jmespath, GameDig, ping, logger, http, Docker, net, stringService, settingsService }) {
|
||||
this.TYPE_PING = "ping";
|
||||
this.TYPE_HTTP = "http";
|
||||
this.TYPE_PAGESPEED = "pagespeed";
|
||||
this.TYPE_HARDWARE = "hardware";
|
||||
this.TYPE_DOCKER = "docker";
|
||||
this.TYPE_PORT = "port";
|
||||
this.TYPE_GAME = "game";
|
||||
this.SERVICE_NAME = SERVICE_NAME;
|
||||
this.NETWORK_ERROR = 5000;
|
||||
this.PING_ERROR = 5001;
|
||||
this.axios = axios;
|
||||
this.https = https;
|
||||
this.jmespath = jmespath;
|
||||
this.GameDig = GameDig;
|
||||
this.ping = ping;
|
||||
this.logger = logger;
|
||||
this.http = http;
|
||||
this.Docker = Docker;
|
||||
this.net = net;
|
||||
this.stringService = stringService;
|
||||
this.settingsService = settingsService;
|
||||
|
||||
const cacheable = new CacheableLookup();
|
||||
|
||||
this.got = got.extend({
|
||||
dnsCache: cacheable,
|
||||
timeout: {
|
||||
request: 30000,
|
||||
},
|
||||
retry: { limit: 1 },
|
||||
});
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
async timeRequest(operation) {
|
||||
const start = process.hrtime.bigint();
|
||||
try {
|
||||
const response = await operation();
|
||||
const elapsedMs = Math.round(Number(process.hrtime.bigint() - start) / 1_000_000);
|
||||
return { response, responseTime: elapsedMs };
|
||||
} catch (error) {
|
||||
const elapsedMs = Math.round(Number(process.hrtime.bigint() - start) / 1_000_000);
|
||||
return { response: null, responseTime: elapsedMs, error };
|
||||
}
|
||||
}
|
||||
|
||||
// Main entry point
|
||||
async requestStatus(monitor) {
|
||||
const type = monitor?.type || "unknown";
|
||||
switch (type) {
|
||||
case this.TYPE_PING:
|
||||
return await this.requestPing(monitor);
|
||||
case this.TYPE_HTTP:
|
||||
return await this.requestHttp(monitor);
|
||||
case this.TYPE_PAGESPEED:
|
||||
return await this.requestPageSpeed(monitor);
|
||||
case this.TYPE_HARDWARE:
|
||||
return await this.requestHardware(monitor);
|
||||
case this.TYPE_DOCKER:
|
||||
return await this.requestDocker(monitor);
|
||||
case this.TYPE_PORT:
|
||||
return await this.requestPort(monitor);
|
||||
case this.TYPE_GAME:
|
||||
return await this.requestGame(monitor);
|
||||
default:
|
||||
return await this.handleUnsupportedType(type);
|
||||
}
|
||||
}
|
||||
|
||||
async requestPing(monitor) {
|
||||
try {
|
||||
if (!monitor?.url) {
|
||||
throw new Error("Monitor URL is required");
|
||||
}
|
||||
|
||||
const rawUrl = monitor.url;
|
||||
const sanitizedHost = rawUrl
|
||||
.replace(/^https?:\/\//, "")
|
||||
.replace(/\/.*$/, "")
|
||||
.replace(/:.*/, "");
|
||||
const { response, error } = await this.timeRequest(() => this.ping.promise.probe(sanitizedHost));
|
||||
|
||||
if (!response) {
|
||||
if (error) {
|
||||
throw error;
|
||||
}
|
||||
throw new Error("Ping failed - no result returned");
|
||||
}
|
||||
|
||||
const pingResponse = {
|
||||
monitorId: monitor._id,
|
||||
type: "ping",
|
||||
status: response.alive,
|
||||
code: 200,
|
||||
responseTime: response.time,
|
||||
message: "Success",
|
||||
payload: response,
|
||||
};
|
||||
|
||||
if (error) {
|
||||
pingResponse.status = false;
|
||||
pingResponse.code = 200;
|
||||
pingResponse.message = "Ping failed";
|
||||
return pingResponse;
|
||||
}
|
||||
|
||||
return pingResponse;
|
||||
} catch (err) {
|
||||
err.service = this.SERVICE_NAME;
|
||||
err.method = "requestPing";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async requestHttp(monitor) {
|
||||
const { url, secret, _id, teamId, type, ignoreTlsErrors, jsonPath, matchMethod, expectedValue } = monitor;
|
||||
const httpResponse = {
|
||||
monitorId: _id,
|
||||
teamId: teamId,
|
||||
type,
|
||||
};
|
||||
|
||||
try {
|
||||
if (!url) {
|
||||
throw new Error("Monitor URL is required");
|
||||
}
|
||||
const config = {
|
||||
headers: secret ? { Authorization: `Bearer ${secret}` } : undefined,
|
||||
};
|
||||
|
||||
if (ignoreTlsErrors) {
|
||||
config.agent = {
|
||||
https: new this.https.Agent({
|
||||
rejectUnauthorized: false,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const response = await this.got(url, config);
|
||||
|
||||
let payload;
|
||||
const contentType = response.headers["content-type"];
|
||||
|
||||
if (contentType && contentType.includes("application/json")) {
|
||||
try {
|
||||
payload = JSON.parse(response.body);
|
||||
} catch {
|
||||
payload = response.body;
|
||||
}
|
||||
} else {
|
||||
payload = response.body;
|
||||
}
|
||||
|
||||
httpResponse.code = response.statusCode;
|
||||
httpResponse.status = response.ok;
|
||||
httpResponse.message = response.statusMessage;
|
||||
httpResponse.responseTime = response.timings.phases.total || 0;
|
||||
httpResponse.payload = payload;
|
||||
httpResponse.timings = response.timings || {};
|
||||
|
||||
if (!expectedValue && !jsonPath) {
|
||||
return httpResponse;
|
||||
}
|
||||
|
||||
if (expectedValue && !jsonPath) {
|
||||
let ok = false;
|
||||
if (matchMethod === "equal") ok = payload === expectedValue;
|
||||
if (matchMethod === "include" && typeof payload === "string") ok = payload.includes(expectedValue);
|
||||
if (matchMethod === "regex" && typeof payload === "string") ok = new RegExp(expectedValue).test(payload);
|
||||
|
||||
if (ok === true) {
|
||||
return httpResponse;
|
||||
} else {
|
||||
httpResponse.code = 500;
|
||||
httpResponse.status = false;
|
||||
httpResponse.message = this.stringService.httpExpectedValueFail;
|
||||
return httpResponse;
|
||||
}
|
||||
}
|
||||
|
||||
if (jsonPath) {
|
||||
const contentType = response.headers["content-type"];
|
||||
const isJson = contentType?.includes("application/json");
|
||||
if (!isJson) {
|
||||
httpResponse.status = false;
|
||||
httpResponse.message = this.stringService.httpNotJson;
|
||||
return httpResponse;
|
||||
}
|
||||
try {
|
||||
const extracted = this.jmespath.search(payload, jsonPath);
|
||||
if (expectedValue) {
|
||||
let ok = false;
|
||||
if (matchMethod === "equal") ok = extracted === expectedValue;
|
||||
if (matchMethod === "include" && typeof extracted === "string") ok = extracted.includes(expectedValue);
|
||||
if (matchMethod === "regex" && typeof extracted === "string") ok = new RegExp(expectedValue).test(extracted);
|
||||
|
||||
if (ok) {
|
||||
httpResponse.extracted = extracted;
|
||||
return httpResponse;
|
||||
} else {
|
||||
httpResponse.status = false;
|
||||
httpResponse.code = 500;
|
||||
httpResponse.message = this.stringService.httpJsonPathFail;
|
||||
httpResponse.extracted = extracted;
|
||||
return httpResponse;
|
||||
}
|
||||
} else {
|
||||
const isFalsey = extracted === false || extracted === "false" || extracted === undefined || extracted === null;
|
||||
if (!isFalsey) {
|
||||
httpResponse.extracted = extracted;
|
||||
return httpResponse;
|
||||
} else {
|
||||
httpResponse.status = false;
|
||||
httpResponse.code = 500;
|
||||
httpResponse.message = this.stringService.httpJsonPathFail;
|
||||
httpResponse.extracted = extracted;
|
||||
return httpResponse;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
httpResponse.status = false;
|
||||
httpResponse.message = this.stringService.httpJsonPathError;
|
||||
return httpResponse;
|
||||
}
|
||||
}
|
||||
return httpResponse;
|
||||
} catch (err) {
|
||||
if (err.name === "HTTPError" || err.name === "RequestError") {
|
||||
httpResponse.code = err?.response?.statusCode || this.NETWORK_ERROR;
|
||||
httpResponse.status = false;
|
||||
httpResponse.message = err?.response?.statusCode || err.message;
|
||||
httpResponse.responseTime = err?.timings?.phases?.total || 0;
|
||||
httpResponse.payload = null;
|
||||
httpResponse.timings = err?.timings || {};
|
||||
return httpResponse;
|
||||
}
|
||||
err.service = this.SERVICE_NAME;
|
||||
err.method = "requestHttp";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async requestPageSpeed(monitor) {
|
||||
try {
|
||||
const url = monitor.url;
|
||||
if (!url) {
|
||||
throw new Error("Monitor URL is required");
|
||||
}
|
||||
let pageSpeedUrl = `https://pagespeedonline.googleapis.com/pagespeedonline/v5/runPagespeed?url=${url}&category=seo&category=accessibility&category=best-practices&category=performance`;
|
||||
const dbSettings = await this.settingsService.getDBSettings();
|
||||
if (dbSettings?.pagespeedApiKey) {
|
||||
pageSpeedUrl += `&key=${dbSettings.pagespeedApiKey}`;
|
||||
} else {
|
||||
this.logger.warn({
|
||||
message: "PageSpeed API key not found, job not executed",
|
||||
service: this.SERVICE_NAME,
|
||||
method: "requestPagespeed",
|
||||
details: { url },
|
||||
});
|
||||
}
|
||||
return await this.requestHttp({
|
||||
...monitor,
|
||||
url: pageSpeedUrl,
|
||||
});
|
||||
} catch (err) {
|
||||
err.service = this.SERVICE_NAME;
|
||||
err.method = "requestPageSpeed";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async requestHardware(monitor) {
|
||||
try {
|
||||
return await this.requestHttp(monitor);
|
||||
} catch (err) {
|
||||
err.service = this.SERVICE_NAME;
|
||||
err.method = "requestHardware";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async requestDocker(monitor) {
|
||||
try {
|
||||
if (!monitor.url) {
|
||||
throw new Error("Monitor URL is required");
|
||||
}
|
||||
|
||||
const docker = new this.Docker({
|
||||
socketPath: "/var/run/docker.sock",
|
||||
handleError: true, // Enable error handling
|
||||
});
|
||||
|
||||
const dockerResponse = {
|
||||
monitorId: monitor._id,
|
||||
type: monitor.type,
|
||||
};
|
||||
|
||||
const containers = await docker.listContainers({ all: true });
|
||||
|
||||
// Normalize input: strip leading slashes and convert to lowercase for comparison
|
||||
const normalizedInput = monitor.url.replace(/^\/+/, "").toLowerCase();
|
||||
|
||||
// Priority-based matching to avoid ambiguity:
|
||||
// 1. Exact full ID match (64-char)
|
||||
let exactIdMatch = containers.find((c) => c.Id.toLowerCase() === normalizedInput);
|
||||
|
||||
// 2. Exact container name match (case-insensitive)
|
||||
let exactNameMatch = containers.find((c) =>
|
||||
c.Names.some((name) => {
|
||||
const cleanName = name.replace(/^\/+/, "").toLowerCase();
|
||||
return cleanName === normalizedInput;
|
||||
})
|
||||
);
|
||||
|
||||
// 3. Partial ID match (fallback for backwards compatibility)
|
||||
let partialIdMatch = containers.find((c) => c.Id.toLowerCase().startsWith(normalizedInput));
|
||||
|
||||
// Select container based on priority
|
||||
let targetContainer = exactIdMatch || exactNameMatch || partialIdMatch;
|
||||
|
||||
// Return negative response if no container
|
||||
if (!targetContainer) {
|
||||
this.logger.warn({
|
||||
message: `No container found for "${monitor.url}".`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "requestDocker",
|
||||
details: { url: monitor.url },
|
||||
});
|
||||
|
||||
dockerResponse.code = 404;
|
||||
dockerResponse.status = false;
|
||||
dockerResponse.message = this.stringService.dockerNotFound;
|
||||
return dockerResponse;
|
||||
}
|
||||
|
||||
// Return negative response if ambiguous matches exist
|
||||
const matchTypes = [];
|
||||
if (exactIdMatch) matchTypes.push("exact ID");
|
||||
if (exactNameMatch) matchTypes.push("exact name");
|
||||
if (partialIdMatch && !exactIdMatch) matchTypes.push("partial ID");
|
||||
|
||||
if (matchTypes.length > 1) {
|
||||
this.logger.warn({
|
||||
message: `Ambiguous container match for "${monitor.url}". Matched by: ${matchTypes.join(", ")}. Using ${exactIdMatch ? "exact ID" : exactNameMatch ? "exact name" : "partial ID"} match.`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "requestDocker",
|
||||
details: { url: monitor.url },
|
||||
});
|
||||
dockerResponse.status = 404;
|
||||
dockerResponse.status = false;
|
||||
dockerResponse.message = `Ambiguous container match for "${monitor.url}". Matched by: ${matchTypes.join(", ")}. Using ${exactIdMatch ? "exact ID" : exactNameMatch ? "exact name" : "partial ID"} match.`;
|
||||
return dockerResponse;
|
||||
}
|
||||
|
||||
const container = docker.getContainer(targetContainer.Id);
|
||||
const { response, responseTime, error } = await this.timeRequest(() => container.inspect());
|
||||
|
||||
dockerResponse.responseTime = responseTime;
|
||||
dockerResponse.status = response?.State?.Status === "running" ? true : false;
|
||||
dockerResponse.code = 200;
|
||||
dockerResponse.message = "Docker container status fetched successfully";
|
||||
|
||||
if (error) {
|
||||
dockerResponse.status = false;
|
||||
dockerResponse.code = error.statusCode || this.NETWORK_ERROR;
|
||||
dockerResponse.message = error.reason || "Failed to fetch Docker container information";
|
||||
return dockerResponse;
|
||||
}
|
||||
|
||||
return dockerResponse;
|
||||
} catch (err) {
|
||||
err.service = this.SERVICE_NAME;
|
||||
err.method = "requestDocker";
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async requestPort(monitor) {
|
||||
try {
|
||||
const { url, port } = monitor;
|
||||
const { response, responseTime, error } = await this.timeRequest(async () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = this.net.createConnection(
|
||||
{
|
||||
host: url,
|
||||
port,
|
||||
},
|
||||
() => {
|
||||
socket.end();
|
||||
socket.destroy();
|
||||
resolve({ success: true });
|
||||
}
|
||||
);
|
||||
|
||||
socket.setTimeout(5000);
|
||||
socket.on("timeout", () => {
|
||||
socket.destroy();
|
||||
reject(new Error("Connection timeout"));
|
||||
});
|
||||
|
||||
socket.on("error", (err) => {
|
||||
socket.destroy();
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const portResponse = {
|
||||
code: 200,
|
||||
status: response.success,
|
||||
message: this.stringService.portSuccess,
|
||||
monitorId: monitor._id,
|
||||
type: monitor.type,
|
||||
responseTime: responseTime,
|
||||
};
|
||||
|
||||
if (error) {
|
||||
portResponse.code = this.NETWORK_ERROR;
|
||||
portResponse.status = false;
|
||||
portResponse.message = this.stringService.portFail;
|
||||
return portResponse;
|
||||
}
|
||||
|
||||
return portResponse;
|
||||
} catch (error) {
|
||||
error.service = this.SERVICE_NAME;
|
||||
error.method = "requestTCP";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async requestGame(monitor) {
|
||||
try {
|
||||
const { url, port, gameId } = monitor;
|
||||
|
||||
const gameResponse = {
|
||||
code: 200,
|
||||
status: true,
|
||||
message: "Success",
|
||||
monitorId: monitor._id,
|
||||
type: "game",
|
||||
};
|
||||
|
||||
const state = await this.GameDig.query({
|
||||
type: gameId,
|
||||
host: url,
|
||||
port: port,
|
||||
}).catch((error) => {
|
||||
this.logger.warn({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "requestGame",
|
||||
details: { url, port, gameId },
|
||||
});
|
||||
});
|
||||
|
||||
if (!state) {
|
||||
gameResponse.code = this.NETWORK_ERROR;
|
||||
gameResponse.status = false;
|
||||
gameResponse.message = "No response";
|
||||
return gameResponse;
|
||||
}
|
||||
|
||||
gameResponse.responseTime = state.ping;
|
||||
gameResponse.payload = state;
|
||||
return gameResponse;
|
||||
} catch (error) {
|
||||
error.service = this.SERVICE_NAME;
|
||||
error.method = "requestPing";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
async handleUnsupportedType(type) {
|
||||
const err = new Error(`Unsupported type: ${type}`);
|
||||
err.service = this.SERVICE_NAME;
|
||||
err.method = "getStatus";
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Other network requests unrelated to monitoring:
|
||||
async requestWebhook(type, url, body) {
|
||||
try {
|
||||
const response = await this.axios.post(url, body, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
type: "webhook",
|
||||
status: true,
|
||||
code: response.status,
|
||||
message: `Successfully sent ${type} notification`,
|
||||
payload: response.data,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.warn({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "requestWebhook",
|
||||
});
|
||||
|
||||
return {
|
||||
type: "webhook",
|
||||
status: false,
|
||||
code: error.response?.status || this.NETWORK_ERROR,
|
||||
message: `Failed to send ${type} notification`,
|
||||
payload: error.response?.data,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async requestPagerDuty({ message, routingKey, monitorUrl }) {
|
||||
try {
|
||||
const response = await this.axios.post(`https://events.pagerduty.com/v2/enqueue`, {
|
||||
routing_key: routingKey,
|
||||
event_action: "trigger",
|
||||
payload: {
|
||||
summary: message,
|
||||
severity: "critical",
|
||||
source: monitorUrl,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
|
||||
if (response?.data?.status !== "success") return false;
|
||||
return true;
|
||||
} catch (error) {
|
||||
error.details = error.response?.data;
|
||||
error.service = this.SERVICE_NAME;
|
||||
error.method = "requestPagerDuty";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async requestMatrix({ homeserverUrl, accessToken, roomId, message }) {
|
||||
try {
|
||||
const url = `${homeserverUrl}/_matrix/client/v3/rooms/${roomId}/send/m.room.message?access_token=${accessToken}`;
|
||||
const body = {
|
||||
msgtype: "m.text",
|
||||
body: message,
|
||||
format: "org.matrix.custom.html",
|
||||
formatted_body: message,
|
||||
};
|
||||
const response = await this.axios.post(url, body, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
status: true,
|
||||
code: response.status,
|
||||
message: "Successfully sent Matrix notification",
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.warn({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "requestMatrix",
|
||||
});
|
||||
|
||||
return {
|
||||
status: false,
|
||||
code: error.response?.status || this.NETWORK_ERROR,
|
||||
message: "Failed to send Matrix notification",
|
||||
payload: error.response?.data,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default NetworkService;
|
||||
@@ -0,0 +1,34 @@
|
||||
const SERVICE_NAME = "Matrix";
|
||||
|
||||
class Matrix {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ networkService, logger }) {
|
||||
this.networkService = networkService;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return Matrix.SERVICE_NAME;
|
||||
}
|
||||
|
||||
async send({ friendlyName, homeserverUrl, accessToken, roomId, message, monitorName }) {
|
||||
const title = `Checkmate status for ${monitorName}`;
|
||||
const formattedMessage = `## ${title}\n${message}`;
|
||||
try {
|
||||
await this.networkService.requestMatrix({
|
||||
homeserverUrl,
|
||||
accessToken,
|
||||
roomId,
|
||||
message: formattedMessage,
|
||||
});
|
||||
this.logger.info(`Successfully sent Matrix notification for ${friendlyName}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to send Matrix notification for ${friendlyName}: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default Matrix;
|
||||
@@ -0,0 +1,128 @@
|
||||
const SERVICE_NAME = "NotificationService";
|
||||
import Matrix from "./notificationProviders/matrix.js";
|
||||
|
||||
class NotificationService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ emailService, db, logger, networkService, stringService, notificationUtils }) {
|
||||
this.emailService = emailService;
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.networkService = networkService;
|
||||
this.stringService = stringService;
|
||||
this.notificationUtils = notificationUtils;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return NotificationService.SERVICE_NAME;
|
||||
}
|
||||
|
||||
sendNotification = async ({ notification, subject, content, html, discordContent = null, webhookBody = null }) => {
|
||||
const { type, address } = notification;
|
||||
|
||||
if (type === "email") {
|
||||
const messageId = await this.emailService.sendEmail(address, subject, html);
|
||||
if (!messageId) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Create a body for webhooks
|
||||
let body = { text: content };
|
||||
if (type === "discord") {
|
||||
body = !discordContent ? { content } : discordContent;
|
||||
}
|
||||
if (type === "webhook") {
|
||||
body = !webhookBody ? { content } : webhookBody;
|
||||
}
|
||||
if (type === "slack" || type === "discord" || type === "webhook") {
|
||||
const response = await this.networkService.requestWebhook(type, address, body);
|
||||
return response.status;
|
||||
}
|
||||
if (type === "pager_duty") {
|
||||
const response = await this.networkService.requestPagerDuty({
|
||||
message: content,
|
||||
monitorUrl: subject,
|
||||
routingKey: address,
|
||||
});
|
||||
|
||||
return response;
|
||||
}
|
||||
if (type === "matrix") {
|
||||
const { friendlyName, homeserverUrl, accessToken, roomId } = notification;
|
||||
const monitorName = subject;
|
||||
const message = content;
|
||||
const matrix = new Matrix({ networkService: this.networkService, logger: this.logger });
|
||||
const success = await matrix.send({ friendlyName, homeserverUrl, accessToken, roomId, message, monitorName });
|
||||
return success;
|
||||
}
|
||||
};
|
||||
|
||||
async handleNotifications(networkResponse) {
|
||||
const { monitor, statusChanged, prevStatus } = networkResponse;
|
||||
const { type } = monitor;
|
||||
if (type !== "hardware" && statusChanged === false) return false;
|
||||
// if prevStatus is undefined, monitor is resuming, we're done
|
||||
if (type !== "hardware" && prevStatus === undefined) return false;
|
||||
|
||||
const notificationIDs = networkResponse.monitor?.notifications ?? [];
|
||||
if (notificationIDs.length === 0) return false;
|
||||
if (networkResponse.monitor.type === "hardware") {
|
||||
const thresholds = networkResponse?.monitor?.thresholds;
|
||||
|
||||
if (thresholds === undefined) return false; // No thresholds set, we're done
|
||||
const metrics = networkResponse?.payload?.data ?? null;
|
||||
if (metrics === null) return false; // No metrics, we're done
|
||||
|
||||
const [alerts, discordContent] = await this.notificationUtils.buildHardwareAlerts(networkResponse);
|
||||
if (alerts.length === 0) return false;
|
||||
|
||||
const { subject, html } = await this.notificationUtils.buildHardwareEmail(networkResponse, alerts);
|
||||
const content = await this.notificationUtils.buildHardwareNotificationMessage(alerts, monitor);
|
||||
const webhookBody = await this.notificationUtils.buildHardwareWebhookBody(alerts, monitor);
|
||||
const success = await this.notifyAll({ notificationIDs, subject, html, content, discordContent, webhookBody });
|
||||
return success;
|
||||
}
|
||||
|
||||
// Status monitors
|
||||
const { subject, html } = await this.notificationUtils.buildStatusEmail(networkResponse);
|
||||
const [content, discordContent] = await this.notificationUtils.buildWebhookMessage(networkResponse);
|
||||
const success = this.notifyAll({ notificationIDs, subject, html, content, discordContent });
|
||||
return success;
|
||||
}
|
||||
|
||||
async notifyAll({ notificationIDs, subject, html, content, discordContent = null, webhookBody = null }) {
|
||||
const notifications = await this.db.notificationModule.getNotificationsByIds(notificationIDs);
|
||||
// Map each notification to a test promise
|
||||
const promises = notifications.map(async (notification) => {
|
||||
try {
|
||||
await this.sendNotification({ notification, subject, content, html, discordContent, webhookBody });
|
||||
return true;
|
||||
} catch (err) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
return results.every((r) => r === true);
|
||||
}
|
||||
|
||||
async getTestNotification() {
|
||||
const html = await this.notificationUtils.buildTestEmail();
|
||||
const content = "This is a test notification";
|
||||
const subject = "Test Notification";
|
||||
return { subject, html, content };
|
||||
}
|
||||
|
||||
async testAllNotifications(notificationIDs) {
|
||||
const { subject, html, content } = await this.getTestNotification();
|
||||
return this.notifyAll({ notificationIDs, subject, html, content });
|
||||
}
|
||||
|
||||
async sendTestNotification(notification) {
|
||||
const { subject, html, content } = await this.getTestNotification();
|
||||
const success = await this.sendNotification({ notification, subject, content, html });
|
||||
return success;
|
||||
}
|
||||
}
|
||||
|
||||
export default NotificationService;
|
||||
@@ -0,0 +1,205 @@
|
||||
const SERVICE_NAME = "NotificationUtils";
|
||||
|
||||
class NotificationUtils {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ stringService, emailService, settingsService }) {
|
||||
this.stringService = stringService;
|
||||
this.emailService = emailService;
|
||||
this.settingsService = settingsService;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return NotificationUtils.SERVICE_NAME;
|
||||
}
|
||||
|
||||
buildTestEmail = async () => {
|
||||
const context = { testName: "Monitoring System" };
|
||||
const html = await this.emailService.buildEmail("testEmailTemplate", context);
|
||||
return html;
|
||||
};
|
||||
|
||||
buildStatusEmail = async (networkResponse) => {
|
||||
const { monitor, status, prevStatus } = networkResponse;
|
||||
const template = prevStatus === false ? "serverIsUpTemplate" : "serverIsDownTemplate";
|
||||
const context = { monitor: monitor.name, url: monitor.url };
|
||||
const subject = `Monitor ${monitor.name} is ${status === true ? "up" : "down"}`;
|
||||
const html = await this.emailService.buildEmail(template, context);
|
||||
return { subject, html };
|
||||
};
|
||||
|
||||
buildWebhookMessage = (networkResponse) => {
|
||||
const { monitor, status, code, timestamp } = networkResponse;
|
||||
// Format timestamp using the local system timezone
|
||||
const formatTime = (timestamp) => {
|
||||
const date = new Date(timestamp);
|
||||
|
||||
// Get timezone abbreviation and format the date
|
||||
const timeZoneAbbr = date.toLocaleTimeString("en-US", { timeZoneName: "short" }).split(" ").pop();
|
||||
|
||||
// Format the date with readable format
|
||||
return (
|
||||
date
|
||||
.toLocaleString("en-US", {
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
hour12: false,
|
||||
})
|
||||
.replace(/(\d+)\/(\d+)\/(\d+),\s/, "$3-$1-$2 ") +
|
||||
" " +
|
||||
timeZoneAbbr
|
||||
);
|
||||
};
|
||||
|
||||
// Get formatted time
|
||||
const formattedTime = timestamp ? formatTime(timestamp) : formatTime(new Date().getTime());
|
||||
|
||||
// Create different messages based on status with extra spacing
|
||||
let messageText;
|
||||
if (status === true) {
|
||||
messageText = this.stringService.monitorUpAlert
|
||||
.replace("{monitorName}", monitor.name)
|
||||
.replace("{time}", formattedTime)
|
||||
.replace("{code}", code || "Unknown");
|
||||
} else {
|
||||
messageText = this.stringService.monitorDownAlert
|
||||
.replace("{monitorName}", monitor.name)
|
||||
.replace("{time}", formattedTime)
|
||||
.replace("{code}", code || "Unknown");
|
||||
}
|
||||
const dn = this.stringService.discordNotification;
|
||||
let discordMessageText = {
|
||||
embeds: [
|
||||
{
|
||||
title: status
|
||||
? dn.uptimeAlert.replace("{monitorName}", monitor?.name ?? dn.unknown)
|
||||
: dn.downtimeAlert.replace("{monitorName}", monitor?.name ?? dn.unknown),
|
||||
color: status ? 5763719 : 15548997,
|
||||
|
||||
fields: [
|
||||
{ name: dn.monitor, value: monitor?.name ?? dn.unknown, inline: true },
|
||||
{ name: dn.status, value: status ? dn.up : dn.down, inline: true },
|
||||
{ name: dn.statusCode, value: String(code ?? dn.unknown), inline: true },
|
||||
{ name: dn.time, value: formattedTime, inline: true },
|
||||
{ name: dn.url, value: monitor?.url ?? dn.unknown, inline: false },
|
||||
],
|
||||
footer: { text: dn.checkmate },
|
||||
},
|
||||
],
|
||||
};
|
||||
return [messageText, discordMessageText];
|
||||
};
|
||||
|
||||
buildHardwareAlerts = async (networkResponse) => {
|
||||
const monitor = networkResponse?.monitor;
|
||||
const thresholds = networkResponse?.monitor?.thresholds;
|
||||
const { usage_cpu: cpuThreshold = -1, usage_memory: memoryThreshold = -1, usage_disk: diskThreshold = -1 } = thresholds;
|
||||
|
||||
const metrics = networkResponse?.payload?.data;
|
||||
const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics;
|
||||
|
||||
const { clientHost } = this.settingsService.getSettings();
|
||||
|
||||
const alerts = {
|
||||
cpu: cpuThreshold !== -1 && cpuUsage > cpuThreshold ? true : false,
|
||||
memory: memoryThreshold !== -1 && memoryUsage > memoryThreshold ? true : false,
|
||||
disk: disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold) ?? false,
|
||||
};
|
||||
|
||||
const alertsToSend = [];
|
||||
const discordEmbeds = [];
|
||||
const monitorInfoFields = [
|
||||
{ name: "Monitor", value: monitor.name, inline: true },
|
||||
{ name: "URL", value: monitor.url, inline: false },
|
||||
];
|
||||
const goToIncidentField = { name: `Go to incident`, value: `${clientHost}/infrastructure/${monitor._id}` };
|
||||
const formatDiscordAlert = {
|
||||
cpu: () => ({
|
||||
title: "CPU alert",
|
||||
description: `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
|
||||
color: 15548997,
|
||||
fields: [...monitorInfoFields, goToIncidentField],
|
||||
footer: { text: "Checkmate" },
|
||||
}),
|
||||
|
||||
memory: () => ({
|
||||
title: "Memory alert",
|
||||
description: `Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
|
||||
color: 15548997,
|
||||
fields: [...monitorInfoFields, goToIncidentField],
|
||||
footer: { text: "Checkmate" },
|
||||
}),
|
||||
|
||||
disk: () => ({
|
||||
title: "Disk alert",
|
||||
description: `Your current disk usage is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
|
||||
color: 15548997,
|
||||
footer: { text: "Checkmate" },
|
||||
fields: [
|
||||
...monitorInfoFields,
|
||||
...(Array.isArray(disk) ? disk : []).map((d, idx) => ({
|
||||
name: `Disk ${idx}`,
|
||||
value: `${(d?.usage_percent * 100).toFixed(0)}%`,
|
||||
inline: true,
|
||||
})),
|
||||
goToIncidentField,
|
||||
],
|
||||
}),
|
||||
};
|
||||
const alertTypes = ["cpu", "memory", "disk"];
|
||||
for (const type of alertTypes) {
|
||||
// Iterate over each alert type to see if any need to be decremented
|
||||
if (alerts[type] === true) {
|
||||
monitor[`${type}AlertThreshold`]--; // Decrement threshold if an alert is triggered
|
||||
|
||||
if (monitor[`${type}AlertThreshold`] <= 0) {
|
||||
// If threshold drops below 0, reset and send notification
|
||||
monitor[`${type}AlertThreshold`] = monitor.alertThreshold;
|
||||
|
||||
const formatAlert = {
|
||||
cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
|
||||
memory: () =>
|
||||
`Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
|
||||
disk: () =>
|
||||
`Your current disk usage: ${disk
|
||||
.map((d, idx) => `(Disk${idx}: ${(d.usage_percent * 100).toFixed(0)}%)`)
|
||||
.join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
|
||||
};
|
||||
alertsToSend.push(formatAlert[type]());
|
||||
discordEmbeds.push(formatDiscordAlert[type]());
|
||||
}
|
||||
}
|
||||
}
|
||||
await monitor.save();
|
||||
const discordPayload = discordEmbeds.length ? { embeds: discordEmbeds } : null;
|
||||
return [alertsToSend, discordPayload];
|
||||
};
|
||||
|
||||
buildHardwareEmail = async (networkResponse, alerts) => {
|
||||
const { monitor } = networkResponse;
|
||||
const template = "hardwareIncidentTemplate";
|
||||
const context = { monitor: monitor.name, url: monitor.url, alerts };
|
||||
const subject = `Monitor ${monitor.name} infrastructure alerts`;
|
||||
const html = await this.emailService.buildEmail(template, context);
|
||||
return { subject, html };
|
||||
};
|
||||
|
||||
buildHardwareNotificationMessage = (alerts, monitor) => {
|
||||
const { clientHost } = this.settingsService.getSettings();
|
||||
const alertsHeader = [`Monitor: ${monitor.name}`, `URL: ${monitor.url}`];
|
||||
const alertFooter = [`Go to incident: ${clientHost}/infrastructure/${monitor._id}`];
|
||||
const alertText = alerts.length > 0 ? [...alertsHeader, ...alerts, ...alertFooter] : [];
|
||||
return alertText.map((alert) => alert).join("\n");
|
||||
};
|
||||
buildHardwareWebhookBody = (alerts, monitor) => {
|
||||
const content = alerts.map((alert) => alert).join("\n");
|
||||
const body = { text: content, name: monitor.name, url: monitor.url };
|
||||
return body;
|
||||
};
|
||||
}
|
||||
|
||||
export default NotificationUtils;
|
||||
+434
@@ -0,0 +1,434 @@
|
||||
import MonitorStats from "../../db/models/MonitorStats.js";
|
||||
import Check from "../../db/models/Check.js";
|
||||
const SERVICE_NAME = "StatusService";
|
||||
|
||||
class StatusService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
/**
|
||||
* @param {{
|
||||
* buffer: import("./bufferService.js").BufferService
|
||||
* incidentService: import("../business/incidentService.js").IncidentService
|
||||
* }}
|
||||
*/ constructor({ db, logger, buffer, incidentService }) {
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.buffer = buffer;
|
||||
this.incidentService = incidentService;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return StatusService.SERVICE_NAME;
|
||||
}
|
||||
|
||||
async updateRunningStats({ monitor, networkResponse }) {
|
||||
try {
|
||||
const monitorId = monitor._id;
|
||||
const { responseTime, status } = networkResponse;
|
||||
// Get stats
|
||||
let stats = await MonitorStats.findOne({ monitorId });
|
||||
if (!stats) {
|
||||
stats = new MonitorStats({
|
||||
monitorId,
|
||||
avgResponseTime: 0,
|
||||
totalChecks: 0,
|
||||
totalUpChecks: 0,
|
||||
totalDownChecks: 0,
|
||||
uptimePercentage: 0,
|
||||
lastCheck: null,
|
||||
});
|
||||
}
|
||||
|
||||
// Update stats
|
||||
|
||||
// Last response time
|
||||
stats.lastResponseTime = responseTime;
|
||||
|
||||
// Avg response time:
|
||||
let avgResponseTime = stats.avgResponseTime;
|
||||
if (typeof responseTime !== "undefined" && responseTime !== null) {
|
||||
if (avgResponseTime === 0) {
|
||||
avgResponseTime = responseTime;
|
||||
} else {
|
||||
avgResponseTime = (avgResponseTime * (stats.totalChecks - 1) + responseTime) / stats.totalChecks;
|
||||
}
|
||||
}
|
||||
stats.avgResponseTime = avgResponseTime;
|
||||
|
||||
// Total checks
|
||||
stats.totalChecks++;
|
||||
if (status === true) {
|
||||
stats.totalUpChecks++;
|
||||
// Update the timeSinceLastFailure if needed
|
||||
if (stats.timeOfLastFailure === 0) {
|
||||
stats.timeOfLastFailure = new Date().getTime();
|
||||
}
|
||||
} else {
|
||||
stats.totalDownChecks++;
|
||||
stats.timeOfLastFailure = 0;
|
||||
}
|
||||
|
||||
// Calculate uptime percentage
|
||||
let uptimePercentage;
|
||||
if (stats.totalChecks > 0) {
|
||||
uptimePercentage = stats.totalUpChecks / stats.totalChecks;
|
||||
} else {
|
||||
uptimePercentage = status === true ? 100 : 0;
|
||||
}
|
||||
stats.uptimePercentage = uptimePercentage;
|
||||
|
||||
// latest check
|
||||
stats.lastCheckTimestamp = new Date().getTime();
|
||||
|
||||
await stats.save();
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
message: error.message,
|
||||
method: "updateRunningStats",
|
||||
stack: error.stack,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
getStatusString = (status) => {
|
||||
if (status === true) return "up";
|
||||
if (status === false) return "down";
|
||||
return "unknown";
|
||||
};
|
||||
|
||||
/**
|
||||
* Saves check if needed and adds to incident buffer
|
||||
* Removes check from checks buffer if it was saved immediately
|
||||
*
|
||||
* @param {Object} check - The check object
|
||||
* @param {Object} monitor - The monitor object
|
||||
* @param {string} action - The incident action ("create" or "resolve")
|
||||
* @param {string} errorContext - Context for error messages
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
handleIncidentForCheck = async (check, monitor, action, errorContext = "incident handling") => {
|
||||
try {
|
||||
let savedCheck = check;
|
||||
|
||||
if (!check._id) {
|
||||
try {
|
||||
const checkModel = new Check(check);
|
||||
savedCheck = await checkModel.save();
|
||||
|
||||
this.buffer.removeCheckFromBuffer(check);
|
||||
} catch (checkError) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "handleIncidentForCheck",
|
||||
message: `Failed to save check immediately for ${errorContext}: ${checkError.message}`,
|
||||
monitorId: monitor._id,
|
||||
stack: checkError.stack,
|
||||
});
|
||||
savedCheck = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (savedCheck && savedCheck._id) {
|
||||
try {
|
||||
this.buffer.addIncidentToBuffer({ monitor, check: savedCheck, action });
|
||||
} catch (incidentError) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "handleIncidentForCheck",
|
||||
message: `Failed to add incident to buffer for ${errorContext}: ${incidentError.message}`,
|
||||
monitorId: monitor._id,
|
||||
action,
|
||||
stack: incidentError.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "handleIncidentForCheck",
|
||||
message: `Error in ${errorContext}: ${error.message}`,
|
||||
monitorId: monitor?._id,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Updates the status of a monitor based on the network response.
|
||||
*
|
||||
* @param {Object} networkResponse - The network response containing monitorId and status.
|
||||
* @param {string} networkResponse.monitorId - The ID of the monitor.
|
||||
* @param {string} networkResponse.status - The new status of the monitor.
|
||||
* @returns {Promise<Object>} - A promise that resolves to an object containinfg the monitor, statusChanged flag, and previous status if the status changed, or false if an error occurred.
|
||||
* @returns {Promise<Object>} returnObject - The object returned by the function.
|
||||
* @returns {Object} returnObject.monitor - The monitor object.
|
||||
* @returns {boolean} returnObject.statusChanged - Flag indicating if the status has changed.
|
||||
* @returns {boolean} returnObject.prevStatus - The previous status of the monitor
|
||||
*/
|
||||
updateStatus = async (networkResponse) => {
|
||||
const check = this.buildCheck(networkResponse);
|
||||
await this.insertCheck(check);
|
||||
try {
|
||||
const { monitorId, status, code } = networkResponse;
|
||||
const monitor = await this.db.monitorModule.getMonitorById(monitorId);
|
||||
|
||||
// Update running stats
|
||||
this.updateRunningStats({ monitor, networkResponse });
|
||||
|
||||
// If the status window size has changed, empty
|
||||
while (monitor.statusWindow.length > monitor.statusWindowSize) {
|
||||
monitor.statusWindow.shift();
|
||||
}
|
||||
|
||||
// Update status sliding window
|
||||
monitor.statusWindow.push(status);
|
||||
if (monitor.statusWindow.length > monitor.statusWindowSize) {
|
||||
monitor.statusWindow.shift();
|
||||
}
|
||||
|
||||
if (monitor.status === undefined || monitor.status === null) {
|
||||
monitor.status = status;
|
||||
}
|
||||
|
||||
let newStatus = monitor.status;
|
||||
let statusChanged = false;
|
||||
const prevStatus = monitor.status;
|
||||
|
||||
// Return early if not enough data points
|
||||
if (monitor.statusWindow.length < monitor.statusWindowSize) {
|
||||
await monitor.save();
|
||||
return {
|
||||
monitor,
|
||||
statusChanged: false,
|
||||
prevStatus,
|
||||
code,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// Check if threshold has been met
|
||||
const failures = monitor.statusWindow.filter((s) => s === false).length;
|
||||
const failureRate = (failures / monitor.statusWindow.length) * 100;
|
||||
|
||||
// If threshold has been met and the monitor is not already down, mark down:
|
||||
if (failureRate >= monitor.statusWindowThreshold && monitor.status !== false) {
|
||||
newStatus = false;
|
||||
statusChanged = true;
|
||||
}
|
||||
// If the failure rate is below the threshold and the monitor is down, recover:
|
||||
else if (failureRate < monitor.statusWindowThreshold && monitor.status === false) {
|
||||
newStatus = true;
|
||||
statusChanged = true;
|
||||
}
|
||||
|
||||
if (statusChanged) {
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
message: `${monitor.name} went from ${this.getStatusString(prevStatus)} to ${this.getStatusString(newStatus)}`,
|
||||
prevStatus,
|
||||
newStatus,
|
||||
});
|
||||
|
||||
if (newStatus === false) {
|
||||
await this.handleIncidentForCheck(check, monitor, "create", "status change to down");
|
||||
} else if (prevStatus === false) {
|
||||
await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up");
|
||||
}
|
||||
}
|
||||
|
||||
if (monitor.status === false && !statusChanged) {
|
||||
try {
|
||||
const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor._id);
|
||||
|
||||
let calculatedFailureRate = failureRate;
|
||||
|
||||
if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) {
|
||||
try {
|
||||
const checksAfterResolution = await Check.find({
|
||||
monitorId: monitor._id,
|
||||
createdAt: { $gt: lastManuallyResolvedIncident.endTime },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.limit(monitor.statusWindowSize)
|
||||
.select("status")
|
||||
.lean();
|
||||
|
||||
if (checksAfterResolution.length > 0) {
|
||||
const checksStatuses = checksAfterResolution.map((c) => c.status);
|
||||
const failuresAfterResolution = checksStatuses.filter((s) => s === false).length;
|
||||
calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100;
|
||||
} else {
|
||||
calculatedFailureRate = 0;
|
||||
}
|
||||
} catch (checkQueryError) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "updateStatus",
|
||||
message: `Failed to query checks after manual resolution: ${checkQueryError.message}`,
|
||||
monitorId: monitor._id,
|
||||
stack: checkQueryError.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (calculatedFailureRate >= monitor.statusWindowThreshold) {
|
||||
await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change");
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "updateStatus",
|
||||
message: `Error handling threshold check without status change: ${error.message}`,
|
||||
monitorId: monitor._id,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
monitor.status = newStatus;
|
||||
await monitor.save();
|
||||
|
||||
return {
|
||||
monitor,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
code,
|
||||
timestamp: new Date().getTime(),
|
||||
};
|
||||
} catch (error) {
|
||||
error.service = this.SERVICE_NAME;
|
||||
error.method = "updateStatus";
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Builds a check object from the network response.
|
||||
*
|
||||
* @param {Object} networkResponse - The network response object.
|
||||
* @param {string} networkResponse.monitorId - The monitor ID.
|
||||
* @param {string} networkResponse.type - The type of the response.
|
||||
* @param {string} networkResponse.status - The status of the response.
|
||||
* @param {number} networkResponse.responseTime - The response time.
|
||||
* @param {number} networkResponse.code - The status code.
|
||||
* @param {string} networkResponse.message - The message.
|
||||
* @param {Object} networkResponse.payload - The payload of the response.
|
||||
* @returns {Object} The check object.
|
||||
*/
|
||||
buildCheck = (networkResponse) => {
|
||||
const {
|
||||
monitorId,
|
||||
teamId,
|
||||
type,
|
||||
status,
|
||||
responseTime,
|
||||
code,
|
||||
message,
|
||||
payload,
|
||||
first_byte_took,
|
||||
body_read_took,
|
||||
dns_took,
|
||||
conn_took,
|
||||
connect_took,
|
||||
tls_took,
|
||||
timings,
|
||||
} = networkResponse;
|
||||
|
||||
const check = {
|
||||
metadata: {
|
||||
monitorId,
|
||||
teamId,
|
||||
type,
|
||||
},
|
||||
status,
|
||||
statusCode: code,
|
||||
responseTime,
|
||||
timings: timings || {},
|
||||
message,
|
||||
first_byte_took,
|
||||
body_read_took,
|
||||
dns_took,
|
||||
conn_took,
|
||||
connect_took,
|
||||
tls_took,
|
||||
};
|
||||
|
||||
if (type === "pagespeed") {
|
||||
if (typeof payload === "undefined") {
|
||||
this.logger.warn({
|
||||
message: "Failed to build check",
|
||||
service: this.SERVICE_NAME,
|
||||
method: "buildCheck",
|
||||
details: "empty payload",
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
const categories = payload?.lighthouseResult?.categories ?? {};
|
||||
const audits = payload?.lighthouseResult?.audits ?? {};
|
||||
const {
|
||||
"cumulative-layout-shift": cls = 0,
|
||||
"speed-index": si = 0,
|
||||
"first-contentful-paint": fcp = 0,
|
||||
"largest-contentful-paint": lcp = 0,
|
||||
"total-blocking-time": tbt = 0,
|
||||
} = audits;
|
||||
check.accessibility = (categories?.accessibility?.score || 0) * 100;
|
||||
check.bestPractices = (categories?.["best-practices"]?.score || 0) * 100;
|
||||
check.seo = (categories?.seo?.score || 0) * 100;
|
||||
check.performance = (categories?.performance?.score || 0) * 100;
|
||||
check.audits = { cls, si, fcp, lcp, tbt };
|
||||
}
|
||||
|
||||
if (type === "hardware") {
|
||||
const { cpu, memory, disk, host, net } = payload?.data ?? {};
|
||||
const { errors } = payload?.errors ?? [];
|
||||
check.cpu = cpu ?? {};
|
||||
check.memory = memory ?? {};
|
||||
check.disk = disk ?? {};
|
||||
check.host = host ?? {};
|
||||
check.errors = errors ?? [];
|
||||
check.capture = payload?.capture ?? {};
|
||||
check.net = net ?? {};
|
||||
}
|
||||
return check;
|
||||
};
|
||||
|
||||
/**
|
||||
* Inserts a check into the database based on the network response.
|
||||
*
|
||||
* @param {Object} networkResponse - The network response object.
|
||||
* @param {string} networkResponse.monitorId - The monitor ID.
|
||||
* @param {string} networkResponse.type - The type of the response.
|
||||
* @param {string} networkResponse.status - The status of the response.
|
||||
* @param {number} networkResponse.responseTime - The response time.
|
||||
* @param {number} networkResponse.code - The status code.
|
||||
* @param {string} networkResponse.message - The message.
|
||||
* @param {Object} networkResponse.payload - The payload of the response.
|
||||
* @returns {Promise<void>} A promise that resolves when the check is inserted.
|
||||
*/
|
||||
insertCheck = async (check) => {
|
||||
try {
|
||||
if (typeof check === "undefined") {
|
||||
this.logger.warn({
|
||||
message: "Failed to build check",
|
||||
service: this.SERVICE_NAME,
|
||||
method: "insertCheck",
|
||||
});
|
||||
return false;
|
||||
}
|
||||
this.buffer.addToBuffer({ check });
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: error.service || this.SERVICE_NAME,
|
||||
method: error.method || "insertCheck",
|
||||
details: error.details || `Error inserting check for monitor: ${check?.monitorId}`,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
export default StatusService;
|
||||
Reference in new issue
Block a user