-> timeseries

This commit is contained in:
Alex Holliday committed 2026-01-12 19:32:23 +00:00
1 parent 5c14c68899
commit e090d3cfd3
91 files changed
+941 -987

No files matched your search

@@ -0,0 +1,176 @@
import Scheduler from "super-simple-scheduler";
const SERVICE_NAME = "JobQueue";
class SuperSimpleQueue {
static SERVICE_NAME = SERVICE_NAME;
constructor({ envSettings, db, logger, helper }) {
this.envSettings = envSettings;
this.db = db;
this.logger = logger;
this.helper = helper;
}
get serviceName() {
return SuperSimpleQueue.SERVICE_NAME;
}
static async create({ envSettings, db, logger, helper }) {
const instance = new SuperSimpleQueue({ envSettings, db, logger, helper });
await instance.init();
return instance;
}
init = async () => {
try {
this.scheduler = new Scheduler({
// storeType: "mongo",
// storeType: "redis",
logLevel: "debug",
debug: true,
// dbUri: this.envSettings.dbConnectionString,
});
this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getMonitorJob());
const monitors = await this.db.monitorModule.getAllMonitors();
for (const monitor of monitors) {
const randomOffset = Math.floor(Math.random() * 100);
setTimeout(() => {
this.addJob(monitor._id, monitor);
}, randomOffset);
}
return true;
} catch (error) {
this.logger.error({
message: "Failed to initialize SuperSimpleQueue",
service: SERVICE_NAME,
method: "init",
details: error,
});
return false;
}
};
addJob = async (monitorId, monitor) => {
this.scheduler.addJob({
id: monitorId.toString(),
template: "monitor-job",
repeat: monitor.interval,
active: monitor.isActive,
data: monitor.toObject(),
});
};
deleteJob = async (monitor) => {
this.scheduler.removeJob(monitor._id.toString());
};
pauseJob = async (monitor) => {
const result = this.scheduler.pauseJob(monitor._id.toString());
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.logger.debug({
message: `Paused monitor ${monitor._id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor) => {
const result = this.scheduler.resumeJob(monitor._id.toString());
if (result === false) {
throw new Error("Failed to resume monitor");
}
this.logger.debug({
message: `Resumed monitor ${monitor._id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
updateJob = async (monitor) => {
this.scheduler.updateJob(monitor._id.toString(), { repeat: monitor.interval, data: monitor.toObject() });
};
shutdown = async () => {
this.scheduler.stop();
};
getMetrics = async () => {
const jobs = await this.scheduler.getJobs();
const metrics = jobs.reduce(
(acc, job) => {
acc.totalRuns += job.runCount || 0;
acc.totalFailures += job.failCount || 0;
acc.jobs++;
if (job.failCount > 0 && job.lastFailedAt >= job.lsatRunAt) {
acc.failingJobs++;
}
if (job.lockedAt) {
acc.activeJobs++;
}
if (job.failCount > 0) {
acc.jobsWithFailures.push({
monitorId: job.id,
monitorUrl: job?.data?.url || null,
monitorType: job?.data?.type || null,
failedAt: job.lastFailedAt,
failCount: job.failCount,
failReason: job.lastFailReason,
});
}
return acc;
},
{
jobs: 0,
activeJobs: 0,
failingJobs: 0,
jobsWithFailures: [],
totalRuns: 0,
totalFailures: 0,
}
);
return metrics;
};
getJobs = async () => {
const jobs = await this.scheduler.getJobs();
return jobs.map((job) => {
return {
monitorId: job.id,
monitorUrl: job?.data?.url || null,
monitorType: job?.data?.type || null,
monitorInterval: job?.data?.interval || null,
active: job.active,
lockedAt: job.lockedAt,
runCount: job.runCount || 0,
failCount: job.failCount || 0,
failReason: job.lastFailReason,
lastRunAt: job.lastRunAt,
lastFinishedAt: job.lastFinishedAt,
lastRunTook: job.lockedAt ? null : job.lastFinishedAt - job.lastRunAt,
lastFailedAt: job.lastFailedAt,
};
});
};
flushQueues = async () => {
const stopRes = this.scheduler.stop();
const flushRes = this.scheduler.flushJobs();
const initRes = await this.init();
return {
success: stopRes && flushRes && initRes,
};
};
obliterate = async () => {
console.log("obliterate not implemented");
};
}
export default SuperSimpleQueue;
@@ -0,0 +1,115 @@
const SERVICE_NAME = "JobQueueHelper";
class SuperSimpleQueueHelper {
static SERVICE_NAME = SERVICE_NAME;
/**
* @param {{
* db: import("../database.js").Database,
* logger: import("../logger.js").Logger,
* networkService: import("../networkService.js").NetworkService,
* statusService: import("../statusService.js").StatusService,
* notificationService: import("../notificationService.js").NotificationService
* }}
*/
constructor({ db, logger, networkService, statusService, notificationService }) {
this.db = db;
this.logger = logger;
this.networkService = networkService;
this.statusService = statusService;
this.notificationService = notificationService;
}
get serviceName() {
return SuperSimpleQueueHelper.SERVICE_NAME;
}
getMonitorJob = () => {
return async (monitor) => {
try {
const monitorId = monitor._id;
const teamId = monitor.teamId;
if (!monitorId) {
throw new Error("No monitor id");
}
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId, teamId);
if (maintenanceWindowActive) {
this.logger.info({
message: `Monitor ${monitorId} is in maintenance window`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
return;
}
const networkResponse = await this.networkService.requestStatus(monitor);
if (!networkResponse) {
throw new Error("No network response");
}
const { monitor: updatedMonitor, statusChanged, prevStatus } = await this.statusService.updateStatus(networkResponse);
this.notificationService
.handleNotifications({
...networkResponse,
monitor: updatedMonitor,
prevStatus,
statusChanged,
})
.catch((error) => {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "getMonitorJob",
details: `Error sending notifications for job ${monitor._id}: ${error.message}`,
stack: error.stack,
});
});
} catch (error) {
this.logger.warn({
message: error.message,
service: error.service || SERVICE_NAME,
method: error.method || "getMonitorJob",
stack: error.stack,
});
throw error;
}
};
};
async isInMaintenanceWindow(monitorId, teamId) {
const maintenanceWindows = await this.db.maintenanceWindowModule.getMaintenanceWindowsByMonitorId({
monitorId: monitorId.toString(),
teamId: teamId.toString(),
});
// Check for active maintenance window:
const maintenanceWindowIsActive = maintenanceWindows.reduce((acc, window) => {
if (window.active) {
const start = new Date(window.start);
const end = new Date(window.end);
const now = new Date();
const repeatInterval = window.repeat || 0;
// If start is < now and end > now, we're in maintenance
if (start <= now && end >= now) return true;
// If maintenance window was set in the past with a repeat,
// we need to advance start and end to see if we are in range
while (start < now && repeatInterval !== 0) {
start.setTime(start.getTime() + repeatInterval);
end.setTime(end.getTime() + repeatInterval);
if (start <= now && end >= now) {
return true;
}
}
return false;
}
return acc;
}, false);
return maintenanceWindowIsActive;
}
}
export default SuperSimpleQueueHelper;
+169
View File
@@ -0,0 +1,169 @@
import { config } from "@/config/index.js";
const SERVICE_NAME = "BufferService";
class BufferService {
static SERVICE_NAME = SERVICE_NAME;
constructor({ db, logger, envSettings, incidentService }) {
console.log(envSettings);
this.BUFFER_TIMEOUT = config.NODE_ENV === "development" ? 10 : 1000 * 60 * 1; // 1 minute
this.db = db;
this.logger = logger;
this.incidentService = incidentService;
this.SERVICE_NAME = SERVICE_NAME;
this.buffer = [];
this.incidentBuffer = [];
this.scheduleNextFlush();
this.logger.info({
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
service: this.SERVICE_NAME,
method: "constructor",
});
}
get serviceName() {
return BufferService.SERVICE_NAME;
}
addToBuffer({ check }) {
try {
this.buffer.push(check);
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "addToBuffer",
stack: error.stack,
});
}
}
addIncidentToBuffer({ monitor, check, action = "create" }) {
try {
if (!monitor || !check) {
this.logger.warn({
message: "Skipping incident buffer item: missing monitor or check",
service: this.SERVICE_NAME,
method: "addIncidentToBuffer",
});
return;
}
this.incidentBuffer.push({ monitor, check, action });
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "addIncidentToBuffer",
stack: error.stack,
});
}
}
removeCheckFromBuffer(checkToRemove) {
try {
if (!checkToRemove) {
return false;
}
const index = this.buffer.findIndex((check) => {
if (checkToRemove._id && check._id) {
return check._id.toString() === checkToRemove._id.toString();
}
return (
check.monitorId?.toString() === checkToRemove.monitorId?.toString() &&
check.teamId?.toString() === checkToRemove.teamId?.toString() &&
check.type === checkToRemove.type &&
check.status === checkToRemove.status &&
check.statusCode === checkToRemove.statusCode &&
check.responseTime === checkToRemove.responseTime &&
check.message === checkToRemove.message
);
});
if (index !== -1) {
this.buffer.splice(index, 1);
return true;
}
return false;
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "removeCheckFromBuffer",
stack: error.stack,
});
return false;
}
}
scheduleNextFlush() {
this.bufferTimer = setTimeout(async () => {
try {
await this.flushBuffer();
} catch (error) {
this.logger.error({
message: `Error in flush cycle: ${error.message}`,
service: this.SERVICE_NAME,
method: "scheduleNextFlush",
stack: error.stack,
});
} finally {
// Schedule the next flush only after the current one completes
this.scheduleNextFlush();
}
}, this.BUFFER_TIMEOUT);
}
async flushBuffer() {
try {
if (this.buffer.length > 0) {
await this.db.checkModule.createChecks(this.buffer);
}
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "flushBuffer",
stack: error.stack,
});
}
try {
if (this.incidentBuffer.length > 0 && this.incidentService) {
await this.flushIncidentBuffer();
}
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "flushBuffer",
stack: error.stack,
});
}
this.buffer = [];
this.incidentBuffer = [];
}
async flushIncidentBuffer() {
if (!this.incidentService || this.incidentBuffer.length === 0) {
return;
}
try {
const itemsToProcess = [...this.incidentBuffer];
await this.incidentService.processIncidentsFromBuffer(itemsToProcess);
} catch (error) {
this.logger.error({
message: `Error flushing incident buffer: ${error.message}`,
service: this.SERVICE_NAME,
method: "flushIncidentBuffer",
stack: error.stack,
});
throw error;
}
}
}
export default BufferService;
+171
View File
@@ -0,0 +1,171 @@
import { fileURLToPath } from "url";
import path from "path";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const SERVICE_NAME = "EmailService";
/**
* Represents an email service that can load templates, build, and send emails.
*/
class EmailService {
static SERVICE_NAME = SERVICE_NAME;
/**
* Constructs an instance of the EmailService, initializing template loaders and the email transporter.
* @param {Object} settingsService - The settings service to get email configuration.
* @param {Object} fs - The file system module.
* @param {Object} path - The path module.
* @param {Function} compile - The Handlebars compile function.
* @param {Function} mjml2html - The MJML to HTML conversion function.
* @param {Object} nodemailer - The nodemailer module.
* @param {Object} logger - The logger module.
*/
constructor(settingsService, fs, path, compile, mjml2html, nodemailer, logger) {
this.settingsService = settingsService;
this.fs = fs;
this.path = path;
this.compile = compile;
this.mjml2html = mjml2html;
this.nodemailer = nodemailer;
this.logger = logger;
this.init();
}
get serviceName() {
return EmailService.SERVICE_NAME;
}
init = async () => {
/**
* Loads an email template from the filesystem.
*
* @param {string} templateName - The name of the template to load.
* @returns {Function} A compiled template function that can be used to generate HTML email content.
*/
this.loadTemplate = (templateName) => {
try {
const templatePath = this.path.join(__dirname, `../../../templates/${templateName}.mjml`);
const templateContent = this.fs.readFileSync(templatePath, "utf8");
return this.compile(templateContent);
} catch (error) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "loadTemplate",
stack: error.stack,
});
}
};
/**
* A lookup object to access preloaded email templates.
* @type {Object.<string, Function>}
* TODO Load less used templates in their respective functions
*/
this.templateLookup = {
welcomeEmailTemplate: this.loadTemplate("welcomeEmail"),
employeeActivationTemplate: this.loadTemplate("employeeActivation"),
noIncidentsThisWeekTemplate: this.loadTemplate("noIncidentsThisWeek"),
serverIsDownTemplate: this.loadTemplate("serverIsDown"),
serverIsUpTemplate: this.loadTemplate("serverIsUp"),
passwordResetTemplate: this.loadTemplate("passwordReset"),
hardwareIncidentTemplate: this.loadTemplate("hardwareIncident"),
testEmailTemplate: this.loadTemplate("testEmailTemplate"),
};
/**
* The email transporter used to send emails.
* @type {Object}
*/
};
buildEmail = async (template, context) => {
try {
const mjml = this.templateLookup[template](context);
const html = await this.mjml2html(mjml);
return html.html;
} catch (error) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "buildEmail",
stack: error.stack,
});
}
};
sendEmail = async (to, subject, html, transportConfig) => {
let config;
if (typeof transportConfig !== "undefined") {
config = transportConfig;
} else {
config = await this.settingsService.getDBSettings();
}
const {
systemEmailHost,
systemEmailPort,
systemEmailSecure,
systemEmailPool,
systemEmailUser,
systemEmailAddress,
systemEmailPassword,
systemEmailConnectionHost,
systemEmailTLSServername,
systemEmailIgnoreTLS,
systemEmailRequireTLS,
systemEmailRejectUnauthorized,
} = config;
const emailConfig = {
host: systemEmailHost,
port: Number(systemEmailPort),
secure: systemEmailSecure,
auth: {
user: systemEmailUser || systemEmailAddress,
pass: systemEmailPassword,
},
name: systemEmailConnectionHost || "localhost",
connectionTimeout: 5000,
pool: systemEmailPool,
tls: {
rejectUnauthorized: systemEmailRejectUnauthorized,
ignoreTLS: systemEmailIgnoreTLS,
requireTLS: systemEmailRequireTLS,
servername: systemEmailTLSServername,
},
};
this.transporter = this.nodemailer.createTransport(emailConfig);
try {
await this.transporter.verify();
} catch (error) {
this.logger.warn({
message: "Email transporter verification failed",
service: SERVICE_NAME,
method: "verifyTransporter",
});
return false;
}
try {
const info = await this.transporter.sendMail({
to: to,
from: systemEmailAddress,
subject: subject,
html: html,
});
return info?.messageId;
} catch (error) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "sendEmail",
stack: error.stack,
});
}
};
}
export default EmailService;
@@ -0,0 +1,101 @@
export class AppError extends Error {
constructor(message, status = 500, service = null, method = null, details = null) {
super(message);
this.status = status;
this.service = service;
this.method = method;
this.details = details;
Error.captureStackTrace(this, this.constructor);
}
}
class ValidationError extends AppError {
constructor(message, details = null, service = null, method = null) {
super(message, 422, service, method, details);
}
}
class AuthenticationError extends AppError {
constructor(message, details = null, service = null, method = null) {
super(message, 401, service, method, details);
}
}
class AuthorizationError extends AppError {
constructor(message, details = null, service = null, method = null) {
super(message, 403, service, method, details);
}
}
class NotFoundError extends AppError {
constructor(message, details = null, service = null, method = null) {
super(message, 404, service, method, details);
}
}
class ConflictError extends AppError {
constructor(message, details = null, service = null, method = null) {
super(message, 409, service, method, details);
}
}
class DatabaseError extends AppError {
constructor(message, details = null, service = null, method = null) {
super(message, 500, service, method, details);
}
}
class BadRequestError extends AppError {
constructor(message, details = null, service = null, method = null) {
super(message, 400, service, method, details);
}
}
const SERVICE_NAME = "ErrorService";
class ErrorService {
static SERVICE_NAME = SERVICE_NAME;
constructor() {}
get serviceName() {
return ErrorService.SERVICE_NAME;
}
createError = (message, status = 500, service = null, method = null, details = null) => {
return new AppError(message, status, service, method, details);
};
createValidationError = (message, details = null, service = null, method = null) => {
return new ValidationError(message, details, service, method);
};
createAuthenticationError = (message = "Unauthorized", details = null, service = null, method = null) => {
return new AuthenticationError(message, details, service, method);
};
createAuthorizationError = (message, details = null, service = null, method = null) => {
return new AuthorizationError(message, details, service, method);
};
createNotFoundError = (message, details = null, service = null, method = null) => {
return new NotFoundError(message, details, service, method);
};
createConflictError = (message, details = null, service = null, method = null) => {
return new ConflictError(message, details, service, method);
};
createDatabaseError = (message, details = null, service = null, method = null) => {
return new DatabaseError(message, details, service, method);
};
createServerError = (message, details = null, service = null, method = null) => {
return this.createError(message, 500, service, method, details);
};
createBadRequestError = (message = "BadRequest", details = null, service = null, method = null) => {
return new BadRequestError(message, details, service, method);
};
}
export default ErrorService;
@@ -0,0 +1,580 @@
import CacheableLookup from "cacheable-lookup";
const SERVICE_NAME = "NetworkService";
class NetworkService {
static SERVICE_NAME = SERVICE_NAME;
constructor({ axios, got, https, jmespath, GameDig, ping, logger, http, Docker, net, stringService, settingsService }) {
this.TYPE_PING = "ping";
this.TYPE_HTTP = "http";
this.TYPE_PAGESPEED = "pagespeed";
this.TYPE_HARDWARE = "hardware";
this.TYPE_DOCKER = "docker";
this.TYPE_PORT = "port";
this.TYPE_GAME = "game";
this.SERVICE_NAME = SERVICE_NAME;
this.NETWORK_ERROR = 5000;
this.PING_ERROR = 5001;
this.axios = axios;
this.https = https;
this.jmespath = jmespath;
this.GameDig = GameDig;
this.ping = ping;
this.logger = logger;
this.http = http;
this.Docker = Docker;
this.net = net;
this.stringService = stringService;
this.settingsService = settingsService;
const cacheable = new CacheableLookup();
this.got = got.extend({
dnsCache: cacheable,
timeout: {
request: 30000,
},
retry: { limit: 1 },
});
}
// Helper functions
async timeRequest(operation) {
const start = process.hrtime.bigint();
try {
const response = await operation();
const elapsedMs = Math.round(Number(process.hrtime.bigint() - start) / 1_000_000);
return { response, responseTime: elapsedMs };
} catch (error) {
const elapsedMs = Math.round(Number(process.hrtime.bigint() - start) / 1_000_000);
return { response: null, responseTime: elapsedMs, error };
}
}
// Main entry point
async requestStatus(monitor) {
const type = monitor?.type || "unknown";
switch (type) {
case this.TYPE_PING:
return await this.requestPing(monitor);
case this.TYPE_HTTP:
return await this.requestHttp(monitor);
case this.TYPE_PAGESPEED:
return await this.requestPageSpeed(monitor);
case this.TYPE_HARDWARE:
return await this.requestHardware(monitor);
case this.TYPE_DOCKER:
return await this.requestDocker(monitor);
case this.TYPE_PORT:
return await this.requestPort(monitor);
case this.TYPE_GAME:
return await this.requestGame(monitor);
default:
return await this.handleUnsupportedType(type);
}
}
async requestPing(monitor) {
try {
if (!monitor?.url) {
throw new Error("Monitor URL is required");
}
const rawUrl = monitor.url;
const sanitizedHost = rawUrl
.replace(/^https?:\/\//, "")
.replace(/\/.*$/, "")
.replace(/:.*/, "");
const { response, error } = await this.timeRequest(() => this.ping.promise.probe(sanitizedHost));
if (!response) {
if (error) {
throw error;
}
throw new Error("Ping failed - no result returned");
}
const pingResponse = {
monitorId: monitor._id,
type: "ping",
status: response.alive,
code: 200,
responseTime: response.time,
message: "Success",
payload: response,
};
if (error) {
pingResponse.status = false;
pingResponse.code = 200;
pingResponse.message = "Ping failed";
return pingResponse;
}
return pingResponse;
} catch (err) {
err.service = this.SERVICE_NAME;
err.method = "requestPing";
throw err;
}
}
async requestHttp(monitor) {
const { url, secret, _id, teamId, type, ignoreTlsErrors, jsonPath, matchMethod, expectedValue } = monitor;
const httpResponse = {
monitorId: _id,
teamId: teamId,
type,
};
try {
if (!url) {
throw new Error("Monitor URL is required");
}
const config = {
headers: secret ? { Authorization: `Bearer ${secret}` } : undefined,
};
if (ignoreTlsErrors) {
config.agent = {
https: new this.https.Agent({
rejectUnauthorized: false,
}),
};
}
const response = await this.got(url, config);
let payload;
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/json")) {
try {
payload = JSON.parse(response.body);
} catch {
payload = response.body;
}
} else {
payload = response.body;
}
httpResponse.code = response.statusCode;
httpResponse.status = response.ok;
httpResponse.message = response.statusMessage;
httpResponse.responseTime = response.timings.phases.total || 0;
httpResponse.payload = payload;
httpResponse.timings = response.timings || {};
if (!expectedValue && !jsonPath) {
return httpResponse;
}
if (expectedValue && !jsonPath) {
let ok = false;
if (matchMethod === "equal") ok = payload === expectedValue;
if (matchMethod === "include" && typeof payload === "string") ok = payload.includes(expectedValue);
if (matchMethod === "regex" && typeof payload === "string") ok = new RegExp(expectedValue).test(payload);
if (ok === true) {
return httpResponse;
} else {
httpResponse.code = 500;
httpResponse.status = false;
httpResponse.message = this.stringService.httpExpectedValueFail;
return httpResponse;
}
}
if (jsonPath) {
const contentType = response.headers["content-type"];
const isJson = contentType?.includes("application/json");
if (!isJson) {
httpResponse.status = false;
httpResponse.message = this.stringService.httpNotJson;
return httpResponse;
}
try {
const extracted = this.jmespath.search(payload, jsonPath);
if (expectedValue) {
let ok = false;
if (matchMethod === "equal") ok = extracted === expectedValue;
if (matchMethod === "include" && typeof extracted === "string") ok = extracted.includes(expectedValue);
if (matchMethod === "regex" && typeof extracted === "string") ok = new RegExp(expectedValue).test(extracted);
if (ok) {
httpResponse.extracted = extracted;
return httpResponse;
} else {
httpResponse.status = false;
httpResponse.code = 500;
httpResponse.message = this.stringService.httpJsonPathFail;
httpResponse.extracted = extracted;
return httpResponse;
}
} else {
const isFalsey = extracted === false || extracted === "false" || extracted === undefined || extracted === null;
if (!isFalsey) {
httpResponse.extracted = extracted;
return httpResponse;
} else {
httpResponse.status = false;
httpResponse.code = 500;
httpResponse.message = this.stringService.httpJsonPathFail;
httpResponse.extracted = extracted;
return httpResponse;
}
}
} catch {
httpResponse.status = false;
httpResponse.message = this.stringService.httpJsonPathError;
return httpResponse;
}
}
return httpResponse;
} catch (err) {
if (err.name === "HTTPError" || err.name === "RequestError") {
httpResponse.code = err?.response?.statusCode || this.NETWORK_ERROR;
httpResponse.status = false;
httpResponse.message = err?.response?.statusCode || err.message;
httpResponse.responseTime = err?.timings?.phases?.total || 0;
httpResponse.payload = null;
httpResponse.timings = err?.timings || {};
return httpResponse;
}
err.service = this.SERVICE_NAME;
err.method = "requestHttp";
throw err;
}
}
async requestPageSpeed(monitor) {
try {
const url = monitor.url;
if (!url) {
throw new Error("Monitor URL is required");
}
let pageSpeedUrl = `https://pagespeedonline.googleapis.com/pagespeedonline/v5/runPagespeed?url=${url}&category=seo&category=accessibility&category=best-practices&category=performance`;
const dbSettings = await this.settingsService.getDBSettings();
if (dbSettings?.pagespeedApiKey) {
pageSpeedUrl += `&key=${dbSettings.pagespeedApiKey}`;
} else {
this.logger.warn({
message: "PageSpeed API key not found, job not executed",
service: this.SERVICE_NAME,
method: "requestPagespeed",
details: { url },
});
}
return await this.requestHttp({
...monitor,
url: pageSpeedUrl,
});
} catch (err) {
err.service = this.SERVICE_NAME;
err.method = "requestPageSpeed";
throw err;
}
}
async requestHardware(monitor) {
try {
return await this.requestHttp(monitor);
} catch (err) {
err.service = this.SERVICE_NAME;
err.method = "requestHardware";
throw err;
}
}
async requestDocker(monitor) {
try {
if (!monitor.url) {
throw new Error("Monitor URL is required");
}
const docker = new this.Docker({
socketPath: "/var/run/docker.sock",
handleError: true, // Enable error handling
});
const dockerResponse = {
monitorId: monitor._id,
type: monitor.type,
};
const containers = await docker.listContainers({ all: true });
// Normalize input: strip leading slashes and convert to lowercase for comparison
const normalizedInput = monitor.url.replace(/^\/+/, "").toLowerCase();
// Priority-based matching to avoid ambiguity:
// 1. Exact full ID match (64-char)
let exactIdMatch = containers.find((c) => c.Id.toLowerCase() === normalizedInput);
// 2. Exact container name match (case-insensitive)
let exactNameMatch = containers.find((c) =>
c.Names.some((name) => {
const cleanName = name.replace(/^\/+/, "").toLowerCase();
return cleanName === normalizedInput;
})
);
// 3. Partial ID match (fallback for backwards compatibility)
let partialIdMatch = containers.find((c) => c.Id.toLowerCase().startsWith(normalizedInput));
// Select container based on priority
let targetContainer = exactIdMatch || exactNameMatch || partialIdMatch;
// Return negative response if no container
if (!targetContainer) {
this.logger.warn({
message: `No container found for "${monitor.url}".`,
service: this.SERVICE_NAME,
method: "requestDocker",
details: { url: monitor.url },
});
dockerResponse.code = 404;
dockerResponse.status = false;
dockerResponse.message = this.stringService.dockerNotFound;
return dockerResponse;
}
// Return negative response if ambiguous matches exist
const matchTypes = [];
if (exactIdMatch) matchTypes.push("exact ID");
if (exactNameMatch) matchTypes.push("exact name");
if (partialIdMatch && !exactIdMatch) matchTypes.push("partial ID");
if (matchTypes.length > 1) {
this.logger.warn({
message: `Ambiguous container match for "${monitor.url}". Matched by: ${matchTypes.join(", ")}. Using ${exactIdMatch ? "exact ID" : exactNameMatch ? "exact name" : "partial ID"} match.`,
service: this.SERVICE_NAME,
method: "requestDocker",
details: { url: monitor.url },
});
dockerResponse.status = 404;
dockerResponse.status = false;
dockerResponse.message = `Ambiguous container match for "${monitor.url}". Matched by: ${matchTypes.join(", ")}. Using ${exactIdMatch ? "exact ID" : exactNameMatch ? "exact name" : "partial ID"} match.`;
return dockerResponse;
}
const container = docker.getContainer(targetContainer.Id);
const { response, responseTime, error } = await this.timeRequest(() => container.inspect());
dockerResponse.responseTime = responseTime;
dockerResponse.status = response?.State?.Status === "running" ? true : false;
dockerResponse.code = 200;
dockerResponse.message = "Docker container status fetched successfully";
if (error) {
dockerResponse.status = false;
dockerResponse.code = error.statusCode || this.NETWORK_ERROR;
dockerResponse.message = error.reason || "Failed to fetch Docker container information";
return dockerResponse;
}
return dockerResponse;
} catch (err) {
err.service = this.SERVICE_NAME;
err.method = "requestDocker";
throw err;
}
}
async requestPort(monitor) {
try {
const { url, port } = monitor;
const { response, responseTime, error } = await this.timeRequest(async () => {
return new Promise((resolve, reject) => {
const socket = this.net.createConnection(
{
host: url,
port,
},
() => {
socket.end();
socket.destroy();
resolve({ success: true });
}
);
socket.setTimeout(5000);
socket.on("timeout", () => {
socket.destroy();
reject(new Error("Connection timeout"));
});
socket.on("error", (err) => {
socket.destroy();
reject(err);
});
});
});
const portResponse = {
code: 200,
status: response.success,
message: this.stringService.portSuccess,
monitorId: monitor._id,
type: monitor.type,
responseTime: responseTime,
};
if (error) {
portResponse.code = this.NETWORK_ERROR;
portResponse.status = false;
portResponse.message = this.stringService.portFail;
return portResponse;
}
return portResponse;
} catch (error) {
error.service = this.SERVICE_NAME;
error.method = "requestTCP";
throw error;
}
}
async requestGame(monitor) {
try {
const { url, port, gameId } = monitor;
const gameResponse = {
code: 200,
status: true,
message: "Success",
monitorId: monitor._id,
type: "game",
};
const state = await this.GameDig.query({
type: gameId,
host: url,
port: port,
}).catch((error) => {
this.logger.warn({
message: error.message,
service: this.SERVICE_NAME,
method: "requestGame",
details: { url, port, gameId },
});
});
if (!state) {
gameResponse.code = this.NETWORK_ERROR;
gameResponse.status = false;
gameResponse.message = "No response";
return gameResponse;
}
gameResponse.responseTime = state.ping;
gameResponse.payload = state;
return gameResponse;
} catch (error) {
error.service = this.SERVICE_NAME;
error.method = "requestPing";
throw error;
}
}
async handleUnsupportedType(type) {
const err = new Error(`Unsupported type: ${type}`);
err.service = this.SERVICE_NAME;
err.method = "getStatus";
throw err;
}
// Other network requests unrelated to monitoring:
async requestWebhook(type, url, body) {
try {
const response = await this.axios.post(url, body, {
headers: {
"Content-Type": "application/json",
},
});
return {
type: "webhook",
status: true,
code: response.status,
message: `Successfully sent ${type} notification`,
payload: response.data,
};
} catch (error) {
this.logger.warn({
message: error.message,
service: this.SERVICE_NAME,
method: "requestWebhook",
});
return {
type: "webhook",
status: false,
code: error.response?.status || this.NETWORK_ERROR,
message: `Failed to send ${type} notification`,
payload: error.response?.data,
};
}
}
async requestPagerDuty({ message, routingKey, monitorUrl }) {
try {
const response = await this.axios.post(`https://events.pagerduty.com/v2/enqueue`, {
routing_key: routingKey,
event_action: "trigger",
payload: {
summary: message,
severity: "critical",
source: monitorUrl,
timestamp: new Date().toISOString(),
},
});
if (response?.data?.status !== "success") return false;
return true;
} catch (error) {
error.details = error.response?.data;
error.service = this.SERVICE_NAME;
error.method = "requestPagerDuty";
throw error;
}
}
async requestMatrix({ homeserverUrl, accessToken, roomId, message }) {
try {
const url = `${homeserverUrl}/_matrix/client/v3/rooms/${roomId}/send/m.room.message?access_token=${accessToken}`;
const body = {
msgtype: "m.text",
body: message,
format: "org.matrix.custom.html",
formatted_body: message,
};
const response = await this.axios.post(url, body, {
headers: {
"Content-Type": "application/json",
},
});
return {
status: true,
code: response.status,
message: "Successfully sent Matrix notification",
};
} catch (error) {
this.logger.warn({
message: error.message,
service: this.SERVICE_NAME,
method: "requestMatrix",
});
return {
status: false,
code: error.response?.status || this.NETWORK_ERROR,
message: "Failed to send Matrix notification",
payload: error.response?.data,
};
}
}
}
export default NetworkService;
@@ -0,0 +1,34 @@
const SERVICE_NAME = "Matrix";
class Matrix {
static SERVICE_NAME = SERVICE_NAME;
constructor({ networkService, logger }) {
this.networkService = networkService;
this.logger = logger;
}
get serviceName() {
return Matrix.SERVICE_NAME;
}
async send({ friendlyName, homeserverUrl, accessToken, roomId, message, monitorName }) {
const title = `Checkmate status for ${monitorName}`;
const formattedMessage = `## ${title}\n${message}`;
try {
await this.networkService.requestMatrix({
homeserverUrl,
accessToken,
roomId,
message: formattedMessage,
});
this.logger.info(`Successfully sent Matrix notification for ${friendlyName}`);
return true;
} catch (error) {
this.logger.error(`Failed to send Matrix notification for ${friendlyName}: ${error.message}`);
return false;
}
}
}
export default Matrix;
@@ -0,0 +1,128 @@
const SERVICE_NAME = "NotificationService";
import Matrix from "./notificationProviders/matrix.js";
class NotificationService {
static SERVICE_NAME = SERVICE_NAME;
constructor({ emailService, db, logger, networkService, stringService, notificationUtils }) {
this.emailService = emailService;
this.db = db;
this.logger = logger;
this.networkService = networkService;
this.stringService = stringService;
this.notificationUtils = notificationUtils;
}
get serviceName() {
return NotificationService.SERVICE_NAME;
}
sendNotification = async ({ notification, subject, content, html, discordContent = null, webhookBody = null }) => {
const { type, address } = notification;
if (type === "email") {
const messageId = await this.emailService.sendEmail(address, subject, html);
if (!messageId) return false;
return true;
}
// Create a body for webhooks
let body = { text: content };
if (type === "discord") {
body = !discordContent ? { content } : discordContent;
}
if (type === "webhook") {
body = !webhookBody ? { content } : webhookBody;
}
if (type === "slack" || type === "discord" || type === "webhook") {
const response = await this.networkService.requestWebhook(type, address, body);
return response.status;
}
if (type === "pager_duty") {
const response = await this.networkService.requestPagerDuty({
message: content,
monitorUrl: subject,
routingKey: address,
});
return response;
}
if (type === "matrix") {
const { friendlyName, homeserverUrl, accessToken, roomId } = notification;
const monitorName = subject;
const message = content;
const matrix = new Matrix({ networkService: this.networkService, logger: this.logger });
const success = await matrix.send({ friendlyName, homeserverUrl, accessToken, roomId, message, monitorName });
return success;
}
};
async handleNotifications(networkResponse) {
const { monitor, statusChanged, prevStatus } = networkResponse;
const { type } = monitor;
if (type !== "hardware" && statusChanged === false) return false;
// if prevStatus is undefined, monitor is resuming, we're done
if (type !== "hardware" && prevStatus === undefined) return false;
const notificationIDs = networkResponse.monitor?.notifications ?? [];
if (notificationIDs.length === 0) return false;
if (networkResponse.monitor.type === "hardware") {
const thresholds = networkResponse?.monitor?.thresholds;
if (thresholds === undefined) return false; // No thresholds set, we're done
const metrics = networkResponse?.payload?.data ?? null;
if (metrics === null) return false; // No metrics, we're done
const [alerts, discordContent] = await this.notificationUtils.buildHardwareAlerts(networkResponse);
if (alerts.length === 0) return false;
const { subject, html } = await this.notificationUtils.buildHardwareEmail(networkResponse, alerts);
const content = await this.notificationUtils.buildHardwareNotificationMessage(alerts, monitor);
const webhookBody = await this.notificationUtils.buildHardwareWebhookBody(alerts, monitor);
const success = await this.notifyAll({ notificationIDs, subject, html, content, discordContent, webhookBody });
return success;
}
// Status monitors
const { subject, html } = await this.notificationUtils.buildStatusEmail(networkResponse);
const [content, discordContent] = await this.notificationUtils.buildWebhookMessage(networkResponse);
const success = this.notifyAll({ notificationIDs, subject, html, content, discordContent });
return success;
}
async notifyAll({ notificationIDs, subject, html, content, discordContent = null, webhookBody = null }) {
const notifications = await this.db.notificationModule.getNotificationsByIds(notificationIDs);
// Map each notification to a test promise
const promises = notifications.map(async (notification) => {
try {
await this.sendNotification({ notification, subject, content, html, discordContent, webhookBody });
return true;
} catch (err) {
return false;
}
});
const results = await Promise.all(promises);
return results.every((r) => r === true);
}
async getTestNotification() {
const html = await this.notificationUtils.buildTestEmail();
const content = "This is a test notification";
const subject = "Test Notification";
return { subject, html, content };
}
async testAllNotifications(notificationIDs) {
const { subject, html, content } = await this.getTestNotification();
return this.notifyAll({ notificationIDs, subject, html, content });
}
async sendTestNotification(notification) {
const { subject, html, content } = await this.getTestNotification();
const success = await this.sendNotification({ notification, subject, content, html });
return success;
}
}
export default NotificationService;
@@ -0,0 +1,205 @@
const SERVICE_NAME = "NotificationUtils";
class NotificationUtils {
static SERVICE_NAME = SERVICE_NAME;
constructor({ stringService, emailService, settingsService }) {
this.stringService = stringService;
this.emailService = emailService;
this.settingsService = settingsService;
}
get serviceName() {
return NotificationUtils.SERVICE_NAME;
}
buildTestEmail = async () => {
const context = { testName: "Monitoring System" };
const html = await this.emailService.buildEmail("testEmailTemplate", context);
return html;
};
buildStatusEmail = async (networkResponse) => {
const { monitor, status, prevStatus } = networkResponse;
const template = prevStatus === false ? "serverIsUpTemplate" : "serverIsDownTemplate";
const context = { monitor: monitor.name, url: monitor.url };
const subject = `Monitor ${monitor.name} is ${status === true ? "up" : "down"}`;
const html = await this.emailService.buildEmail(template, context);
return { subject, html };
};
buildWebhookMessage = (networkResponse) => {
const { monitor, status, code, timestamp } = networkResponse;
// Format timestamp using the local system timezone
const formatTime = (timestamp) => {
const date = new Date(timestamp);
// Get timezone abbreviation and format the date
const timeZoneAbbr = date.toLocaleTimeString("en-US", { timeZoneName: "short" }).split(" ").pop();
// Format the date with readable format
return (
date
.toLocaleString("en-US", {
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
hour12: false,
})
.replace(/(\d+)\/(\d+)\/(\d+),\s/, "$3-$1-$2 ") +
" " +
timeZoneAbbr
);
};
// Get formatted time
const formattedTime = timestamp ? formatTime(timestamp) : formatTime(new Date().getTime());
// Create different messages based on status with extra spacing
let messageText;
if (status === true) {
messageText = this.stringService.monitorUpAlert
.replace("{monitorName}", monitor.name)
.replace("{time}", formattedTime)
.replace("{code}", code || "Unknown");
} else {
messageText = this.stringService.monitorDownAlert
.replace("{monitorName}", monitor.name)
.replace("{time}", formattedTime)
.replace("{code}", code || "Unknown");
}
const dn = this.stringService.discordNotification;
let discordMessageText = {
embeds: [
{
title: status
? dn.uptimeAlert.replace("{monitorName}", monitor?.name ?? dn.unknown)
: dn.downtimeAlert.replace("{monitorName}", monitor?.name ?? dn.unknown),
color: status ? 5763719 : 15548997,
fields: [
{ name: dn.monitor, value: monitor?.name ?? dn.unknown, inline: true },
{ name: dn.status, value: status ? dn.up : dn.down, inline: true },
{ name: dn.statusCode, value: String(code ?? dn.unknown), inline: true },
{ name: dn.time, value: formattedTime, inline: true },
{ name: dn.url, value: monitor?.url ?? dn.unknown, inline: false },
],
footer: { text: dn.checkmate },
},
],
};
return [messageText, discordMessageText];
};
buildHardwareAlerts = async (networkResponse) => {
const monitor = networkResponse?.monitor;
const thresholds = networkResponse?.monitor?.thresholds;
const { usage_cpu: cpuThreshold = -1, usage_memory: memoryThreshold = -1, usage_disk: diskThreshold = -1 } = thresholds;
const metrics = networkResponse?.payload?.data;
const { cpu: { usage_percent: cpuUsage = -1 } = {}, memory: { usage_percent: memoryUsage = -1 } = {}, disk = [] } = metrics;
const { clientHost } = this.settingsService.getSettings();
const alerts = {
cpu: cpuThreshold !== -1 && cpuUsage > cpuThreshold ? true : false,
memory: memoryThreshold !== -1 && memoryUsage > memoryThreshold ? true : false,
disk: disk?.some((d) => diskThreshold !== -1 && typeof d?.usage_percent === "number" && d?.usage_percent > diskThreshold) ?? false,
};
const alertsToSend = [];
const discordEmbeds = [];
const monitorInfoFields = [
{ name: "Monitor", value: monitor.name, inline: true },
{ name: "URL", value: monitor.url, inline: false },
];
const goToIncidentField = { name: `Go to incident`, value: `${clientHost}/infrastructure/${monitor._id}` };
const formatDiscordAlert = {
cpu: () => ({
title: "CPU alert",
description: `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
color: 15548997,
fields: [...monitorInfoFields, goToIncidentField],
footer: { text: "Checkmate" },
}),
memory: () => ({
title: "Memory alert",
description: `Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
color: 15548997,
fields: [...monitorInfoFields, goToIncidentField],
footer: { text: "Checkmate" },
}),
disk: () => ({
title: "Disk alert",
description: `Your current disk usage is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
color: 15548997,
footer: { text: "Checkmate" },
fields: [
...monitorInfoFields,
...(Array.isArray(disk) ? disk : []).map((d, idx) => ({
name: `Disk ${idx}`,
value: `${(d?.usage_percent * 100).toFixed(0)}%`,
inline: true,
})),
goToIncidentField,
],
}),
};
const alertTypes = ["cpu", "memory", "disk"];
for (const type of alertTypes) {
// Iterate over each alert type to see if any need to be decremented
if (alerts[type] === true) {
monitor[`${type}AlertThreshold`]--; // Decrement threshold if an alert is triggered
if (monitor[`${type}AlertThreshold`] <= 0) {
// If threshold drops below 0, reset and send notification
monitor[`${type}AlertThreshold`] = monitor.alertThreshold;
const formatAlert = {
cpu: () => `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`,
memory: () =>
`Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`,
disk: () =>
`Your current disk usage: ${disk
.map((d, idx) => `(Disk${idx}: ${(d.usage_percent * 100).toFixed(0)}%)`)
.join(", ")} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`,
};
alertsToSend.push(formatAlert[type]());
discordEmbeds.push(formatDiscordAlert[type]());
}
}
}
await monitor.save();
const discordPayload = discordEmbeds.length ? { embeds: discordEmbeds } : null;
return [alertsToSend, discordPayload];
};
buildHardwareEmail = async (networkResponse, alerts) => {
const { monitor } = networkResponse;
const template = "hardwareIncidentTemplate";
const context = { monitor: monitor.name, url: monitor.url, alerts };
const subject = `Monitor ${monitor.name} infrastructure alerts`;
const html = await this.emailService.buildEmail(template, context);
return { subject, html };
};
buildHardwareNotificationMessage = (alerts, monitor) => {
const { clientHost } = this.settingsService.getSettings();
const alertsHeader = [`Monitor: ${monitor.name}`, `URL: ${monitor.url}`];
const alertFooter = [`Go to incident: ${clientHost}/infrastructure/${monitor._id}`];
const alertText = alerts.length > 0 ? [...alertsHeader, ...alerts, ...alertFooter] : [];
return alertText.map((alert) => alert).join("\n");
};
buildHardwareWebhookBody = (alerts, monitor) => {
const content = alerts.map((alert) => alert).join("\n");
const body = { text: content, name: monitor.name, url: monitor.url };
return body;
};
}
export default NotificationUtils;
+434
View File
@@ -0,0 +1,434 @@
import MonitorStats from "../../db/models/MonitorStats.js";
import Check from "../../db/models/Check.js";
const SERVICE_NAME = "StatusService";
class StatusService {
static SERVICE_NAME = SERVICE_NAME;
/**
* @param {{
* buffer: import("./bufferService.js").BufferService
* incidentService: import("../business/incidentService.js").IncidentService
* }}
*/ constructor({ db, logger, buffer, incidentService }) {
this.db = db;
this.logger = logger;
this.buffer = buffer;
this.incidentService = incidentService;
}
get serviceName() {
return StatusService.SERVICE_NAME;
}
async updateRunningStats({ monitor, networkResponse }) {
try {
const monitorId = monitor._id;
const { responseTime, status } = networkResponse;
// Get stats
let stats = await MonitorStats.findOne({ monitorId });
if (!stats) {
stats = new MonitorStats({
monitorId,
avgResponseTime: 0,
totalChecks: 0,
totalUpChecks: 0,
totalDownChecks: 0,
uptimePercentage: 0,
lastCheck: null,
});
}
// Update stats
// Last response time
stats.lastResponseTime = responseTime;
// Avg response time:
let avgResponseTime = stats.avgResponseTime;
if (typeof responseTime !== "undefined" && responseTime !== null) {
if (avgResponseTime === 0) {
avgResponseTime = responseTime;
} else {
avgResponseTime = (avgResponseTime * (stats.totalChecks - 1) + responseTime) / stats.totalChecks;
}
}
stats.avgResponseTime = avgResponseTime;
// Total checks
stats.totalChecks++;
if (status === true) {
stats.totalUpChecks++;
// Update the timeSinceLastFailure if needed
if (stats.timeOfLastFailure === 0) {
stats.timeOfLastFailure = new Date().getTime();
}
} else {
stats.totalDownChecks++;
stats.timeOfLastFailure = 0;
}
// Calculate uptime percentage
let uptimePercentage;
if (stats.totalChecks > 0) {
uptimePercentage = stats.totalUpChecks / stats.totalChecks;
} else {
uptimePercentage = status === true ? 100 : 0;
}
stats.uptimePercentage = uptimePercentage;
// latest check
stats.lastCheckTimestamp = new Date().getTime();
await stats.save();
return true;
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
message: error.message,
method: "updateRunningStats",
stack: error.stack,
});
return false;
}
}
getStatusString = (status) => {
if (status === true) return "up";
if (status === false) return "down";
return "unknown";
};
/**
* Saves check if needed and adds to incident buffer
* Removes check from checks buffer if it was saved immediately
*
* @param {Object} check - The check object
* @param {Object} monitor - The monitor object
* @param {string} action - The incident action ("create" or "resolve")
* @param {string} errorContext - Context for error messages
* @returns {Promise<void>}
*/
handleIncidentForCheck = async (check, monitor, action, errorContext = "incident handling") => {
try {
let savedCheck = check;
if (!check._id) {
try {
const checkModel = new Check(check);
savedCheck = await checkModel.save();
this.buffer.removeCheckFromBuffer(check);
} catch (checkError) {
this.logger.error({
service: this.SERVICE_NAME,
method: "handleIncidentForCheck",
message: `Failed to save check immediately for ${errorContext}: ${checkError.message}`,
monitorId: monitor._id,
stack: checkError.stack,
});
savedCheck = null;
}
}
if (savedCheck && savedCheck._id) {
try {
this.buffer.addIncidentToBuffer({ monitor, check: savedCheck, action });
} catch (incidentError) {
this.logger.error({
service: this.SERVICE_NAME,
method: "handleIncidentForCheck",
message: `Failed to add incident to buffer for ${errorContext}: ${incidentError.message}`,
monitorId: monitor._id,
action,
stack: incidentError.stack,
});
}
}
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "handleIncidentForCheck",
message: `Error in ${errorContext}: ${error.message}`,
monitorId: monitor?._id,
stack: error.stack,
});
}
};
/**
* Updates the status of a monitor based on the network response.
*
* @param {Object} networkResponse - The network response containing monitorId and status.
* @param {string} networkResponse.monitorId - The ID of the monitor.
* @param {string} networkResponse.status - The new status of the monitor.
* @returns {Promise<Object>} - A promise that resolves to an object containinfg the monitor, statusChanged flag, and previous status if the status changed, or false if an error occurred.
* @returns {Promise<Object>} returnObject - The object returned by the function.
* @returns {Object} returnObject.monitor - The monitor object.
* @returns {boolean} returnObject.statusChanged - Flag indicating if the status has changed.
* @returns {boolean} returnObject.prevStatus - The previous status of the monitor
*/
updateStatus = async (networkResponse) => {
const check = this.buildCheck(networkResponse);
await this.insertCheck(check);
try {
const { monitorId, status, code } = networkResponse;
const monitor = await this.db.monitorModule.getMonitorById(monitorId);
// Update running stats
this.updateRunningStats({ monitor, networkResponse });
// If the status window size has changed, empty
while (monitor.statusWindow.length > monitor.statusWindowSize) {
monitor.statusWindow.shift();
}
// Update status sliding window
monitor.statusWindow.push(status);
if (monitor.statusWindow.length > monitor.statusWindowSize) {
monitor.statusWindow.shift();
}
if (monitor.status === undefined || monitor.status === null) {
monitor.status = status;
}
let newStatus = monitor.status;
let statusChanged = false;
const prevStatus = monitor.status;
// Return early if not enough data points
if (monitor.statusWindow.length < monitor.statusWindowSize) {
await monitor.save();
return {
monitor,
statusChanged: false,
prevStatus,
code,
timestamp: Date.now(),
};
}
// Check if threshold has been met
const failures = monitor.statusWindow.filter((s) => s === false).length;
const failureRate = (failures / monitor.statusWindow.length) * 100;
// If threshold has been met and the monitor is not already down, mark down:
if (failureRate >= monitor.statusWindowThreshold && monitor.status !== false) {
newStatus = false;
statusChanged = true;
}
// If the failure rate is below the threshold and the monitor is down, recover:
else if (failureRate < monitor.statusWindowThreshold && monitor.status === false) {
newStatus = true;
statusChanged = true;
}
if (statusChanged) {
this.logger.info({
service: this.SERVICE_NAME,
message: `${monitor.name} went from ${this.getStatusString(prevStatus)} to ${this.getStatusString(newStatus)}`,
prevStatus,
newStatus,
});
if (newStatus === false) {
await this.handleIncidentForCheck(check, monitor, "create", "status change to down");
} else if (prevStatus === false) {
await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up");
}
}
if (monitor.status === false && !statusChanged) {
try {
const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor._id);
let calculatedFailureRate = failureRate;
if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) {
try {
const checksAfterResolution = await Check.find({
monitorId: monitor._id,
createdAt: { $gt: lastManuallyResolvedIncident.endTime },
})
.sort({ createdAt: 1 })
.limit(monitor.statusWindowSize)
.select("status")
.lean();
if (checksAfterResolution.length > 0) {
const checksStatuses = checksAfterResolution.map((c) => c.status);
const failuresAfterResolution = checksStatuses.filter((s) => s === false).length;
calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100;
} else {
calculatedFailureRate = 0;
}
} catch (checkQueryError) {
this.logger.error({
service: this.SERVICE_NAME,
method: "updateStatus",
message: `Failed to query checks after manual resolution: ${checkQueryError.message}`,
monitorId: monitor._id,
stack: checkQueryError.stack,
});
}
}
if (calculatedFailureRate >= monitor.statusWindowThreshold) {
await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change");
}
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "updateStatus",
message: `Error handling threshold check without status change: ${error.message}`,
monitorId: monitor._id,
stack: error.stack,
});
}
}
monitor.status = newStatus;
await monitor.save();
return {
monitor,
statusChanged,
prevStatus,
code,
timestamp: new Date().getTime(),
};
} catch (error) {
error.service = this.SERVICE_NAME;
error.method = "updateStatus";
throw error;
}
};
/**
* Builds a check object from the network response.
*
* @param {Object} networkResponse - The network response object.
* @param {string} networkResponse.monitorId - The monitor ID.
* @param {string} networkResponse.type - The type of the response.
* @param {string} networkResponse.status - The status of the response.
* @param {number} networkResponse.responseTime - The response time.
* @param {number} networkResponse.code - The status code.
* @param {string} networkResponse.message - The message.
* @param {Object} networkResponse.payload - The payload of the response.
* @returns {Object} The check object.
*/
buildCheck = (networkResponse) => {
const {
monitorId,
teamId,
type,
status,
responseTime,
code,
message,
payload,
first_byte_took,
body_read_took,
dns_took,
conn_took,
connect_took,
tls_took,
timings,
} = networkResponse;
const check = {
metadata: {
monitorId,
teamId,
type,
},
status,
statusCode: code,
responseTime,
timings: timings || {},
message,
first_byte_took,
body_read_took,
dns_took,
conn_took,
connect_took,
tls_took,
};
if (type === "pagespeed") {
if (typeof payload === "undefined") {
this.logger.warn({
message: "Failed to build check",
service: this.SERVICE_NAME,
method: "buildCheck",
details: "empty payload",
});
return undefined;
}
const categories = payload?.lighthouseResult?.categories ?? {};
const audits = payload?.lighthouseResult?.audits ?? {};
const {
"cumulative-layout-shift": cls = 0,
"speed-index": si = 0,
"first-contentful-paint": fcp = 0,
"largest-contentful-paint": lcp = 0,
"total-blocking-time": tbt = 0,
} = audits;
check.accessibility = (categories?.accessibility?.score || 0) * 100;
check.bestPractices = (categories?.["best-practices"]?.score || 0) * 100;
check.seo = (categories?.seo?.score || 0) * 100;
check.performance = (categories?.performance?.score || 0) * 100;
check.audits = { cls, si, fcp, lcp, tbt };
}
if (type === "hardware") {
const { cpu, memory, disk, host, net } = payload?.data ?? {};
const { errors } = payload?.errors ?? [];
check.cpu = cpu ?? {};
check.memory = memory ?? {};
check.disk = disk ?? {};
check.host = host ?? {};
check.errors = errors ?? [];
check.capture = payload?.capture ?? {};
check.net = net ?? {};
}
return check;
};
/**
* Inserts a check into the database based on the network response.
*
* @param {Object} networkResponse - The network response object.
* @param {string} networkResponse.monitorId - The monitor ID.
* @param {string} networkResponse.type - The type of the response.
* @param {string} networkResponse.status - The status of the response.
* @param {number} networkResponse.responseTime - The response time.
* @param {number} networkResponse.code - The status code.
* @param {string} networkResponse.message - The message.
* @param {Object} networkResponse.payload - The payload of the response.
* @returns {Promise<void>} A promise that resolves when the check is inserted.
*/
insertCheck = async (check) => {
try {
if (typeof check === "undefined") {
this.logger.warn({
message: "Failed to build check",
service: this.SERVICE_NAME,
method: "insertCheck",
});
return false;
}
this.buffer.addToBuffer({ check });
return true;
} catch (error) {
this.logger.error({
message: error.message,
service: error.service || this.SERVICE_NAME,
method: error.method || "insertCheck",
details: error.details || `Error inserting check for monitor: ${check?.monitorId}`,
stack: error.stack,
});
}
};
}
export default StatusService;