Incidents management implementation

This commit is contained in:
karenvicent committed 2025-11-18 16:01:47 -05:00
1 parent eb65c41d15
commit 131ced10fc
10 files changed
+713 -139

No files matched your search

+5
View File
@@ -14,6 +14,7 @@ import LogController from "../controllers/v1/logController.js";
import StatusPageController from "../controllers/v1/statusPageController.js";
import NotificationController from "../controllers/v1/notificationController.js";
import DiagnosticController from "../controllers/v1/diagnosticController.js";
import IncidentController from "../controllers/v1/incidentController.js";
// V2 Controllers
import AuthControllerV2 from "../controllers/v2/AuthController.js";
@@ -70,6 +71,10 @@ export const initializeControllers = (services) => {
diagnosticService: services.diagnosticService,
});
controllers.incidentController = new IncidentController(commonDependencies, {
incidentService: services.incidentService,
});
//V2
controllers.authControllerV2 = new AuthControllerV2(services.authServiceV2, services.inviteServiceV2);
controllers.inviteControllerV2 = new InviteControllerV2(services.inviteServiceV2);
+4
View File
@@ -13,6 +13,8 @@ import LogRoutes from "../routes/v1/logRoutes.js";
import DiagnosticRoutes from "../routes/v1//diagnosticRoute.js";
import NotificationRoutes from "../routes/v1/notificationRoute.js";
import IncidentRoutes from "../routes/v1/incidentRoute.js";
//V2
import AuthRoutesV2 from "../routes/v2/auth.js";
import InviteRoutesV2 from "../routes/v2/invite.js";
@@ -34,6 +36,7 @@ export const setupRoutes = (app, controllers) => {
const statusPageRoutes = new StatusPageRoutes(controllers.statusPageController);
const notificationRoutes = new NotificationRoutes(controllers.notificationController);
const diagnosticRoutes = new DiagnosticRoutes(controllers.diagnosticController);
const incidentRoutes = new IncidentRoutes(controllers.incidentController);
app.use("/api/v1/auth", authApiLimiter, authRoutes.getRouter());
app.use("/api/v1/monitors", verifyJWT, monitorRoutes.getRouter());
@@ -46,6 +49,7 @@ export const setupRoutes = (app, controllers) => {
app.use("/api/v1/status-page", statusPageRoutes.getRouter());
app.use("/api/v1/notifications", verifyJWT, notificationRoutes.getRouter());
app.use("/api/v1/diagnostic", verifyJWT, diagnosticRoutes.getRouter());
app.use("/api/v1/incidents", verifyJWT, incidentRoutes.getRouter());
// V2
const authRoutesV2 = new AuthRoutesV2(controllers.authControllerV2);
+2 -2
View File
@@ -157,8 +157,6 @@ export const initializeServices = async ({ logger, envSettings, settingsService
settingsService,
});
const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger);
const bufferService = new BufferService({ db, logger, envSettings });
const errorService = new ErrorService();
const incidentService = new IncidentService({
@@ -168,6 +166,8 @@ export const initializeServices = async ({ logger, envSettings, settingsService
stringService,
});
const bufferService = new BufferService({ db, logger, envSettings, incidentService });
const statusService = new StatusService({ db, logger, buffer: bufferService, incidentService });
const notificationUtils = new NotificationUtils({
@@ -0,0 +1,184 @@
import BaseController from "./baseController.js";
const SERVICE_NAME = "incidentController";
/**
* Incident Controller
*
* Handles all incident-related HTTP requests including retrieving incidents,
* resolving incidents manually, and getting incident summaries.
*
* @class IncidentController
* @description Manages incident operations and tracking
*/
class IncidentController extends BaseController {
static SERVICE_NAME = SERVICE_NAME;
/**
* Creates an instance of IncidentController.
*
* @param {Object} commonDependencies - Common dependencies injected into the controller
* @param {Object} dependencies - The dependencies required by the controller
* @param {Object} dependencies.incidentService - Incident business logic service
*/
constructor(commonDependencies, { incidentService }) {
super(commonDependencies);
this.incidentService = incidentService;
}
get serviceName() {
return IncidentController.SERVICE_NAME;
}
/**
* Retrieves all incidents for the current user's team with filtering and pagination.
*
* @async
* @function getIncidentsByTeam
* @param {Object} req - Express request object
* @param {Object} req.query - Query parameters for filtering and pagination
* @param {string} [req.query.sortOrder] - Sort order (asc/desc)
* @param {string} [req.query.dateRange] - Date range filter
* @param {string} [req.query.filter] - General filter string
* @param {number} [req.query.page] - Page number for pagination
* @param {number} [req.query.rowsPerPage] - Number of rows per page
* @param {boolean} [req.query.status] - Filter by incident status (true=active, false=resolved)
* @param {string} [req.query.monitorId] - Filter by monitor ID
* @param {string} [req.query.resolutionType] - Filter by resolution type (automatic/manual)
* @param {Object} req.user - Current authenticated user (from JWT)
* @param {string} req.user.teamId - User's team ID
* @param {Object} res - Express response object
* @returns {Promise<Object>} Success response with incidents data
* @throws {Error} 422 - Validation error if query parameters are invalid
* @example
* GET /incidents/team?page=1&rowsPerPage=20&status=true&filter=active
* // Requires JWT authentication
*/
getIncidentsByTeam = this.asyncHandler(
async (req, res) => {
const result = await this.incidentService.getIncidentsByTeam({
teamId: req?.user?.teamId,
query: req?.query,
});
return res.success({
msg: "Incidents retrieved successfully",
data: result,
});
},
SERVICE_NAME,
"getIncidentsByTeam"
);
/**
* Retrieves a summary of incidents for the current user's team.
*
* @async
* @function getIncidentSummary
* @param {Object} req - Express request object
* @param {Object} req.query - Query parameters
* @param {string} [req.query.dateRange] - Date range filter
* @param {Object} req.user - Current authenticated user (from JWT)
* @param {string} req.user.teamId - User's team ID
* @param {Object} res - Express response object
* @returns {Promise<Object>} Success response with incidents summary
* @example
* GET /incidents/summary?dateRange=week
* // Requires JWT authentication
*/
getIncidentSummary = this.asyncHandler(
async (req, res) => {
const summary = await this.incidentService.getIncidentSummary({
teamId: req?.user?.teamId,
query: req?.query,
});
return res.success({
msg: "Incident summary retrieved successfully",
data: summary,
});
},
SERVICE_NAME,
"getIncidentSummary"
);
/**
* Retrieves a specific incident by ID.
*
* @async
* @function getIncidentById
* @param {Object} req - Express request object
* @param {Object} req.params - URL parameters
* @param {string} req.params.incidentId - ID of the incident to retrieve
* @param {Object} req.user - Current authenticated user (from JWT)
* @param {string} req.user.teamId - User's team ID
* @param {Object} res - Express response object
* @returns {Promise<Object>} Success response with incident data
* @throws {Error} 404 - Not found if incident doesn't exist
* @throws {Error} 403 - Forbidden if user doesn't have access to incident
* @example
* GET /incidents/507f1f77bcf86cd799439011
* // Requires JWT authentication
*/
getIncidentById = this.asyncHandler(
async (req, res) => {
const incident = await this.incidentService.getIncidentById({
incidentId: req?.params?.incidentId,
teamId: req?.user?.teamId,
});
return res.success({
msg: "Incident retrieved successfully",
data: incident,
});
},
SERVICE_NAME,
"getIncidentById"
);
/**
* Manually resolves a specific incident by ID.
*
* @async
* @function resolveIncidentManually
* @param {Object} req - Express request object
* @param {Object} req.params - URL parameters
* @param {string} req.params.incidentId - ID of the incident to resolve
* @param {Object} req.body - Request body containing resolution data
* @param {string} [req.body.comment] - Optional comment about the resolution
* @param {Object} req.user - Current authenticated user (from JWT)
* @param {string} req.user.teamId - User's team ID
* @param {string} req.user._id - User's ID
* @param {Object} res - Express response object
* @returns {Promise<Object>} Success response with updated incident data
* @throws {Error} 422 - Validation error if request body is invalid
* @throws {Error} 404 - Not found if incident doesn't exist
* @throws {Error} 403 - Forbidden if user doesn't have access to incident
* @throws {Error} 400 - Bad request if incident is already resolved
* @example
* PUT /incidents/507f1f77bcf86cd799439011/resolve
* {
* "comment": "Issue resolved by restarting the service"
* }
* // Requires JWT authentication
*/
resolveIncidentManually = this.asyncHandler(
async (req, res) => {
const resolvedIncident = await this.incidentService.resolveIncidentManually({
incidentId: req?.params?.incidentId,
userId: req?.user?._id,
teamId: req?.user?.teamId,
comment: req?.body?.comment,
});
return res.success({
msg: "Incident resolved successfully",
data: resolvedIncident,
});
},
SERVICE_NAME,
"resolveIncidentManually"
);
}
export default IncidentController;
+62 -58
View File
@@ -1,63 +1,67 @@
import mongoose from "mongoose";
const IncidentSchema = mongoose.Schema({
monitorId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Monitor",
required: true,
immutable: true,
index: true
},
teamId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Team",
required: true,
immutable: true,
index: true
},
startTime: {
type: Date,
required: true,
immutable: true
},
endTime: {
type: Date,
default: null
},
status: {
type: String,
enum: ["active", "resolved"],
default: "active",
index: true
},
message: {
type: String,
default: null
},
statusCode: {
type: Number,
index: true,
default: null
},
resolutionType: {
type: String,
enum: ["automatic", "manual"],
default: null
},
resolvedBy: {
type: mongoose.Schema.Types.ObjectId,
ref: "User",
default: null
},
comment: {
type: String,
default: null
},
checks: [{
type: mongoose.Schema.Types.ObjectId,
ref: "Check"
}]
}, { timestamps: true });
const IncidentSchema = mongoose.Schema(
{
monitorId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Monitor",
required: true,
immutable: true,
index: true,
},
teamId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Team",
required: true,
immutable: true,
index: true,
},
startTime: {
type: Date,
required: true,
immutable: true,
},
endTime: {
type: Date,
default: null,
},
status: {
type: Boolean,
default: true,
index: true,
},
message: {
type: String,
default: null,
},
statusCode: {
type: Number,
index: true,
default: null,
},
resolutionType: {
type: String,
enum: ["automatic", "manual"],
default: null,
},
resolvedBy: {
type: mongoose.Schema.Types.ObjectId,
ref: "User",
default: null,
},
comment: {
type: String,
default: null,
},
checks: [
{
type: mongoose.Schema.Types.ObjectId,
ref: "Check",
},
],
},
{ timestamps: true }
);
IncidentSchema.index({ monitorId: 1, status: 1 });
IncidentSchema.index({ teamId: 1, status: 1 });
+83 -36
View File
@@ -42,7 +42,7 @@ class IncidentModule {
getActiveIncidentByMonitor = async (monitorId) => {
try {
return await this.Incident.findOne({ monitorId: new ObjectId(monitorId), status: "active" });
return await this.Incident.findOne({ monitorId: new ObjectId(monitorId), status: true });
} catch (error) {
error.service = SERVICE_NAME;
error.method = "getActiveIncidentByMonitor";
@@ -50,11 +50,51 @@ class IncidentModule {
}
};
getActiveIncidentsByMonitors = async (monitorIds) => {
try {
if (!monitorIds || monitorIds.length === 0) {
return new Map();
}
const objectIds = monitorIds.map((id) => new ObjectId(id));
const incidents = await this.Incident.find({
monitorId: { $in: objectIds },
status: true,
});
const map = new Map();
incidents.forEach((incident) => {
const monitorIdStr = incident.monitorId.toString();
map.set(monitorIdStr, incident);
});
return map;
} catch (error) {
error.service = SERVICE_NAME;
error.method = "getActiveIncidentsByMonitors";
throw error;
}
};
getLastManuallyResolvedIncident = async (monitorId) => {
try {
return await this.Incident.findOne({
monitorId: new ObjectId(monitorId),
status: false,
resolutionType: "manual",
})
.sort({ endTime: -1 })
.limit(1);
} catch (error) {
error.service = SERVICE_NAME;
error.method = "getLastManuallyResolvedIncident";
throw error;
}
};
getIncidentById = async (incidentId) => {
try {
return await this.Incident.findById(incidentId)
.populate("monitorId", "name type url")
.populate("resolvedBy", "name email");
return await this.Incident.findById(incidentId).populate("monitorId", "name type url").populate("resolvedBy", "name email");
} catch (error) {
error.service = SERVICE_NAME;
error.method = "getIncidentById";
@@ -65,17 +105,13 @@ class IncidentModule {
resolveIncident = async (incidentId, { resolutionType = "automatic", resolvedBy = null, comment = null, endTime = new Date() } = {}) => {
try {
const update = {
status: "resolved",
status: false,
endTime,
resolutionType,
resolvedBy,
...(comment !== null && { comment }),
};
const incident = await this.Incident.findOneAndUpdate(
{ _id: new ObjectId(incidentId) },
{ $set: update },
{ new: true }
);
const incident = await this.Incident.findOneAndUpdate({ _id: new ObjectId(incidentId) }, { $set: update }, { new: true });
if (!incident) {
throw new Error("Incident not found");
@@ -109,14 +145,44 @@ class IncidentModule {
}
};
getIncidentsByTeam = async ({ teamId, sortOrder, dateRange, filter, page, rowsPerPage, status, monitorId, resolutionType }) => {
addChecksToIncidentsBatch = async (operations) => {
try {
page = parseInt(page);
rowsPerPage = parseInt(rowsPerPage);
if (!operations || operations.length === 0) {
return;
}
const bulkOps = operations.map((op) => ({
updateOne: {
filter: { _id: new ObjectId(op.incidentId) },
update: { $addToSet: { checks: new ObjectId(op.checkId) } },
},
}));
await this.Incident.bulkWrite(bulkOps, { ordered: false });
} catch (error) {
error.service = SERVICE_NAME;
error.method = "addChecksToIncidentsBatch";
throw error;
}
};
getIncidentsByTeam = async ({ teamId, sortOrder, dateRange, page, rowsPerPage, status, monitorId, resolutionType }) => {
try {
page = Number.isFinite(parseInt(page)) ? parseInt(page) : 0;
rowsPerPage = Number.isFinite(parseInt(rowsPerPage)) ? parseInt(rowsPerPage) : 20;
let statusBoolean = undefined;
if (status !== undefined && status !== null) {
if (typeof status === "string") {
statusBoolean = status === "true";
} else if (typeof status === "boolean") {
statusBoolean = status;
}
}
const matchStage = {
teamId: new ObjectId(teamId),
...(status && { status }),
...(statusBoolean !== undefined && { status: statusBoolean }),
...(monitorId && { monitorId: new ObjectId(monitorId) }),
...(resolutionType && { resolutionType }),
...(dateRangeLookup[dateRange] && {
@@ -126,27 +192,9 @@ class IncidentModule {
}),
};
if (filter !== undefined) {
switch (filter) {
case "all":
break;
case "active":
matchStage.status = "active";
break;
case "resolved":
matchStage.status = "resolved";
break;
default:
break;
}
}
sortOrder = sortOrder === "asc" ? 1 : -1;
let skip = 0;
if (page && rowsPerPage) {
skip = page * rowsPerPage;
}
const skip = page * rowsPerPage;
const incidents = await this.Incident.aggregate([
{ $match: matchStage },
@@ -212,10 +260,10 @@ class IncidentModule {
summary.forEach((item) => {
result.total += item.count;
if (item._id === "active") {
if (item._id === true) {
result.active = item.count;
}
if (item._id === "resolved") {
if (item._id === false) {
result.resolved = item.count;
}
result.manual += item.manualResolutions;
@@ -255,4 +303,3 @@ class IncidentModule {
}
export default IncidentModule;
+26
View File
@@ -0,0 +1,26 @@
import { Router } from "express";
import { isAllowed } from "../../middleware/v1/isAllowed.js";
class IncidentRoutes {
constructor(incidentController) {
this.router = Router();
this.incidentController = incidentController;
this.initRoutes();
}
initRoutes() {
// Team routes
this.router.get("/team", this.incidentController.getIncidentsByTeam);
this.router.get("/team/summary", this.incidentController.getIncidentSummary);
// Individual incident routes
this.router.get("/:incidentId", this.incidentController.getIncidentById);
this.router.put("/:incidentId/resolve", isAllowed(["admin", "superadmin"]), this.incidentController.resolveIncidentManually);
}
getRouter() {
return this.router;
}
}
export default IncidentRoutes;
@@ -28,7 +28,7 @@ class IncidentService {
if (activeIncident) {
await this.db.incidentModule.addCheckToIncident(activeIncident._id, check._id);
this.logger.info({
service: this.SERVICE_NAME,
method: "createIncident",
@@ -45,7 +45,7 @@ class IncidentService {
teamId: monitor.teamId,
type: monitor.type,
startTime: new Date(),
status: "active",
status: true,
message: check.message || null,
statusCode: check.statusCode || null,
checks: [check._id],
@@ -121,7 +121,6 @@ class IncidentService {
}
};
resolveIncidentManually = async ({ incidentId, userId, teamId, comment }) => {
try {
if (!incidentId) {
@@ -146,7 +145,7 @@ class IncidentService {
throw this.errorService.createAuthorizationError();
}
if (incident.status === "resolved") {
if (incident.status === false) {
throw this.errorService.createBadRequestError("Incident is already resolved");
}
@@ -178,20 +177,18 @@ class IncidentService {
}
};
getIncidentsByTeam = async ({ teamId, query }) => {
try {
if (!teamId) {
throw this.errorService.createBadRequestError("No team ID in request");
}
const { sortOrder, dateRange, filter, page, rowsPerPage, status, monitorId, resolutionType } = query || {};
const { sortOrder, dateRange, page, rowsPerPage, status, monitorId, resolutionType } = query || {};
const result = await this.db.incidentModule.getIncidentsByTeam({
teamId,
sortOrder,
dateRange,
filter,
page,
rowsPerPage,
status,
@@ -212,7 +209,6 @@ class IncidentService {
}
};
getIncidentSummary = async ({ teamId, query }) => {
try {
if (!teamId) {
@@ -271,7 +267,148 @@ class IncidentService {
throw error;
}
};
processIncidentsFromBuffer = async (incidentBufferItems) => {
try {
if (!incidentBufferItems || incidentBufferItems.length === 0) {
return;
}
const createItems = [];
const resolveItems = [];
for (const item of incidentBufferItems) {
if (item.action === "resolve") {
resolveItems.push(item);
} else {
createItems.push(item);
}
}
for (const item of resolveItems) {
try {
await this.resolveIncident(item.monitor, item.check);
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: `Failed to resolve incident from buffer: ${error.message}`,
monitorId: item.monitor?._id,
error: error.stack,
});
}
}
if (createItems.length === 0) {
return;
}
const groupedByMonitor = {};
for (const item of createItems) {
if (!item.monitor || !item.monitor._id || !item.check || !item.check._id) {
this.logger.warn({
service: this.SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: "Skipping item with missing monitor or check data",
item,
});
continue;
}
const monitorId = item.monitor._id.toString();
if (!groupedByMonitor[monitorId]) {
groupedByMonitor[monitorId] = [];
}
groupedByMonitor[monitorId].push(item);
}
const monitorIds = Object.keys(groupedByMonitor);
if (monitorIds.length === 0) {
return;
}
const activeIncidents = await this.db.incidentModule.getActiveIncidentsByMonitors(monitorIds);
const incidentsCreatedInFlush = {};
const checksToAddToIncidents = [];
const newIncidentsToCreate = [];
for (const [monitorId, items] of Object.entries(groupedByMonitor)) {
const existingIncident = activeIncidents.get(monitorId) || incidentsCreatedInFlush[monitorId];
if (existingIncident) {
const incidentId = existingIncident._id ? existingIncident._id.toString() : existingIncident;
for (const item of items) {
checksToAddToIncidents.push({
incidentId,
checkId: item.check._id.toString(),
});
}
} else {
const firstItem = items[0];
const incidentData = {
monitorId: firstItem.monitor._id,
teamId: firstItem.monitor.teamId,
type: firstItem.monitor.type,
startTime: new Date(),
status: true,
message: firstItem.check.message || null,
statusCode: firstItem.check.statusCode || null,
checks: [firstItem.check._id],
};
newIncidentsToCreate.push({
incidentData,
monitorId,
remainingChecks: items.slice(1), // Checks restantes para agregar después
});
}
}
if (newIncidentsToCreate.length > 0) {
const incidentDataArray = newIncidentsToCreate.map((item) => item.incidentData);
await this.db.incidentModule.createIncidents(incidentDataArray);
const createdIncidentsMap = await this.db.incidentModule.getActiveIncidentsByMonitors(newIncidentsToCreate.map((item) => item.monitorId));
for (const item of newIncidentsToCreate) {
const createdIncident = createdIncidentsMap.get(item.monitorId);
if (createdIncident && createdIncident._id) {
const incidentId = createdIncident._id.toString();
incidentsCreatedInFlush[item.monitorId] = incidentId;
for (const remainingItem of item.remainingChecks) {
checksToAddToIncidents.push({
incidentId,
checkId: remainingItem.check._id.toString(),
});
}
}
}
}
if (checksToAddToIncidents.length > 0) {
await this.db.incidentModule.addChecksToIncidentsBatch(checksToAddToIncidents);
}
this.logger.info({
service: this.SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: `Processed ${incidentBufferItems.length} incident buffer items`,
created: newIncidentsToCreate.length,
checksAdded: checksToAddToIncidents.length,
resolved: resolveItems.length,
});
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: error.message,
error: error.stack,
});
throw error;
}
};
}
export default IncidentService;
@@ -2,12 +2,14 @@ const SERVICE_NAME = "BufferService";
class BufferService {
static SERVICE_NAME = SERVICE_NAME;
constructor({ db, logger, envSettings }) {
constructor({ db, logger, envSettings, incidentService }) {
this.BUFFER_TIMEOUT = envSettings.nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute
this.db = db;
this.logger = logger;
this.incidentService = incidentService;
this.SERVICE_NAME = SERVICE_NAME;
this.buffer = [];
this.incidentBuffer = [];
this.scheduleNextFlush();
this.logger.info({
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
@@ -33,6 +35,66 @@ class BufferService {
}
}
addIncidentToBuffer({ monitor, check, action = "create" }) {
try {
if (!monitor || !check) {
this.logger.warn({
message: "Skipping incident buffer item: missing monitor or check",
service: this.SERVICE_NAME,
method: "addIncidentToBuffer",
});
return;
}
this.incidentBuffer.push({ monitor, check, action });
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "addIncidentToBuffer",
stack: error.stack,
});
}
}
removeCheckFromBuffer(checkToRemove) {
try {
if (!checkToRemove) {
return false;
}
const index = this.buffer.findIndex((check) => {
if (checkToRemove._id && check._id) {
return check._id.toString() === checkToRemove._id.toString();
}
return (
check.monitorId?.toString() === checkToRemove.monitorId?.toString() &&
check.teamId?.toString() === checkToRemove.teamId?.toString() &&
check.type === checkToRemove.type &&
check.status === checkToRemove.status &&
check.statusCode === checkToRemove.statusCode &&
check.responseTime === checkToRemove.responseTime &&
check.message === checkToRemove.message
);
});
if (index !== -1) {
this.buffer.splice(index, 1);
return true;
}
return false;
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "removeCheckFromBuffer",
stack: error.stack,
});
return false;
}
}
scheduleNextFlush() {
this.bufferTimer = setTimeout(async () => {
try {
@@ -51,10 +113,10 @@ class BufferService {
}, this.BUFFER_TIMEOUT);
}
async flushBuffer() {
let items = this.buffer.length;
try {
await this.db.checkModule.createChecks(this.buffer);
if (this.buffer.length > 0) {
await this.db.checkModule.createChecks(this.buffer);
}
} catch (error) {
this.logger.error({
message: error.message,
@@ -63,13 +125,41 @@ class BufferService {
stack: error.stack,
});
}
try {
if (this.incidentBuffer.length > 0 && this.incidentService) {
await this.flushIncidentBuffer();
}
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "flushBuffer",
stack: error.stack,
});
}
this.buffer = [];
// this.logger.debug({
// message: `Flushed ${items} items`,
// service: this.SERVICE_NAME,
// method: "flushBuffer",
// });
items = 0;
this.incidentBuffer = [];
}
async flushIncidentBuffer() {
if (!this.incidentService || this.incidentBuffer.length === 0) {
return;
}
try {
const itemsToProcess = [...this.incidentBuffer];
await this.incidentService.processIncidentsFromBuffer(itemsToProcess);
} catch (error) {
this.logger.error({
message: `Error flushing incident buffer: ${error.message}`,
service: this.SERVICE_NAME,
method: "flushIncidentBuffer",
stack: error.stack,
});
throw error;
}
}
}
@@ -98,6 +98,63 @@ class StatusService {
if (status === false) return "down";
return "unknown";
};
/**
* Saves check if needed and adds to incident buffer
* Removes check from checks buffer if it was saved immediately
*
* @param {Object} check - The check object
* @param {Object} monitor - The monitor object
* @param {string} action - The incident action ("create" or "resolve")
* @param {string} errorContext - Context for error messages
* @returns {Promise<void>}
*/
handleIncidentForCheck = async (check, monitor, action, errorContext = "incident handling") => {
try {
let savedCheck = check;
if (!check._id) {
try {
const checkModel = new Check(check);
savedCheck = await checkModel.save();
this.buffer.removeCheckFromBuffer(check);
} catch (checkError) {
this.logger.error({
service: this.SERVICE_NAME,
method: "handleIncidentForCheck",
message: `Failed to save check immediately for ${errorContext}: ${checkError.message}`,
monitorId: monitor._id,
stack: checkError.stack,
});
savedCheck = null;
}
}
if (savedCheck && savedCheck._id) {
try {
this.buffer.addIncidentToBuffer({ monitor, check: savedCheck, action });
} catch (incidentError) {
this.logger.error({
service: this.SERVICE_NAME,
method: "handleIncidentForCheck",
message: `Failed to add incident to buffer for ${errorContext}: ${incidentError.message}`,
monitorId: monitor._id,
action,
stack: incidentError.stack,
});
}
}
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "handleIncidentForCheck",
message: `Error in ${errorContext}: ${error.message}`,
monitorId: monitor?._id,
stack: error.stack,
});
}
};
/**
* Updates the status of a monitor based on the network response.
*
@@ -174,38 +231,58 @@ class StatusService {
newStatus,
});
let savedCheck = check;
if (!check._id) {
try {
const checkModel = new Check(check);
savedCheck = await checkModel.save();
} catch (checkError) {
this.logger.error({
service: this.SERVICE_NAME,
method: "updateStatus",
message: `Failed to save check immediately: ${checkError.message}`,
monitorId: monitor._id,
stack: checkError.stack,
});
}
if (newStatus === false) {
await this.handleIncidentForCheck(check, monitor, "create", "status change to down");
} else if (prevStatus === false) {
await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up");
}
}
if (monitor.status === false && !statusChanged) {
try {
if (newStatus === false) {
await this.incidentService.createIncident(monitor, savedCheck);
} else if (prevStatus === false) {
await this.incidentService.resolveIncident(monitor, savedCheck);
const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor._id);
let calculatedFailureRate = failureRate;
if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) {
try {
const checksAfterResolution = await Check.find({
monitorId: monitor._id,
createdAt: { $gt: lastManuallyResolvedIncident.endTime },
})
.sort({ createdAt: 1 })
.limit(monitor.statusWindowSize)
.select("status")
.lean();
if (checksAfterResolution.length > 0) {
const checksStatuses = checksAfterResolution.map((c) => c.status);
const failuresAfterResolution = checksStatuses.filter((s) => s === false).length;
calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100;
} else {
calculatedFailureRate = 0;
}
} catch (checkQueryError) {
this.logger.error({
service: this.SERVICE_NAME,
method: "updateStatus",
message: `Failed to query checks after manual resolution: ${checkQueryError.message}`,
monitorId: monitor._id,
stack: checkQueryError.stack,
});
}
}
} catch (incidentError) {
if (calculatedFailureRate >= monitor.statusWindowThreshold) {
await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change");
}
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "updateStatus",
message: `Failed to handle incident: ${incidentError.message}`,
message: `Error handling threshold check without status change: ${error.message}`,
monitorId: monitor._id,
statusChanged,
prevStatus,
newStatus,
stack: incidentError.stack,
stack: error.stack,
});
}
}