From 131ced10fc14907846b363d4b54f68a764edc30d Mon Sep 17 00:00:00 2001 From: karenvicent Date: Tue, 18 Nov 2025 16:01:47 -0500 Subject: [PATCH] Incidents management implementation --- server/src/config/controllers.js | 5 + server/src/config/routes.js | 4 + server/src/config/services.js | 4 +- .../src/controllers/v1/incidentController.js | 184 ++++++++++++++++++ server/src/db/v1/models/Incident.js | 120 ++++++------ server/src/db/v1/modules/incidentModule.js | 119 +++++++---- server/src/routes/v1/incidentRoute.js | 26 +++ .../service/v1/business/incidentService.js | 155 ++++++++++++++- .../v1/infrastructure/bufferService.js | 110 ++++++++++- .../v1/infrastructure/statusService.js | 125 +++++++++--- 10 files changed, 713 insertions(+), 139 deletions(-) create mode 100644 server/src/controllers/v1/incidentController.js create mode 100644 server/src/routes/v1/incidentRoute.js diff --git a/server/src/config/controllers.js b/server/src/config/controllers.js index 927acd1a2..746a447a3 100644 --- a/server/src/config/controllers.js +++ b/server/src/config/controllers.js @@ -14,6 +14,7 @@ import LogController from "../controllers/v1/logController.js"; import StatusPageController from "../controllers/v1/statusPageController.js"; import NotificationController from "../controllers/v1/notificationController.js"; import DiagnosticController from "../controllers/v1/diagnosticController.js"; +import IncidentController from "../controllers/v1/incidentController.js"; // V2 Controllers import AuthControllerV2 from "../controllers/v2/AuthController.js"; @@ -70,6 +71,10 @@ export const initializeControllers = (services) => { diagnosticService: services.diagnosticService, }); + controllers.incidentController = new IncidentController(commonDependencies, { + incidentService: services.incidentService, + }); + //V2 controllers.authControllerV2 = new AuthControllerV2(services.authServiceV2, services.inviteServiceV2); controllers.inviteControllerV2 = new InviteControllerV2(services.inviteServiceV2); diff --git a/server/src/config/routes.js b/server/src/config/routes.js index 13b369262..5dace493c 100644 --- a/server/src/config/routes.js +++ b/server/src/config/routes.js @@ -13,6 +13,8 @@ import LogRoutes from "../routes/v1/logRoutes.js"; import DiagnosticRoutes from "../routes/v1//diagnosticRoute.js"; import NotificationRoutes from "../routes/v1/notificationRoute.js"; +import IncidentRoutes from "../routes/v1/incidentRoute.js"; + //V2 import AuthRoutesV2 from "../routes/v2/auth.js"; import InviteRoutesV2 from "../routes/v2/invite.js"; @@ -34,6 +36,7 @@ export const setupRoutes = (app, controllers) => { const statusPageRoutes = new StatusPageRoutes(controllers.statusPageController); const notificationRoutes = new NotificationRoutes(controllers.notificationController); const diagnosticRoutes = new DiagnosticRoutes(controllers.diagnosticController); + const incidentRoutes = new IncidentRoutes(controllers.incidentController); app.use("/api/v1/auth", authApiLimiter, authRoutes.getRouter()); app.use("/api/v1/monitors", verifyJWT, monitorRoutes.getRouter()); @@ -46,6 +49,7 @@ export const setupRoutes = (app, controllers) => { app.use("/api/v1/status-page", statusPageRoutes.getRouter()); app.use("/api/v1/notifications", verifyJWT, notificationRoutes.getRouter()); app.use("/api/v1/diagnostic", verifyJWT, diagnosticRoutes.getRouter()); + app.use("/api/v1/incidents", verifyJWT, incidentRoutes.getRouter()); // V2 const authRoutesV2 = new AuthRoutesV2(controllers.authControllerV2); diff --git a/server/src/config/services.js b/server/src/config/services.js index cecc3bba9..004acc885 100644 --- a/server/src/config/services.js +++ b/server/src/config/services.js @@ -157,8 +157,6 @@ export const initializeServices = async ({ logger, envSettings, settingsService settingsService, }); const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger); - const bufferService = new BufferService({ db, logger, envSettings }); - const errorService = new ErrorService(); const incidentService = new IncidentService({ @@ -168,6 +166,8 @@ export const initializeServices = async ({ logger, envSettings, settingsService stringService, }); + const bufferService = new BufferService({ db, logger, envSettings, incidentService }); + const statusService = new StatusService({ db, logger, buffer: bufferService, incidentService }); const notificationUtils = new NotificationUtils({ diff --git a/server/src/controllers/v1/incidentController.js b/server/src/controllers/v1/incidentController.js new file mode 100644 index 000000000..27bb93ee8 --- /dev/null +++ b/server/src/controllers/v1/incidentController.js @@ -0,0 +1,184 @@ +import BaseController from "./baseController.js"; + +const SERVICE_NAME = "incidentController"; + +/** + * Incident Controller + * + * Handles all incident-related HTTP requests including retrieving incidents, + * resolving incidents manually, and getting incident summaries. + * + * @class IncidentController + * @description Manages incident operations and tracking + */ +class IncidentController extends BaseController { + static SERVICE_NAME = SERVICE_NAME; + + /** + * Creates an instance of IncidentController. + * + * @param {Object} commonDependencies - Common dependencies injected into the controller + * @param {Object} dependencies - The dependencies required by the controller + * @param {Object} dependencies.incidentService - Incident business logic service + */ + constructor(commonDependencies, { incidentService }) { + super(commonDependencies); + this.incidentService = incidentService; + } + + get serviceName() { + return IncidentController.SERVICE_NAME; + } + + /** + * Retrieves all incidents for the current user's team with filtering and pagination. + * + * @async + * @function getIncidentsByTeam + * @param {Object} req - Express request object + * @param {Object} req.query - Query parameters for filtering and pagination + * @param {string} [req.query.sortOrder] - Sort order (asc/desc) + * @param {string} [req.query.dateRange] - Date range filter + * @param {string} [req.query.filter] - General filter string + * @param {number} [req.query.page] - Page number for pagination + * @param {number} [req.query.rowsPerPage] - Number of rows per page + * @param {boolean} [req.query.status] - Filter by incident status (true=active, false=resolved) + * @param {string} [req.query.monitorId] - Filter by monitor ID + * @param {string} [req.query.resolutionType] - Filter by resolution type (automatic/manual) + * @param {Object} req.user - Current authenticated user (from JWT) + * @param {string} req.user.teamId - User's team ID + * @param {Object} res - Express response object + * @returns {Promise} Success response with incidents data + * @throws {Error} 422 - Validation error if query parameters are invalid + * @example + * GET /incidents/team?page=1&rowsPerPage=20&status=true&filter=active + * // Requires JWT authentication + */ + getIncidentsByTeam = this.asyncHandler( + async (req, res) => { + const result = await this.incidentService.getIncidentsByTeam({ + teamId: req?.user?.teamId, + query: req?.query, + }); + + return res.success({ + msg: "Incidents retrieved successfully", + data: result, + }); + }, + SERVICE_NAME, + "getIncidentsByTeam" + ); + + /** + * Retrieves a summary of incidents for the current user's team. + * + * @async + * @function getIncidentSummary + * @param {Object} req - Express request object + * @param {Object} req.query - Query parameters + * @param {string} [req.query.dateRange] - Date range filter + * @param {Object} req.user - Current authenticated user (from JWT) + * @param {string} req.user.teamId - User's team ID + * @param {Object} res - Express response object + * @returns {Promise} Success response with incidents summary + * @example + * GET /incidents/summary?dateRange=week + * // Requires JWT authentication + */ + getIncidentSummary = this.asyncHandler( + async (req, res) => { + const summary = await this.incidentService.getIncidentSummary({ + teamId: req?.user?.teamId, + query: req?.query, + }); + + return res.success({ + msg: "Incident summary retrieved successfully", + data: summary, + }); + }, + SERVICE_NAME, + "getIncidentSummary" + ); + + /** + * Retrieves a specific incident by ID. + * + * @async + * @function getIncidentById + * @param {Object} req - Express request object + * @param {Object} req.params - URL parameters + * @param {string} req.params.incidentId - ID of the incident to retrieve + * @param {Object} req.user - Current authenticated user (from JWT) + * @param {string} req.user.teamId - User's team ID + * @param {Object} res - Express response object + * @returns {Promise} Success response with incident data + * @throws {Error} 404 - Not found if incident doesn't exist + * @throws {Error} 403 - Forbidden if user doesn't have access to incident + * @example + * GET /incidents/507f1f77bcf86cd799439011 + * // Requires JWT authentication + */ + getIncidentById = this.asyncHandler( + async (req, res) => { + const incident = await this.incidentService.getIncidentById({ + incidentId: req?.params?.incidentId, + teamId: req?.user?.teamId, + }); + + return res.success({ + msg: "Incident retrieved successfully", + data: incident, + }); + }, + SERVICE_NAME, + "getIncidentById" + ); + + /** + * Manually resolves a specific incident by ID. + * + * @async + * @function resolveIncidentManually + * @param {Object} req - Express request object + * @param {Object} req.params - URL parameters + * @param {string} req.params.incidentId - ID of the incident to resolve + * @param {Object} req.body - Request body containing resolution data + * @param {string} [req.body.comment] - Optional comment about the resolution + * @param {Object} req.user - Current authenticated user (from JWT) + * @param {string} req.user.teamId - User's team ID + * @param {string} req.user._id - User's ID + * @param {Object} res - Express response object + * @returns {Promise} Success response with updated incident data + * @throws {Error} 422 - Validation error if request body is invalid + * @throws {Error} 404 - Not found if incident doesn't exist + * @throws {Error} 403 - Forbidden if user doesn't have access to incident + * @throws {Error} 400 - Bad request if incident is already resolved + * @example + * PUT /incidents/507f1f77bcf86cd799439011/resolve + * { + * "comment": "Issue resolved by restarting the service" + * } + * // Requires JWT authentication + */ + resolveIncidentManually = this.asyncHandler( + async (req, res) => { + const resolvedIncident = await this.incidentService.resolveIncidentManually({ + incidentId: req?.params?.incidentId, + userId: req?.user?._id, + teamId: req?.user?.teamId, + comment: req?.body?.comment, + }); + + return res.success({ + msg: "Incident resolved successfully", + data: resolvedIncident, + }); + }, + SERVICE_NAME, + "resolveIncidentManually" + ); +} + +export default IncidentController; diff --git a/server/src/db/v1/models/Incident.js b/server/src/db/v1/models/Incident.js index c37df5bde..a9477301b 100644 --- a/server/src/db/v1/models/Incident.js +++ b/server/src/db/v1/models/Incident.js @@ -1,63 +1,67 @@ import mongoose from "mongoose"; -const IncidentSchema = mongoose.Schema({ - monitorId: { - type: mongoose.Schema.Types.ObjectId, - ref: "Monitor", - required: true, - immutable: true, - index: true - }, - teamId: { - type: mongoose.Schema.Types.ObjectId, - ref: "Team", - required: true, - immutable: true, - index: true - }, - startTime: { - type: Date, - required: true, - immutable: true - }, - endTime: { - type: Date, - default: null - }, - status: { - type: String, - enum: ["active", "resolved"], - default: "active", - index: true - }, - message: { - type: String, - default: null - }, - statusCode: { - type: Number, - index: true, - default: null - }, - resolutionType: { - type: String, - enum: ["automatic", "manual"], - default: null - }, - resolvedBy: { - type: mongoose.Schema.Types.ObjectId, - ref: "User", - default: null - }, - comment: { - type: String, - default: null - }, - checks: [{ - type: mongoose.Schema.Types.ObjectId, - ref: "Check" - }] -}, { timestamps: true }); +const IncidentSchema = mongoose.Schema( + { + monitorId: { + type: mongoose.Schema.Types.ObjectId, + ref: "Monitor", + required: true, + immutable: true, + index: true, + }, + teamId: { + type: mongoose.Schema.Types.ObjectId, + ref: "Team", + required: true, + immutable: true, + index: true, + }, + startTime: { + type: Date, + required: true, + immutable: true, + }, + endTime: { + type: Date, + default: null, + }, + status: { + type: Boolean, + default: true, + index: true, + }, + message: { + type: String, + default: null, + }, + statusCode: { + type: Number, + index: true, + default: null, + }, + resolutionType: { + type: String, + enum: ["automatic", "manual"], + default: null, + }, + resolvedBy: { + type: mongoose.Schema.Types.ObjectId, + ref: "User", + default: null, + }, + comment: { + type: String, + default: null, + }, + checks: [ + { + type: mongoose.Schema.Types.ObjectId, + ref: "Check", + }, + ], + }, + { timestamps: true } +); IncidentSchema.index({ monitorId: 1, status: 1 }); IncidentSchema.index({ teamId: 1, status: 1 }); diff --git a/server/src/db/v1/modules/incidentModule.js b/server/src/db/v1/modules/incidentModule.js index 8cfa0f0b1..c007853d4 100644 --- a/server/src/db/v1/modules/incidentModule.js +++ b/server/src/db/v1/modules/incidentModule.js @@ -42,7 +42,7 @@ class IncidentModule { getActiveIncidentByMonitor = async (monitorId) => { try { - return await this.Incident.findOne({ monitorId: new ObjectId(monitorId), status: "active" }); + return await this.Incident.findOne({ monitorId: new ObjectId(monitorId), status: true }); } catch (error) { error.service = SERVICE_NAME; error.method = "getActiveIncidentByMonitor"; @@ -50,11 +50,51 @@ class IncidentModule { } }; + getActiveIncidentsByMonitors = async (monitorIds) => { + try { + if (!monitorIds || monitorIds.length === 0) { + return new Map(); + } + + const objectIds = monitorIds.map((id) => new ObjectId(id)); + const incidents = await this.Incident.find({ + monitorId: { $in: objectIds }, + status: true, + }); + + const map = new Map(); + incidents.forEach((incident) => { + const monitorIdStr = incident.monitorId.toString(); + map.set(monitorIdStr, incident); + }); + + return map; + } catch (error) { + error.service = SERVICE_NAME; + error.method = "getActiveIncidentsByMonitors"; + throw error; + } + }; + + getLastManuallyResolvedIncident = async (monitorId) => { + try { + return await this.Incident.findOne({ + monitorId: new ObjectId(monitorId), + status: false, + resolutionType: "manual", + }) + .sort({ endTime: -1 }) + .limit(1); + } catch (error) { + error.service = SERVICE_NAME; + error.method = "getLastManuallyResolvedIncident"; + throw error; + } + }; + getIncidentById = async (incidentId) => { try { - return await this.Incident.findById(incidentId) - .populate("monitorId", "name type url") - .populate("resolvedBy", "name email"); + return await this.Incident.findById(incidentId).populate("monitorId", "name type url").populate("resolvedBy", "name email"); } catch (error) { error.service = SERVICE_NAME; error.method = "getIncidentById"; @@ -65,17 +105,13 @@ class IncidentModule { resolveIncident = async (incidentId, { resolutionType = "automatic", resolvedBy = null, comment = null, endTime = new Date() } = {}) => { try { const update = { - status: "resolved", + status: false, endTime, resolutionType, resolvedBy, ...(comment !== null && { comment }), }; - const incident = await this.Incident.findOneAndUpdate( - { _id: new ObjectId(incidentId) }, - { $set: update }, - { new: true } - ); + const incident = await this.Incident.findOneAndUpdate({ _id: new ObjectId(incidentId) }, { $set: update }, { new: true }); if (!incident) { throw new Error("Incident not found"); @@ -109,14 +145,44 @@ class IncidentModule { } }; - getIncidentsByTeam = async ({ teamId, sortOrder, dateRange, filter, page, rowsPerPage, status, monitorId, resolutionType }) => { + addChecksToIncidentsBatch = async (operations) => { try { - page = parseInt(page); - rowsPerPage = parseInt(rowsPerPage); + if (!operations || operations.length === 0) { + return; + } + + const bulkOps = operations.map((op) => ({ + updateOne: { + filter: { _id: new ObjectId(op.incidentId) }, + update: { $addToSet: { checks: new ObjectId(op.checkId) } }, + }, + })); + + await this.Incident.bulkWrite(bulkOps, { ordered: false }); + } catch (error) { + error.service = SERVICE_NAME; + error.method = "addChecksToIncidentsBatch"; + throw error; + } + }; + + getIncidentsByTeam = async ({ teamId, sortOrder, dateRange, page, rowsPerPage, status, monitorId, resolutionType }) => { + try { + page = Number.isFinite(parseInt(page)) ? parseInt(page) : 0; + rowsPerPage = Number.isFinite(parseInt(rowsPerPage)) ? parseInt(rowsPerPage) : 20; + + let statusBoolean = undefined; + if (status !== undefined && status !== null) { + if (typeof status === "string") { + statusBoolean = status === "true"; + } else if (typeof status === "boolean") { + statusBoolean = status; + } + } const matchStage = { teamId: new ObjectId(teamId), - ...(status && { status }), + ...(statusBoolean !== undefined && { status: statusBoolean }), ...(monitorId && { monitorId: new ObjectId(monitorId) }), ...(resolutionType && { resolutionType }), ...(dateRangeLookup[dateRange] && { @@ -126,27 +192,9 @@ class IncidentModule { }), }; - if (filter !== undefined) { - switch (filter) { - case "all": - break; - case "active": - matchStage.status = "active"; - break; - case "resolved": - matchStage.status = "resolved"; - break; - default: - break; - } - } - sortOrder = sortOrder === "asc" ? 1 : -1; - let skip = 0; - if (page && rowsPerPage) { - skip = page * rowsPerPage; - } + const skip = page * rowsPerPage; const incidents = await this.Incident.aggregate([ { $match: matchStage }, @@ -212,10 +260,10 @@ class IncidentModule { summary.forEach((item) => { result.total += item.count; - if (item._id === "active") { + if (item._id === true) { result.active = item.count; } - if (item._id === "resolved") { + if (item._id === false) { result.resolved = item.count; } result.manual += item.manualResolutions; @@ -255,4 +303,3 @@ class IncidentModule { } export default IncidentModule; - diff --git a/server/src/routes/v1/incidentRoute.js b/server/src/routes/v1/incidentRoute.js new file mode 100644 index 000000000..6825ce21e --- /dev/null +++ b/server/src/routes/v1/incidentRoute.js @@ -0,0 +1,26 @@ +import { Router } from "express"; +import { isAllowed } from "../../middleware/v1/isAllowed.js"; + +class IncidentRoutes { + constructor(incidentController) { + this.router = Router(); + this.incidentController = incidentController; + this.initRoutes(); + } + + initRoutes() { + // Team routes + this.router.get("/team", this.incidentController.getIncidentsByTeam); + this.router.get("/team/summary", this.incidentController.getIncidentSummary); + + // Individual incident routes + this.router.get("/:incidentId", this.incidentController.getIncidentById); + this.router.put("/:incidentId/resolve", isAllowed(["admin", "superadmin"]), this.incidentController.resolveIncidentManually); + } + + getRouter() { + return this.router; + } +} + +export default IncidentRoutes; diff --git a/server/src/service/v1/business/incidentService.js b/server/src/service/v1/business/incidentService.js index e2108d1b3..a472dbd57 100644 --- a/server/src/service/v1/business/incidentService.js +++ b/server/src/service/v1/business/incidentService.js @@ -28,7 +28,7 @@ class IncidentService { if (activeIncident) { await this.db.incidentModule.addCheckToIncident(activeIncident._id, check._id); - + this.logger.info({ service: this.SERVICE_NAME, method: "createIncident", @@ -45,7 +45,7 @@ class IncidentService { teamId: monitor.teamId, type: monitor.type, startTime: new Date(), - status: "active", + status: true, message: check.message || null, statusCode: check.statusCode || null, checks: [check._id], @@ -121,7 +121,6 @@ class IncidentService { } }; - resolveIncidentManually = async ({ incidentId, userId, teamId, comment }) => { try { if (!incidentId) { @@ -146,7 +145,7 @@ class IncidentService { throw this.errorService.createAuthorizationError(); } - if (incident.status === "resolved") { + if (incident.status === false) { throw this.errorService.createBadRequestError("Incident is already resolved"); } @@ -178,20 +177,18 @@ class IncidentService { } }; - getIncidentsByTeam = async ({ teamId, query }) => { try { if (!teamId) { throw this.errorService.createBadRequestError("No team ID in request"); } - const { sortOrder, dateRange, filter, page, rowsPerPage, status, monitorId, resolutionType } = query || {}; + const { sortOrder, dateRange, page, rowsPerPage, status, monitorId, resolutionType } = query || {}; const result = await this.db.incidentModule.getIncidentsByTeam({ teamId, sortOrder, dateRange, - filter, page, rowsPerPage, status, @@ -212,7 +209,6 @@ class IncidentService { } }; - getIncidentSummary = async ({ teamId, query }) => { try { if (!teamId) { @@ -271,7 +267,148 @@ class IncidentService { throw error; } }; + + processIncidentsFromBuffer = async (incidentBufferItems) => { + try { + if (!incidentBufferItems || incidentBufferItems.length === 0) { + return; + } + + const createItems = []; + const resolveItems = []; + + for (const item of incidentBufferItems) { + if (item.action === "resolve") { + resolveItems.push(item); + } else { + createItems.push(item); + } + } + + for (const item of resolveItems) { + try { + await this.resolveIncident(item.monitor, item.check); + } catch (error) { + this.logger.error({ + service: this.SERVICE_NAME, + method: "processIncidentsFromBuffer", + message: `Failed to resolve incident from buffer: ${error.message}`, + monitorId: item.monitor?._id, + error: error.stack, + }); + } + } + + if (createItems.length === 0) { + return; + } + + const groupedByMonitor = {}; + for (const item of createItems) { + if (!item.monitor || !item.monitor._id || !item.check || !item.check._id) { + this.logger.warn({ + service: this.SERVICE_NAME, + method: "processIncidentsFromBuffer", + message: "Skipping item with missing monitor or check data", + item, + }); + continue; + } + + const monitorId = item.monitor._id.toString(); + if (!groupedByMonitor[monitorId]) { + groupedByMonitor[monitorId] = []; + } + groupedByMonitor[monitorId].push(item); + } + + const monitorIds = Object.keys(groupedByMonitor); + if (monitorIds.length === 0) { + return; + } + + const activeIncidents = await this.db.incidentModule.getActiveIncidentsByMonitors(monitorIds); + + const incidentsCreatedInFlush = {}; + const checksToAddToIncidents = []; + const newIncidentsToCreate = []; + + for (const [monitorId, items] of Object.entries(groupedByMonitor)) { + const existingIncident = activeIncidents.get(monitorId) || incidentsCreatedInFlush[monitorId]; + + if (existingIncident) { + const incidentId = existingIncident._id ? existingIncident._id.toString() : existingIncident; + for (const item of items) { + checksToAddToIncidents.push({ + incidentId, + checkId: item.check._id.toString(), + }); + } + } else { + const firstItem = items[0]; + const incidentData = { + monitorId: firstItem.monitor._id, + teamId: firstItem.monitor.teamId, + type: firstItem.monitor.type, + startTime: new Date(), + status: true, + message: firstItem.check.message || null, + statusCode: firstItem.check.statusCode || null, + checks: [firstItem.check._id], + }; + + newIncidentsToCreate.push({ + incidentData, + monitorId, + remainingChecks: items.slice(1), // Checks restantes para agregar después + }); + } + } + + if (newIncidentsToCreate.length > 0) { + const incidentDataArray = newIncidentsToCreate.map((item) => item.incidentData); + await this.db.incidentModule.createIncidents(incidentDataArray); + + const createdIncidentsMap = await this.db.incidentModule.getActiveIncidentsByMonitors(newIncidentsToCreate.map((item) => item.monitorId)); + + for (const item of newIncidentsToCreate) { + const createdIncident = createdIncidentsMap.get(item.monitorId); + if (createdIncident && createdIncident._id) { + const incidentId = createdIncident._id.toString(); + incidentsCreatedInFlush[item.monitorId] = incidentId; + + for (const remainingItem of item.remainingChecks) { + checksToAddToIncidents.push({ + incidentId, + checkId: remainingItem.check._id.toString(), + }); + } + } + } + } + + if (checksToAddToIncidents.length > 0) { + await this.db.incidentModule.addChecksToIncidentsBatch(checksToAddToIncidents); + } + + this.logger.info({ + service: this.SERVICE_NAME, + method: "processIncidentsFromBuffer", + message: `Processed ${incidentBufferItems.length} incident buffer items`, + created: newIncidentsToCreate.length, + checksAdded: checksToAddToIncidents.length, + resolved: resolveItems.length, + }); + } catch (error) { + this.logger.error({ + service: this.SERVICE_NAME, + method: "processIncidentsFromBuffer", + message: error.message, + error: error.stack, + }); + throw error; + } + }; } export default IncidentService; - diff --git a/server/src/service/v1/infrastructure/bufferService.js b/server/src/service/v1/infrastructure/bufferService.js index 459d25af3..9b05d05c8 100755 --- a/server/src/service/v1/infrastructure/bufferService.js +++ b/server/src/service/v1/infrastructure/bufferService.js @@ -2,12 +2,14 @@ const SERVICE_NAME = "BufferService"; class BufferService { static SERVICE_NAME = SERVICE_NAME; - constructor({ db, logger, envSettings }) { + constructor({ db, logger, envSettings, incidentService }) { this.BUFFER_TIMEOUT = envSettings.nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute this.db = db; this.logger = logger; + this.incidentService = incidentService; this.SERVICE_NAME = SERVICE_NAME; this.buffer = []; + this.incidentBuffer = []; this.scheduleNextFlush(); this.logger.info({ message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`, @@ -33,6 +35,66 @@ class BufferService { } } + addIncidentToBuffer({ monitor, check, action = "create" }) { + try { + if (!monitor || !check) { + this.logger.warn({ + message: "Skipping incident buffer item: missing monitor or check", + service: this.SERVICE_NAME, + method: "addIncidentToBuffer", + }); + return; + } + + this.incidentBuffer.push({ monitor, check, action }); + } catch (error) { + this.logger.error({ + message: error.message, + service: this.SERVICE_NAME, + method: "addIncidentToBuffer", + stack: error.stack, + }); + } + } + + removeCheckFromBuffer(checkToRemove) { + try { + if (!checkToRemove) { + return false; + } + + const index = this.buffer.findIndex((check) => { + if (checkToRemove._id && check._id) { + return check._id.toString() === checkToRemove._id.toString(); + } + return ( + check.monitorId?.toString() === checkToRemove.monitorId?.toString() && + check.teamId?.toString() === checkToRemove.teamId?.toString() && + check.type === checkToRemove.type && + check.status === checkToRemove.status && + check.statusCode === checkToRemove.statusCode && + check.responseTime === checkToRemove.responseTime && + check.message === checkToRemove.message + ); + }); + + if (index !== -1) { + this.buffer.splice(index, 1); + return true; + } + + return false; + } catch (error) { + this.logger.error({ + message: error.message, + service: this.SERVICE_NAME, + method: "removeCheckFromBuffer", + stack: error.stack, + }); + return false; + } + } + scheduleNextFlush() { this.bufferTimer = setTimeout(async () => { try { @@ -51,10 +113,10 @@ class BufferService { }, this.BUFFER_TIMEOUT); } async flushBuffer() { - let items = this.buffer.length; - try { - await this.db.checkModule.createChecks(this.buffer); + if (this.buffer.length > 0) { + await this.db.checkModule.createChecks(this.buffer); + } } catch (error) { this.logger.error({ message: error.message, @@ -63,13 +125,41 @@ class BufferService { stack: error.stack, }); } + + try { + if (this.incidentBuffer.length > 0 && this.incidentService) { + await this.flushIncidentBuffer(); + } + } catch (error) { + this.logger.error({ + message: error.message, + service: this.SERVICE_NAME, + method: "flushBuffer", + stack: error.stack, + }); + } + this.buffer = []; - // this.logger.debug({ - // message: `Flushed ${items} items`, - // service: this.SERVICE_NAME, - // method: "flushBuffer", - // }); - items = 0; + this.incidentBuffer = []; + } + + async flushIncidentBuffer() { + if (!this.incidentService || this.incidentBuffer.length === 0) { + return; + } + + try { + const itemsToProcess = [...this.incidentBuffer]; + await this.incidentService.processIncidentsFromBuffer(itemsToProcess); + } catch (error) { + this.logger.error({ + message: `Error flushing incident buffer: ${error.message}`, + service: this.SERVICE_NAME, + method: "flushIncidentBuffer", + stack: error.stack, + }); + throw error; + } } } diff --git a/server/src/service/v1/infrastructure/statusService.js b/server/src/service/v1/infrastructure/statusService.js index e03c7d529..829b6d850 100755 --- a/server/src/service/v1/infrastructure/statusService.js +++ b/server/src/service/v1/infrastructure/statusService.js @@ -98,6 +98,63 @@ class StatusService { if (status === false) return "down"; return "unknown"; }; + + /** + * Saves check if needed and adds to incident buffer + * Removes check from checks buffer if it was saved immediately + * + * @param {Object} check - The check object + * @param {Object} monitor - The monitor object + * @param {string} action - The incident action ("create" or "resolve") + * @param {string} errorContext - Context for error messages + * @returns {Promise} + */ + handleIncidentForCheck = async (check, monitor, action, errorContext = "incident handling") => { + try { + let savedCheck = check; + + if (!check._id) { + try { + const checkModel = new Check(check); + savedCheck = await checkModel.save(); + + this.buffer.removeCheckFromBuffer(check); + } catch (checkError) { + this.logger.error({ + service: this.SERVICE_NAME, + method: "handleIncidentForCheck", + message: `Failed to save check immediately for ${errorContext}: ${checkError.message}`, + monitorId: monitor._id, + stack: checkError.stack, + }); + savedCheck = null; + } + } + + if (savedCheck && savedCheck._id) { + try { + this.buffer.addIncidentToBuffer({ monitor, check: savedCheck, action }); + } catch (incidentError) { + this.logger.error({ + service: this.SERVICE_NAME, + method: "handleIncidentForCheck", + message: `Failed to add incident to buffer for ${errorContext}: ${incidentError.message}`, + monitorId: monitor._id, + action, + stack: incidentError.stack, + }); + } + } + } catch (error) { + this.logger.error({ + service: this.SERVICE_NAME, + method: "handleIncidentForCheck", + message: `Error in ${errorContext}: ${error.message}`, + monitorId: monitor?._id, + stack: error.stack, + }); + } + }; /** * Updates the status of a monitor based on the network response. * @@ -174,38 +231,58 @@ class StatusService { newStatus, }); - let savedCheck = check; - if (!check._id) { - try { - const checkModel = new Check(check); - savedCheck = await checkModel.save(); - } catch (checkError) { - this.logger.error({ - service: this.SERVICE_NAME, - method: "updateStatus", - message: `Failed to save check immediately: ${checkError.message}`, - monitorId: monitor._id, - stack: checkError.stack, - }); - } + if (newStatus === false) { + await this.handleIncidentForCheck(check, monitor, "create", "status change to down"); + } else if (prevStatus === false) { + await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up"); } + } + if (monitor.status === false && !statusChanged) { try { - if (newStatus === false) { - await this.incidentService.createIncident(monitor, savedCheck); - } else if (prevStatus === false) { - await this.incidentService.resolveIncident(monitor, savedCheck); + const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor._id); + + let calculatedFailureRate = failureRate; + + if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) { + try { + const checksAfterResolution = await Check.find({ + monitorId: monitor._id, + createdAt: { $gt: lastManuallyResolvedIncident.endTime }, + }) + .sort({ createdAt: 1 }) + .limit(monitor.statusWindowSize) + .select("status") + .lean(); + + if (checksAfterResolution.length > 0) { + const checksStatuses = checksAfterResolution.map((c) => c.status); + const failuresAfterResolution = checksStatuses.filter((s) => s === false).length; + calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100; + } else { + calculatedFailureRate = 0; + } + } catch (checkQueryError) { + this.logger.error({ + service: this.SERVICE_NAME, + method: "updateStatus", + message: `Failed to query checks after manual resolution: ${checkQueryError.message}`, + monitorId: monitor._id, + stack: checkQueryError.stack, + }); + } } - } catch (incidentError) { + + if (calculatedFailureRate >= monitor.statusWindowThreshold) { + await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change"); + } + } catch (error) { this.logger.error({ service: this.SERVICE_NAME, method: "updateStatus", - message: `Failed to handle incident: ${incidentError.message}`, + message: `Error handling threshold check without status change: ${error.message}`, monitorId: monitor._id, - statusChanged, - prevStatus, - newStatus, - stack: incidentError.stack, + stack: error.stack, }); } }