Incidents management implementation
This commit is contained in:
1 parent
eb65c41d15
commit
131ced10fc
10 files changed
+713
-139
No files matched your search
@@ -28,7 +28,7 @@ class IncidentService {
|
||||
|
||||
if (activeIncident) {
|
||||
await this.db.incidentModule.addCheckToIncident(activeIncident._id, check._id);
|
||||
|
||||
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "createIncident",
|
||||
@@ -45,7 +45,7 @@ class IncidentService {
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
startTime: new Date(),
|
||||
status: "active",
|
||||
status: true,
|
||||
message: check.message || null,
|
||||
statusCode: check.statusCode || null,
|
||||
checks: [check._id],
|
||||
@@ -121,7 +121,6 @@ class IncidentService {
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
resolveIncidentManually = async ({ incidentId, userId, teamId, comment }) => {
|
||||
try {
|
||||
if (!incidentId) {
|
||||
@@ -146,7 +145,7 @@ class IncidentService {
|
||||
throw this.errorService.createAuthorizationError();
|
||||
}
|
||||
|
||||
if (incident.status === "resolved") {
|
||||
if (incident.status === false) {
|
||||
throw this.errorService.createBadRequestError("Incident is already resolved");
|
||||
}
|
||||
|
||||
@@ -178,20 +177,18 @@ class IncidentService {
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
getIncidentsByTeam = async ({ teamId, query }) => {
|
||||
try {
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError("No team ID in request");
|
||||
}
|
||||
|
||||
const { sortOrder, dateRange, filter, page, rowsPerPage, status, monitorId, resolutionType } = query || {};
|
||||
const { sortOrder, dateRange, page, rowsPerPage, status, monitorId, resolutionType } = query || {};
|
||||
|
||||
const result = await this.db.incidentModule.getIncidentsByTeam({
|
||||
teamId,
|
||||
sortOrder,
|
||||
dateRange,
|
||||
filter,
|
||||
page,
|
||||
rowsPerPage,
|
||||
status,
|
||||
@@ -212,7 +209,6 @@ class IncidentService {
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
getIncidentSummary = async ({ teamId, query }) => {
|
||||
try {
|
||||
if (!teamId) {
|
||||
@@ -271,7 +267,148 @@ class IncidentService {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
processIncidentsFromBuffer = async (incidentBufferItems) => {
|
||||
try {
|
||||
if (!incidentBufferItems || incidentBufferItems.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const createItems = [];
|
||||
const resolveItems = [];
|
||||
|
||||
for (const item of incidentBufferItems) {
|
||||
if (item.action === "resolve") {
|
||||
resolveItems.push(item);
|
||||
} else {
|
||||
createItems.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
for (const item of resolveItems) {
|
||||
try {
|
||||
await this.resolveIncident(item.monitor, item.check);
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: `Failed to resolve incident from buffer: ${error.message}`,
|
||||
monitorId: item.monitor?._id,
|
||||
error: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (createItems.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const groupedByMonitor = {};
|
||||
for (const item of createItems) {
|
||||
if (!item.monitor || !item.monitor._id || !item.check || !item.check._id) {
|
||||
this.logger.warn({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: "Skipping item with missing monitor or check data",
|
||||
item,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const monitorId = item.monitor._id.toString();
|
||||
if (!groupedByMonitor[monitorId]) {
|
||||
groupedByMonitor[monitorId] = [];
|
||||
}
|
||||
groupedByMonitor[monitorId].push(item);
|
||||
}
|
||||
|
||||
const monitorIds = Object.keys(groupedByMonitor);
|
||||
if (monitorIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeIncidents = await this.db.incidentModule.getActiveIncidentsByMonitors(monitorIds);
|
||||
|
||||
const incidentsCreatedInFlush = {};
|
||||
const checksToAddToIncidents = [];
|
||||
const newIncidentsToCreate = [];
|
||||
|
||||
for (const [monitorId, items] of Object.entries(groupedByMonitor)) {
|
||||
const existingIncident = activeIncidents.get(monitorId) || incidentsCreatedInFlush[monitorId];
|
||||
|
||||
if (existingIncident) {
|
||||
const incidentId = existingIncident._id ? existingIncident._id.toString() : existingIncident;
|
||||
for (const item of items) {
|
||||
checksToAddToIncidents.push({
|
||||
incidentId,
|
||||
checkId: item.check._id.toString(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const firstItem = items[0];
|
||||
const incidentData = {
|
||||
monitorId: firstItem.monitor._id,
|
||||
teamId: firstItem.monitor.teamId,
|
||||
type: firstItem.monitor.type,
|
||||
startTime: new Date(),
|
||||
status: true,
|
||||
message: firstItem.check.message || null,
|
||||
statusCode: firstItem.check.statusCode || null,
|
||||
checks: [firstItem.check._id],
|
||||
};
|
||||
|
||||
newIncidentsToCreate.push({
|
||||
incidentData,
|
||||
monitorId,
|
||||
remainingChecks: items.slice(1), // Checks restantes para agregar después
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (newIncidentsToCreate.length > 0) {
|
||||
const incidentDataArray = newIncidentsToCreate.map((item) => item.incidentData);
|
||||
await this.db.incidentModule.createIncidents(incidentDataArray);
|
||||
|
||||
const createdIncidentsMap = await this.db.incidentModule.getActiveIncidentsByMonitors(newIncidentsToCreate.map((item) => item.monitorId));
|
||||
|
||||
for (const item of newIncidentsToCreate) {
|
||||
const createdIncident = createdIncidentsMap.get(item.monitorId);
|
||||
if (createdIncident && createdIncident._id) {
|
||||
const incidentId = createdIncident._id.toString();
|
||||
incidentsCreatedInFlush[item.monitorId] = incidentId;
|
||||
|
||||
for (const remainingItem of item.remainingChecks) {
|
||||
checksToAddToIncidents.push({
|
||||
incidentId,
|
||||
checkId: remainingItem.check._id.toString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (checksToAddToIncidents.length > 0) {
|
||||
await this.db.incidentModule.addChecksToIncidentsBatch(checksToAddToIncidents);
|
||||
}
|
||||
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: `Processed ${incidentBufferItems.length} incident buffer items`,
|
||||
created: newIncidentsToCreate.length,
|
||||
checksAdded: checksToAddToIncidents.length,
|
||||
resolved: resolveItems.length,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: error.message,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default IncidentService;
|
||||
|
||||
@@ -2,12 +2,14 @@ const SERVICE_NAME = "BufferService";
|
||||
|
||||
class BufferService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
constructor({ db, logger, envSettings }) {
|
||||
constructor({ db, logger, envSettings, incidentService }) {
|
||||
this.BUFFER_TIMEOUT = envSettings.nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.incidentService = incidentService;
|
||||
this.SERVICE_NAME = SERVICE_NAME;
|
||||
this.buffer = [];
|
||||
this.incidentBuffer = [];
|
||||
this.scheduleNextFlush();
|
||||
this.logger.info({
|
||||
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
|
||||
@@ -33,6 +35,66 @@ class BufferService {
|
||||
}
|
||||
}
|
||||
|
||||
addIncidentToBuffer({ monitor, check, action = "create" }) {
|
||||
try {
|
||||
if (!monitor || !check) {
|
||||
this.logger.warn({
|
||||
message: "Skipping incident buffer item: missing monitor or check",
|
||||
service: this.SERVICE_NAME,
|
||||
method: "addIncidentToBuffer",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
this.incidentBuffer.push({ monitor, check, action });
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "addIncidentToBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
removeCheckFromBuffer(checkToRemove) {
|
||||
try {
|
||||
if (!checkToRemove) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const index = this.buffer.findIndex((check) => {
|
||||
if (checkToRemove._id && check._id) {
|
||||
return check._id.toString() === checkToRemove._id.toString();
|
||||
}
|
||||
return (
|
||||
check.monitorId?.toString() === checkToRemove.monitorId?.toString() &&
|
||||
check.teamId?.toString() === checkToRemove.teamId?.toString() &&
|
||||
check.type === checkToRemove.type &&
|
||||
check.status === checkToRemove.status &&
|
||||
check.statusCode === checkToRemove.statusCode &&
|
||||
check.responseTime === checkToRemove.responseTime &&
|
||||
check.message === checkToRemove.message
|
||||
);
|
||||
});
|
||||
|
||||
if (index !== -1) {
|
||||
this.buffer.splice(index, 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "removeCheckFromBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
scheduleNextFlush() {
|
||||
this.bufferTimer = setTimeout(async () => {
|
||||
try {
|
||||
@@ -51,10 +113,10 @@ class BufferService {
|
||||
}, this.BUFFER_TIMEOUT);
|
||||
}
|
||||
async flushBuffer() {
|
||||
let items = this.buffer.length;
|
||||
|
||||
try {
|
||||
await this.db.checkModule.createChecks(this.buffer);
|
||||
if (this.buffer.length > 0) {
|
||||
await this.db.checkModule.createChecks(this.buffer);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
@@ -63,13 +125,41 @@ class BufferService {
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.incidentBuffer.length > 0 && this.incidentService) {
|
||||
await this.flushIncidentBuffer();
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
|
||||
this.buffer = [];
|
||||
// this.logger.debug({
|
||||
// message: `Flushed ${items} items`,
|
||||
// service: this.SERVICE_NAME,
|
||||
// method: "flushBuffer",
|
||||
// });
|
||||
items = 0;
|
||||
this.incidentBuffer = [];
|
||||
}
|
||||
|
||||
async flushIncidentBuffer() {
|
||||
if (!this.incidentService || this.incidentBuffer.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const itemsToProcess = [...this.incidentBuffer];
|
||||
await this.incidentService.processIncidentsFromBuffer(itemsToProcess);
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: `Error flushing incident buffer: ${error.message}`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushIncidentBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +98,63 @@ class StatusService {
|
||||
if (status === false) return "down";
|
||||
return "unknown";
|
||||
};
|
||||
|
||||
/**
|
||||
* Saves check if needed and adds to incident buffer
|
||||
* Removes check from checks buffer if it was saved immediately
|
||||
*
|
||||
* @param {Object} check - The check object
|
||||
* @param {Object} monitor - The monitor object
|
||||
* @param {string} action - The incident action ("create" or "resolve")
|
||||
* @param {string} errorContext - Context for error messages
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
handleIncidentForCheck = async (check, monitor, action, errorContext = "incident handling") => {
|
||||
try {
|
||||
let savedCheck = check;
|
||||
|
||||
if (!check._id) {
|
||||
try {
|
||||
const checkModel = new Check(check);
|
||||
savedCheck = await checkModel.save();
|
||||
|
||||
this.buffer.removeCheckFromBuffer(check);
|
||||
} catch (checkError) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "handleIncidentForCheck",
|
||||
message: `Failed to save check immediately for ${errorContext}: ${checkError.message}`,
|
||||
monitorId: monitor._id,
|
||||
stack: checkError.stack,
|
||||
});
|
||||
savedCheck = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (savedCheck && savedCheck._id) {
|
||||
try {
|
||||
this.buffer.addIncidentToBuffer({ monitor, check: savedCheck, action });
|
||||
} catch (incidentError) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "handleIncidentForCheck",
|
||||
message: `Failed to add incident to buffer for ${errorContext}: ${incidentError.message}`,
|
||||
monitorId: monitor._id,
|
||||
action,
|
||||
stack: incidentError.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "handleIncidentForCheck",
|
||||
message: `Error in ${errorContext}: ${error.message}`,
|
||||
monitorId: monitor?._id,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Updates the status of a monitor based on the network response.
|
||||
*
|
||||
@@ -174,38 +231,58 @@ class StatusService {
|
||||
newStatus,
|
||||
});
|
||||
|
||||
let savedCheck = check;
|
||||
if (!check._id) {
|
||||
try {
|
||||
const checkModel = new Check(check);
|
||||
savedCheck = await checkModel.save();
|
||||
} catch (checkError) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "updateStatus",
|
||||
message: `Failed to save check immediately: ${checkError.message}`,
|
||||
monitorId: monitor._id,
|
||||
stack: checkError.stack,
|
||||
});
|
||||
}
|
||||
if (newStatus === false) {
|
||||
await this.handleIncidentForCheck(check, monitor, "create", "status change to down");
|
||||
} else if (prevStatus === false) {
|
||||
await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up");
|
||||
}
|
||||
}
|
||||
|
||||
if (monitor.status === false && !statusChanged) {
|
||||
try {
|
||||
if (newStatus === false) {
|
||||
await this.incidentService.createIncident(monitor, savedCheck);
|
||||
} else if (prevStatus === false) {
|
||||
await this.incidentService.resolveIncident(monitor, savedCheck);
|
||||
const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor._id);
|
||||
|
||||
let calculatedFailureRate = failureRate;
|
||||
|
||||
if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) {
|
||||
try {
|
||||
const checksAfterResolution = await Check.find({
|
||||
monitorId: monitor._id,
|
||||
createdAt: { $gt: lastManuallyResolvedIncident.endTime },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.limit(monitor.statusWindowSize)
|
||||
.select("status")
|
||||
.lean();
|
||||
|
||||
if (checksAfterResolution.length > 0) {
|
||||
const checksStatuses = checksAfterResolution.map((c) => c.status);
|
||||
const failuresAfterResolution = checksStatuses.filter((s) => s === false).length;
|
||||
calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100;
|
||||
} else {
|
||||
calculatedFailureRate = 0;
|
||||
}
|
||||
} catch (checkQueryError) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "updateStatus",
|
||||
message: `Failed to query checks after manual resolution: ${checkQueryError.message}`,
|
||||
monitorId: monitor._id,
|
||||
stack: checkQueryError.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (incidentError) {
|
||||
|
||||
if (calculatedFailureRate >= monitor.statusWindowThreshold) {
|
||||
await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change");
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "updateStatus",
|
||||
message: `Failed to handle incident: ${incidentError.message}`,
|
||||
message: `Error handling threshold check without status change: ${error.message}`,
|
||||
monitorId: monitor._id,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
newStatus,
|
||||
stack: incidentError.stack,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user