Merge pull request #3153 from bluewave-labs/fix/incidents
fix/incidents
This commit is contained in:
18 files changed
+646
-574
No files matched your search
@@ -15,6 +15,7 @@ import { useTranslation } from "react-i18next";
|
||||
import { Button, Typography, useTheme } from "@mui/material";
|
||||
|
||||
const IncidentTable = ({
|
||||
monitors = [],
|
||||
incidents = [],
|
||||
incidentsCount = 0,
|
||||
isLoading = false,
|
||||
@@ -44,7 +45,11 @@ const IncidentTable = ({
|
||||
{
|
||||
id: "monitorName",
|
||||
content: t("incidentsTableMonitorName"),
|
||||
render: (row) => row.monitorName ?? "N/A",
|
||||
render: (row) => {
|
||||
console.log(monitors, row);
|
||||
const monitor = monitors.find((monitor) => monitor.id === row.monitorId);
|
||||
return monitor ? monitor.name : "N/A";
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "status",
|
||||
@@ -130,7 +135,7 @@ const IncidentTable = ({
|
||||
lineHeight: 1.2,
|
||||
}}
|
||||
onClick={() => {
|
||||
handleResolveIncident(row._id);
|
||||
handleResolveIncident(row.id);
|
||||
}}
|
||||
>
|
||||
{t("incidentsPage.incidentsTableActionResolveManually")}
|
||||
|
||||
@@ -45,9 +45,8 @@ const useFetchIncidents = () => {
|
||||
setNetworkError(false);
|
||||
|
||||
const res = await networkService.getIncidentsByTeam(config);
|
||||
|
||||
setIncidents(res.data?.data?.incidents || []);
|
||||
setIncidentsCount(res.data?.data?.incidentsCount || 0);
|
||||
setIncidentsCount(res.data?.data?.count || 0);
|
||||
} catch (error) {
|
||||
setNetworkError(true);
|
||||
console.error(t("incidentsPage.errorFetchingIncidents"), error);
|
||||
|
||||
@@ -88,7 +88,7 @@ const Incidents2 = () => {
|
||||
useEffect(() => {
|
||||
const lookup = monitors?.reduce((acc, monitor) => {
|
||||
acc[monitor.id] = {
|
||||
_id: monitor.id,
|
||||
id: monitor.id,
|
||||
name: monitor.name,
|
||||
type: monitor.type,
|
||||
};
|
||||
@@ -132,6 +132,7 @@ const Incidents2 = () => {
|
||||
/>
|
||||
|
||||
<IncidentTable
|
||||
monitors={monitors || []}
|
||||
incidents={incidents || []}
|
||||
incidentsCount={incidentsCount || 0}
|
||||
isLoading={isLoadingIncidents}
|
||||
|
||||
@@ -84,6 +84,7 @@ import {
|
||||
MongoRecoveryTokensRepository,
|
||||
MongoSettingsRepository,
|
||||
MongoNotificationsRepository,
|
||||
MongoIncidentRepository,
|
||||
IMonitorsRepository,
|
||||
IChecksRepository,
|
||||
IMonitorStatsRepository,
|
||||
@@ -93,6 +94,7 @@ import {
|
||||
IRecoveryTokensRepository,
|
||||
ISettingsRepository,
|
||||
INotificationsRepository,
|
||||
IIncidentsRepository,
|
||||
} from "@/repositories/index.js";
|
||||
|
||||
export type InitializedSerivces = {
|
||||
@@ -127,6 +129,7 @@ export type InitializedSerivces = {
|
||||
recoveryTokensRepository: IRecoveryTokensRepository;
|
||||
settingsRepository: ISettingsRepository;
|
||||
notificationsRepository: INotificationsRepository;
|
||||
incidentsRepository: IIncidentsRepository;
|
||||
};
|
||||
|
||||
export const initializeServices = async ({
|
||||
@@ -183,6 +186,7 @@ export const initializeServices = async ({
|
||||
const recoveryTokensRepository = new MongoRecoveryTokensRepository();
|
||||
const settingsRepository = new MongoSettingsRepository();
|
||||
const notificationsRepository = new MongoNotificationsRepository();
|
||||
const incidentsRepository = new MongoIncidentRepository();
|
||||
const networkService = new NetworkService({
|
||||
axios,
|
||||
got,
|
||||
@@ -205,6 +209,7 @@ export const initializeServices = async ({
|
||||
logger,
|
||||
errorService,
|
||||
stringService,
|
||||
incidentsRepository,
|
||||
});
|
||||
|
||||
const checkService = new CheckService({
|
||||
@@ -217,7 +222,7 @@ export const initializeServices = async ({
|
||||
checksRepository,
|
||||
});
|
||||
|
||||
const bufferService = new BufferService({ logger, incidentService, checkService });
|
||||
const bufferService = new BufferService({ logger, checkService });
|
||||
|
||||
const statusService = new StatusService({ db, logger, buffer: bufferService, monitorsRepository });
|
||||
|
||||
@@ -247,6 +252,7 @@ export const initializeServices = async ({
|
||||
notificationsService,
|
||||
checkService,
|
||||
buffer: bufferService,
|
||||
incidentService,
|
||||
});
|
||||
|
||||
const superSimpleQueue = await SuperSimpleQueue.create({
|
||||
@@ -332,6 +338,7 @@ export const initializeServices = async ({
|
||||
recoveryTokensRepository,
|
||||
settingsRepository,
|
||||
notificationsRepository,
|
||||
incidentsRepository,
|
||||
};
|
||||
|
||||
Object.values(services).forEach((service) => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { Request, Response, NextFunction } from "express";
|
||||
|
||||
const SERVICE_NAME = "incidentController";
|
||||
@@ -33,10 +34,15 @@ class IncidentController {
|
||||
|
||||
getIncidentSummary = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const summary = await this.incidentService.getIncidentSummary({
|
||||
teamId: req?.user?.teamId,
|
||||
query: req?.query,
|
||||
});
|
||||
const teamId = req.user?.teamId;
|
||||
|
||||
if (!teamId) {
|
||||
throw new AppError({ message: "Team ID is required", service: SERVICE_NAME, status: 400 });
|
||||
}
|
||||
|
||||
const limit = req.query.limit ? parseInt(req.query.limit as string, 10) : 10;
|
||||
|
||||
const summary = await this.incidentService.getIncidentSummary(req?.user?.teamId, limit);
|
||||
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
@@ -67,12 +73,12 @@ class IncidentController {
|
||||
|
||||
resolveIncidentManually = async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const resolvedIncident = await this.incidentService.resolveIncidentManually({
|
||||
incidentId: req?.params?.incidentId,
|
||||
userId: req?.user?.id,
|
||||
teamId: req?.user?.teamId,
|
||||
comment: req?.body?.comment,
|
||||
});
|
||||
const resolvedIncident = await this.incidentService.resolveIncident(
|
||||
req?.params?.incidentId,
|
||||
req?.user?.id,
|
||||
req?.user?.teamId,
|
||||
req?.body?.comment
|
||||
);
|
||||
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
|
||||
@@ -1,16 +1,31 @@
|
||||
import mongoose from "mongoose";
|
||||
import { Schema, model, type Types } from "mongoose";
|
||||
import { IncidentResolutionTypes, type Incident } from "@/types/incident.js";
|
||||
|
||||
const IncidentSchema = mongoose.Schema(
|
||||
type IncidentDocumentBase = Omit<Incident, "id" | "monitorId" | "teamId" | "resolvedBy" | "startTime" | "endTime" | "createdAt" | "updatedAt"> & {
|
||||
monitorId: Types.ObjectId;
|
||||
teamId: Types.ObjectId;
|
||||
resolvedBy?: Types.ObjectId | null;
|
||||
startTime: Date;
|
||||
endTime: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
};
|
||||
|
||||
export interface IncidentDocument extends IncidentDocumentBase {
|
||||
_id: Types.ObjectId;
|
||||
}
|
||||
|
||||
const IncidentSchema = new Schema<IncidentDocument>(
|
||||
{
|
||||
monitorId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: "Monitor",
|
||||
required: true,
|
||||
immutable: true,
|
||||
index: true,
|
||||
},
|
||||
teamId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: "Team",
|
||||
required: true,
|
||||
immutable: true,
|
||||
@@ -18,8 +33,8 @@ const IncidentSchema = mongoose.Schema(
|
||||
},
|
||||
startTime: {
|
||||
type: Date,
|
||||
required: true,
|
||||
immutable: true,
|
||||
required: true,
|
||||
},
|
||||
endTime: {
|
||||
type: Date,
|
||||
@@ -36,16 +51,16 @@ const IncidentSchema = mongoose.Schema(
|
||||
},
|
||||
statusCode: {
|
||||
type: Number,
|
||||
index: true,
|
||||
default: null,
|
||||
index: true,
|
||||
},
|
||||
resolutionType: {
|
||||
type: String,
|
||||
enum: ["automatic", "manual"],
|
||||
enum: IncidentResolutionTypes,
|
||||
default: null,
|
||||
},
|
||||
resolvedBy: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: "User",
|
||||
default: null,
|
||||
},
|
||||
@@ -53,12 +68,6 @@ const IncidentSchema = mongoose.Schema(
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
checks: [
|
||||
{
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: "Check",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
@@ -71,6 +80,7 @@ IncidentSchema.index({ resolutionType: 1, status: 1 });
|
||||
IncidentSchema.index({ resolvedBy: 1, status: 1 });
|
||||
IncidentSchema.index({ createdAt: -1 });
|
||||
|
||||
const Incident = mongoose.model("Incident", IncidentSchema);
|
||||
const IncidentModel = model<IncidentDocument>("Incident", IncidentSchema);
|
||||
|
||||
export default Incident;
|
||||
export { IncidentModel };
|
||||
export default IncidentModel;
|
||||
@@ -24,3 +24,6 @@ export { default as RecoveryTokenModel } from "@/db/models/RecoveryToken.js";
|
||||
|
||||
export * from "@/db/models/Notification.js";
|
||||
export { default as NotificationModel } from "@/db/models/Notification.js";
|
||||
|
||||
export * from "@/db/models/Incident.js";
|
||||
export { default as IncidentModel } from "@/db/models/Incident.js";
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { Incident } from "@/types/index.js";
|
||||
import type { IncidentSummary } from "@/types/index.js";
|
||||
export interface IIncidentsRepository {
|
||||
// create
|
||||
create(incident: Partial<Incident>): Promise<Incident>;
|
||||
// fetch
|
||||
findActiveByIncidentId(incidentId: string, teamId: string): Promise<Incident | null>;
|
||||
findActiveByMonitorId(monitorId: string, teamId: string): Promise<Incident | null>;
|
||||
findByTeamId(
|
||||
teamId: string,
|
||||
startDate: Date | undefined,
|
||||
page: number,
|
||||
rowsPerPage: number,
|
||||
sortOrder?: string,
|
||||
status?: boolean,
|
||||
monitorId?: string,
|
||||
resolutionType?: string
|
||||
): Promise<Incident[]>;
|
||||
findSummaryByTeamId(teamId: string, limit?: number): Promise<IncidentSummary>;
|
||||
countByTeamId(teamId: string, startDate: Date | undefined, status?: boolean, monitorId?: string, resolutionType?: string): Promise<number>;
|
||||
|
||||
// update
|
||||
updateById(incidentId: string, teamId: string, updateData: Partial<Incident>): Promise<Incident>;
|
||||
// delete
|
||||
// other
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
import { IncidentModel } from "@/db/models/index.js";
|
||||
import type { IncidentDocument } from "@/db/models/Incident.js";
|
||||
import type { Incident, IncidentSummary } from "@/types/index.js";
|
||||
import type { IIncidentsRepository } from "@/repositories/index.js";
|
||||
import mongoose from "mongoose";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
|
||||
class MongoIncidentRepository implements IIncidentsRepository {
|
||||
private toStringId = (value?: mongoose.Types.ObjectId | string | null): string => {
|
||||
if (!value) {
|
||||
return "";
|
||||
}
|
||||
return value instanceof mongoose.Types.ObjectId ? value.toString() : String(value);
|
||||
};
|
||||
|
||||
private toDateString = (value?: Date | string | null): string => {
|
||||
if (!value) {
|
||||
return new Date(0).toISOString();
|
||||
}
|
||||
return value instanceof Date ? value.toISOString() : new Date(value).toISOString();
|
||||
};
|
||||
|
||||
private buildMatchStage({
|
||||
teamId,
|
||||
startDate,
|
||||
status,
|
||||
monitorId,
|
||||
resolutionType,
|
||||
}: {
|
||||
teamId: string;
|
||||
startDate: Date | undefined;
|
||||
status?: boolean;
|
||||
monitorId?: string;
|
||||
resolutionType?: string;
|
||||
}): Record<string, any> {
|
||||
const matchStage: Record<string, any> = {
|
||||
teamId: new mongoose.Types.ObjectId(teamId),
|
||||
...(status !== undefined && { status }),
|
||||
...(monitorId && { monitorId: new mongoose.Types.ObjectId(monitorId) }),
|
||||
...(resolutionType && { resolutionType }),
|
||||
};
|
||||
|
||||
if (startDate) {
|
||||
matchStage.createdAt = { $gte: startDate };
|
||||
}
|
||||
return matchStage;
|
||||
}
|
||||
|
||||
protected toEntity = (doc: IncidentDocument): Incident => {
|
||||
return {
|
||||
id: this.toStringId(doc._id),
|
||||
monitorId: this.toStringId(doc.monitorId),
|
||||
teamId: this.toStringId(doc.teamId),
|
||||
startTime: this.toDateString(doc.startTime),
|
||||
endTime: doc.endTime ? this.toDateString(doc.endTime) : null,
|
||||
status: doc.status,
|
||||
message: doc.message ?? null,
|
||||
statusCode: doc.statusCode ?? null,
|
||||
resolutionType: doc.resolutionType ?? null,
|
||||
resolvedBy: doc.resolvedBy ? this.toStringId(doc.resolvedBy) : null,
|
||||
comment: doc.comment ?? null,
|
||||
createdAt: this.toDateString(doc.createdAt),
|
||||
updatedAt: this.toDateString(doc.updatedAt),
|
||||
};
|
||||
};
|
||||
|
||||
protected mapDocuments = (documents: IncidentDocument[] | IncidentDocument | null): Incident[] => {
|
||||
if (!documents) {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(documents)) {
|
||||
return documents.map((doc) => this.toEntity(doc));
|
||||
}
|
||||
return [this.toEntity(documents)];
|
||||
};
|
||||
|
||||
async create(incident: Partial<Incident>): Promise<Incident> {
|
||||
const newIncident = await IncidentModel.create(incident);
|
||||
return this.toEntity(newIncident);
|
||||
}
|
||||
|
||||
findActiveByIncidentId = async (incidentId: string, teamId: string): Promise<Incident | null> => {
|
||||
const incident = await IncidentModel.findOne({
|
||||
_id: new mongoose.Types.ObjectId(incidentId),
|
||||
teamId: new mongoose.Types.ObjectId(teamId),
|
||||
status: true,
|
||||
});
|
||||
if (!incident) {
|
||||
return null;
|
||||
}
|
||||
return this.toEntity(incident);
|
||||
};
|
||||
|
||||
findActiveByMonitorId = async (monitorId: string, teamId: string): Promise<Incident | null> => {
|
||||
const incident = await IncidentModel.findOne({
|
||||
monitorId: new mongoose.Types.ObjectId(monitorId),
|
||||
teamId: new mongoose.Types.ObjectId(teamId),
|
||||
status: true,
|
||||
});
|
||||
if (!incident) {
|
||||
return null;
|
||||
}
|
||||
return this.toEntity(incident);
|
||||
};
|
||||
|
||||
findByTeamId = async (
|
||||
teamId: string,
|
||||
startDate: Date | undefined,
|
||||
page: number,
|
||||
rowsPerPage: number,
|
||||
sortOrder?: string,
|
||||
status?: boolean,
|
||||
monitorId?: string,
|
||||
resolutionType?: string
|
||||
): Promise<Incident[]> => {
|
||||
const matchStage = this.buildMatchStage({ teamId, startDate, status, monitorId, resolutionType });
|
||||
const incidents = await IncidentModel.find(matchStage)
|
||||
.sort({ createdAt: sortOrder === "asc" ? 1 : -1 })
|
||||
.skip(page * rowsPerPage)
|
||||
.limit(rowsPerPage);
|
||||
return this.mapDocuments(incidents);
|
||||
};
|
||||
|
||||
updateById = async (incidentId: string, teamId: string, patch: Partial<Incident>) => {
|
||||
const updatedIncident = await IncidentModel.findOneAndUpdate(
|
||||
{ _id: new mongoose.Types.ObjectId(incidentId), teamId: new mongoose.Types.ObjectId(teamId) },
|
||||
{
|
||||
$set: {
|
||||
...patch,
|
||||
},
|
||||
},
|
||||
{ new: true, runValidators: true }
|
||||
);
|
||||
if (!updatedIncident) {
|
||||
throw new AppError({ message: `Failed to update incident with id ${incidentId}`, status: 500 });
|
||||
}
|
||||
return this.toEntity(updatedIncident);
|
||||
};
|
||||
|
||||
countByTeamId = async (
|
||||
teamId: string,
|
||||
startDate: Date | undefined,
|
||||
status?: boolean,
|
||||
monitorId?: string,
|
||||
resolutionType?: string
|
||||
): Promise<number> => {
|
||||
const matchStage = this.buildMatchStage({ teamId, startDate, status, monitorId, resolutionType });
|
||||
return IncidentModel.countDocuments(matchStage);
|
||||
};
|
||||
|
||||
findSummaryByTeamId = async (teamId: string, limit: number): Promise<IncidentSummary> => {
|
||||
const matchStage = { teamId: new mongoose.Types.ObjectId(teamId) };
|
||||
|
||||
const counts = await IncidentModel.aggregate([
|
||||
{ $match: matchStage },
|
||||
{
|
||||
$group: {
|
||||
_id: "$status",
|
||||
count: { $sum: 1 },
|
||||
manualResolutions: {
|
||||
$sum: { $cond: [{ $eq: ["$resolutionType", "manual"] }, 1, 0] },
|
||||
},
|
||||
automaticResolutions: {
|
||||
$sum: { $cond: [{ $eq: ["$resolutionType", "automatic"] }, 1, 0] },
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
let total = 0;
|
||||
let active = 0;
|
||||
let manual = 0;
|
||||
let automatic = 0;
|
||||
|
||||
counts.forEach((item) => {
|
||||
total += item.count;
|
||||
if (item._id === true) {
|
||||
active = item.count;
|
||||
}
|
||||
manual += item.manualResolutions;
|
||||
automatic += item.automaticResolutions;
|
||||
});
|
||||
|
||||
const resolutionTimeResult = await IncidentModel.aggregate([
|
||||
{ $match: { ...matchStage, status: false, endTime: { $exists: true, $ne: null } } },
|
||||
{ $project: { resolutionTime: { $subtract: ["$endTime", "$startTime"] } } },
|
||||
{ $group: { _id: null, avgResolutionTime: { $avg: "$resolutionTime" } } },
|
||||
]);
|
||||
const avgResolutionTimeMs = resolutionTimeResult[0]?.avgResolutionTime || 0;
|
||||
const avgResolutionTimeHours = Math.round((avgResolutionTimeMs / (1000 * 60 * 60) || 0) * 100) / 100;
|
||||
|
||||
const monitorResult = await IncidentModel.aggregate([
|
||||
{ $match: matchStage },
|
||||
{ $group: { _id: "$monitorId", count: { $sum: 1 } } },
|
||||
{ $sort: { count: -1 } },
|
||||
{ $limit: 1 },
|
||||
{
|
||||
$lookup: {
|
||||
from: "monitors",
|
||||
localField: "_id",
|
||||
foreignField: "_id",
|
||||
as: "monitor",
|
||||
},
|
||||
},
|
||||
{ $project: { monitorId: "$_id", count: 1, monitorName: { $arrayElemAt: ["$monitor.name", 0] } } },
|
||||
]);
|
||||
|
||||
const latestLimit = Math.max(1, Number.isFinite(Number(limit)) ? Number(limit) : 10);
|
||||
const latestIncidents = await IncidentModel.aggregate([
|
||||
{ $match: matchStage },
|
||||
{ $sort: { createdAt: -1 } },
|
||||
{ $limit: latestLimit },
|
||||
{
|
||||
$lookup: {
|
||||
from: "monitors",
|
||||
localField: "monitorId",
|
||||
foreignField: "_id",
|
||||
as: "monitor",
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
_id: 1,
|
||||
monitorId: 1,
|
||||
monitorName: { $arrayElemAt: ["$monitor.name", 0] },
|
||||
status: 1,
|
||||
startTime: 1,
|
||||
endTime: 1,
|
||||
resolutionType: 1,
|
||||
message: 1,
|
||||
statusCode: 1,
|
||||
createdAt: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
return {
|
||||
total,
|
||||
totalActive: active,
|
||||
totalManualResolutions: manual,
|
||||
totalAutomaticResolutions: automatic,
|
||||
avgResolutionTimeHours,
|
||||
topMonitor: monitorResult[0]
|
||||
? {
|
||||
monitorId: this.toStringId(monitorResult[0].monitorId),
|
||||
monitorName: monitorResult[0].monitorName ?? null,
|
||||
incidentCount: monitorResult[0].count,
|
||||
}
|
||||
: null,
|
||||
latestIncidents: latestIncidents.map((incident) => ({
|
||||
id: this.toStringId(incident._id),
|
||||
monitorId: this.toStringId(incident.monitorId),
|
||||
monitorName: incident.monitorName ?? null,
|
||||
status: incident.status,
|
||||
startTime: this.toDateString(incident.startTime),
|
||||
endTime: incident.endTime ? this.toDateString(incident.endTime) : null,
|
||||
resolutionType: incident.resolutionType ?? null,
|
||||
message: incident.message ?? null,
|
||||
statusCode: incident.statusCode ?? null,
|
||||
createdAt: this.toDateString(incident.createdAt),
|
||||
})),
|
||||
};
|
||||
};
|
||||
}
|
||||
export default MongoIncidentRepository;
|
||||
@@ -24,3 +24,6 @@ export { default as MongoSettingsRepository } from "@/repositories/settings/Mong
|
||||
|
||||
export * from "@/repositories/notifications/INotificationsRepository.js";
|
||||
export { default as MongoNotificationsRepository } from "@/repositories/notifications/MongoNotificationsRepository.js";
|
||||
|
||||
export * from "@/repositories/incidents/IIncidentsRepository.js";
|
||||
export { default as MongoIncidentRepository } from "@/repositories/incidents/MongoIncidentRepository.js";
|
||||
@@ -1,414 +0,0 @@
|
||||
const SERVICE_NAME = "incidentService";
|
||||
|
||||
class IncidentService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ db, logger, errorService, stringService }) {
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.errorService = errorService;
|
||||
this.stringService = stringService;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return IncidentService.SERVICE_NAME;
|
||||
}
|
||||
|
||||
createIncident = async (monitor, check) => {
|
||||
try {
|
||||
if (!monitor || !monitor._id) {
|
||||
throw this.errorService.createBadRequestError("Monitor is required");
|
||||
}
|
||||
|
||||
if (!check || !check._id) {
|
||||
throw this.errorService.createBadRequestError("Check is required");
|
||||
}
|
||||
|
||||
const activeIncident = await this.db.incidentModule.getActiveIncidentByMonitor(monitor._id);
|
||||
|
||||
if (activeIncident) {
|
||||
await this.db.incidentModule.addCheckToIncident(activeIncident._id, check._id);
|
||||
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "createIncident",
|
||||
message: `Check added to existing active incident for monitor ${monitor.name}`,
|
||||
incidentId: activeIncident._id,
|
||||
monitorId: monitor._id,
|
||||
});
|
||||
|
||||
return activeIncident;
|
||||
}
|
||||
|
||||
const incidentData = {
|
||||
monitorId: monitor._id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
startTime: new Date(),
|
||||
status: true,
|
||||
message: check.message || null,
|
||||
statusCode: check.statusCode || null,
|
||||
checks: [check._id],
|
||||
};
|
||||
|
||||
const incident = await this.db.incidentModule.createIncident(incidentData);
|
||||
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "createIncident",
|
||||
message: `New incident created for monitor ${monitor.name}`,
|
||||
incidentId: incident._id,
|
||||
monitorId: monitor._id,
|
||||
});
|
||||
|
||||
return incident;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "createIncident",
|
||||
message: error.message,
|
||||
monitorId: monitor?._id,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
resolveIncident = async (monitor, check) => {
|
||||
try {
|
||||
if (!monitor || !monitor._id) {
|
||||
throw this.errorService.createBadRequestError("Monitor is required");
|
||||
}
|
||||
|
||||
const activeIncident = await this.db.incidentModule.getActiveIncidentByMonitor(monitor._id);
|
||||
|
||||
if (!activeIncident) {
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "resolveIncident",
|
||||
message: `No active incident found for monitor ${monitor.name}`,
|
||||
monitorId: monitor._id,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
await this.db.incidentModule.addCheckToIncident(activeIncident._id, check._id);
|
||||
|
||||
const resolvedIncident = await this.db.incidentModule.resolveIncident(activeIncident._id, {
|
||||
resolutionType: "automatic",
|
||||
resolvedBy: null,
|
||||
endTime: new Date(),
|
||||
});
|
||||
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "resolveIncident",
|
||||
message: `Incident automatically resolved for monitor ${monitor.name}`,
|
||||
incidentId: resolvedIncident._id,
|
||||
monitorId: monitor._id,
|
||||
});
|
||||
|
||||
return resolvedIncident;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "resolveIncident",
|
||||
message: error.message,
|
||||
monitorId: monitor?._id,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
resolveIncidentManually = async ({ incidentId, userId, teamId, comment }) => {
|
||||
try {
|
||||
if (!incidentId) {
|
||||
throw this.errorService.createBadRequestError("No incident ID in request");
|
||||
}
|
||||
|
||||
if (!userId) {
|
||||
throw this.errorService.createBadRequestError("No user ID in request");
|
||||
}
|
||||
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError("No team ID in request");
|
||||
}
|
||||
|
||||
const incident = await this.db.incidentModule.getIncidentById(incidentId);
|
||||
|
||||
if (!incident) {
|
||||
throw this.errorService.createNotFoundError("Incident not found");
|
||||
}
|
||||
|
||||
if (!incident.teamId.equals(teamId)) {
|
||||
throw this.errorService.createAuthorizationError();
|
||||
}
|
||||
|
||||
if (incident.status === false) {
|
||||
throw this.errorService.createBadRequestError("Incident is already resolved");
|
||||
}
|
||||
|
||||
const resolvedIncident = await this.db.incidentModule.resolveIncident(incidentId, {
|
||||
resolutionType: "manual",
|
||||
resolvedBy: userId,
|
||||
comment: comment || null,
|
||||
endTime: new Date(),
|
||||
});
|
||||
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "resolveIncidentManually",
|
||||
message: `Incident manually resolved by user`,
|
||||
incidentId: resolvedIncident._id,
|
||||
userId,
|
||||
});
|
||||
|
||||
return resolvedIncident;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "resolveIncidentManually",
|
||||
message: error.message,
|
||||
incidentId,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
getIncidentsByTeam = async ({ teamId, query }) => {
|
||||
try {
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError("No team ID in request");
|
||||
}
|
||||
|
||||
const { sortOrder, dateRange, page, rowsPerPage, status, monitorId, resolutionType } = query || {};
|
||||
|
||||
const result = await this.db.incidentModule.getIncidentsByTeam({
|
||||
teamId,
|
||||
sortOrder,
|
||||
dateRange,
|
||||
page,
|
||||
rowsPerPage,
|
||||
status,
|
||||
monitorId,
|
||||
resolutionType,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "getIncidentsByTeam",
|
||||
message: error.message,
|
||||
teamId,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
getIncidentSummary = async ({ teamId, query }) => {
|
||||
try {
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError("No team ID in request");
|
||||
}
|
||||
|
||||
const { limit } = query || {};
|
||||
|
||||
const summary = await this.db.incidentModule.getIncidentSummary({
|
||||
teamId,
|
||||
limit,
|
||||
});
|
||||
|
||||
return summary;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "getIncidentSummary",
|
||||
message: error.message,
|
||||
teamId,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
getIncidentById = async ({ incidentId, teamId }) => {
|
||||
try {
|
||||
if (!incidentId) {
|
||||
throw this.errorService.createBadRequestError("No incident ID in request");
|
||||
}
|
||||
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError("No team ID in request");
|
||||
}
|
||||
|
||||
const incident = await this.db.incidentModule.getIncidentById(incidentId);
|
||||
|
||||
if (!incident) {
|
||||
throw this.errorService.createNotFoundError("Incident not found");
|
||||
}
|
||||
|
||||
if (!incident.teamId.equals(teamId)) {
|
||||
throw this.errorService.createAuthorizationError();
|
||||
}
|
||||
|
||||
return incident;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "getIncidentById",
|
||||
message: error.message,
|
||||
incidentId,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
processIncidentsFromBuffer = async (incidentBufferItems) => {
|
||||
try {
|
||||
if (!incidentBufferItems || incidentBufferItems.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const createItems = [];
|
||||
const resolveItems = [];
|
||||
|
||||
for (const item of incidentBufferItems) {
|
||||
if (item.action === "resolve") {
|
||||
resolveItems.push(item);
|
||||
} else {
|
||||
createItems.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
for (const item of resolveItems) {
|
||||
try {
|
||||
await this.resolveIncident(item.monitor, item.check);
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: `Failed to resolve incident from buffer: ${error.message}`,
|
||||
monitorId: item.monitor?._id,
|
||||
error: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (createItems.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const groupedByMonitor = {};
|
||||
for (const item of createItems) {
|
||||
if (!item.monitor || !item.monitor.id || !item.check || !item.check.id) {
|
||||
this.logger.warn({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: "Skipping item with missing monitor or check data",
|
||||
item,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const monitorId = item.monitor.id.toString();
|
||||
if (!groupedByMonitor[monitorId]) {
|
||||
groupedByMonitor[monitorId] = [];
|
||||
}
|
||||
groupedByMonitor[monitorId].push(item);
|
||||
}
|
||||
|
||||
const monitorIds = Object.keys(groupedByMonitor);
|
||||
if (monitorIds.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeIncidents = await this.db.incidentModule.getActiveIncidentsByMonitors(monitorIds);
|
||||
|
||||
const incidentsCreatedInFlush = {};
|
||||
const checksToAddToIncidents = [];
|
||||
const newIncidentsToCreate = [];
|
||||
|
||||
for (const [monitorId, items] of Object.entries(groupedByMonitor)) {
|
||||
const existingIncident = activeIncidents.get(monitorId) || incidentsCreatedInFlush[monitorId];
|
||||
|
||||
if (existingIncident) {
|
||||
const incidentId = existingIncident._id ? existingIncident._id.toString() : existingIncident;
|
||||
for (const item of items) {
|
||||
checksToAddToIncidents.push({
|
||||
incidentId,
|
||||
checkId: item.check.id.toString(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const firstItem = items[0];
|
||||
const incidentData = {
|
||||
monitorId: firstItem.monitor.id,
|
||||
teamId: firstItem.monitor.teamId,
|
||||
type: firstItem.monitor.type,
|
||||
startTime: new Date(),
|
||||
status: true,
|
||||
message: firstItem.check.message || null,
|
||||
statusCode: firstItem.check.statusCode || null,
|
||||
checks: [firstItem.check.id],
|
||||
};
|
||||
|
||||
newIncidentsToCreate.push({
|
||||
incidentData,
|
||||
monitorId,
|
||||
remainingChecks: items.slice(1), // Checks restantes para agregar después
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (newIncidentsToCreate.length > 0) {
|
||||
const incidentDataArray = newIncidentsToCreate.map((item) => item.incidentData);
|
||||
await this.db.incidentModule.createIncidents(incidentDataArray);
|
||||
|
||||
const createdIncidentsMap = await this.db.incidentModule.getActiveIncidentsByMonitors(newIncidentsToCreate.map((item) => item.monitorId));
|
||||
|
||||
for (const item of newIncidentsToCreate) {
|
||||
const createdIncident = createdIncidentsMap.get(item.monitorId);
|
||||
if (createdIncident && createdIncident._id) {
|
||||
const incidentId = createdIncident._id.toString();
|
||||
incidentsCreatedInFlush[item.monitorId] = incidentId;
|
||||
|
||||
for (const remainingItem of item.remainingChecks) {
|
||||
checksToAddToIncidents.push({
|
||||
incidentId,
|
||||
checkId: remainingItem.check.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (checksToAddToIncidents.length > 0) {
|
||||
await this.db.incidentModule.addChecksToIncidentsBatch(checksToAddToIncidents);
|
||||
}
|
||||
|
||||
this.logger.info({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: `Processed ${incidentBufferItems.length} incident buffer items`,
|
||||
created: newIncidentsToCreate.length,
|
||||
checksAdded: checksToAddToIncidents.length,
|
||||
resolved: resolveItems.length,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
service: this.SERVICE_NAME,
|
||||
method: "processIncidentsFromBuffer",
|
||||
message: error.message,
|
||||
error: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default IncidentService;
|
||||
@@ -0,0 +1,225 @@
|
||||
const SERVICE_NAME = "incidentService";
|
||||
import type { Monitor } from "@/types/monitor.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ParseBoolean } from "@/utils/utils.js";
|
||||
import type { IIncidentsRepository } from "@/repositories/index.js";
|
||||
import type { Incident } from "@/types/index.js";
|
||||
|
||||
const dateRangeLookup: Record<string, Date | undefined> = {
|
||||
recent: new Date(new Date().setDate(new Date().getDate() - 2)),
|
||||
hour: new Date(new Date().setHours(new Date().getHours() - 1)),
|
||||
day: new Date(new Date().setDate(new Date().getDate() - 1)),
|
||||
week: new Date(new Date().setDate(new Date().getDate() - 7)),
|
||||
month: new Date(new Date().setMonth(new Date().getMonth() - 1)),
|
||||
all: undefined,
|
||||
};
|
||||
|
||||
class IncidentService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private db: any;
|
||||
private logger: any;
|
||||
private errorService: any;
|
||||
private stringService: any;
|
||||
private incidentsRepository: IIncidentsRepository;
|
||||
|
||||
constructor({
|
||||
db,
|
||||
logger,
|
||||
errorService,
|
||||
stringService,
|
||||
incidentsRepository,
|
||||
}: {
|
||||
db: any;
|
||||
logger: any;
|
||||
errorService: any;
|
||||
stringService: any;
|
||||
incidentsRepository: IIncidentsRepository;
|
||||
}) {
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.errorService = errorService;
|
||||
this.stringService = stringService;
|
||||
this.incidentsRepository = incidentsRepository;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
return IncidentService.SERVICE_NAME;
|
||||
}
|
||||
|
||||
handleIncident = async (monitor: Monitor, code: number): Promise<Incident | null> => {
|
||||
const activeIncident = await this.incidentsRepository.findActiveByMonitorId(monitor.id, monitor.teamId);
|
||||
// Monitor is down, create an incident
|
||||
if (monitor.status === false) {
|
||||
if (activeIncident) {
|
||||
return activeIncident;
|
||||
} else {
|
||||
const incident = {
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
startTime: Date.now().toString(),
|
||||
status: true,
|
||||
statusCode: code,
|
||||
};
|
||||
return await this.incidentsRepository.create(incident);
|
||||
}
|
||||
}
|
||||
|
||||
// Monitor is up, resolve active incidents
|
||||
if (!activeIncident) {
|
||||
return null;
|
||||
}
|
||||
activeIncident.status = false;
|
||||
activeIncident.endTime = Date.now().toString();
|
||||
activeIncident.resolutionType = "automatic";
|
||||
return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
|
||||
};
|
||||
|
||||
resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string) => {
|
||||
try {
|
||||
if (!incidentId) {
|
||||
throw new AppError({ message: "No incident ID in request", service: SERVICE_NAME, method: "resolveIncident" });
|
||||
}
|
||||
|
||||
if (!userId) {
|
||||
throw new AppError({ message: "No user ID in request", service: SERVICE_NAME, method: "resolveIncident" });
|
||||
}
|
||||
|
||||
if (!teamId) {
|
||||
throw new AppError({ message: "No team ID in request", service: SERVICE_NAME, method: "resolveIncident" });
|
||||
}
|
||||
|
||||
const incident = await this.incidentsRepository.findActiveByIncidentId(incidentId, teamId);
|
||||
|
||||
if (!incident) {
|
||||
throw new AppError({ message: "Incident not found", service: SERVICE_NAME, method: "resolveIncident" });
|
||||
}
|
||||
|
||||
if (incident.status === false) {
|
||||
throw new AppError({ message: "Incident is already resolved", service: SERVICE_NAME, method: "resolveIncident" });
|
||||
}
|
||||
|
||||
incident.resolutionType = "manual";
|
||||
incident.status = false;
|
||||
incident.resolvedBy = userId;
|
||||
incident.comment = comment || null;
|
||||
incident.endTime = Date.now().toString();
|
||||
|
||||
const resolvedIncident = await this.incidentsRepository.updateById(incident.id, teamId, incident);
|
||||
|
||||
this.logger.debug({
|
||||
service: SERVICE_NAME,
|
||||
method: "resolveIncidentManually",
|
||||
message: `Incident manually resolved by user`,
|
||||
details: resolvedIncident.id,
|
||||
});
|
||||
|
||||
return resolvedIncident;
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
service: SERVICE_NAME,
|
||||
method: "resolveIncident",
|
||||
message: error.message,
|
||||
details: incidentId,
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
getIncidentsByTeam = async ({ teamId, query }: { teamId: string; query?: any }) => {
|
||||
try {
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError("No team ID in request");
|
||||
}
|
||||
|
||||
const { sortOrder, dateRange, page, rowsPerPage, status, monitorId, resolutionType } = query || {};
|
||||
const startDate = dateRangeLookup[dateRange];
|
||||
|
||||
const parsedPage = Number.isFinite(parseInt(page)) ? parseInt(page) : 0;
|
||||
const parsedRowsPerPage = Number.isFinite(parseInt(rowsPerPage)) ? parseInt(rowsPerPage) : 20;
|
||||
const parsedStatus = typeof status === "undefined" ? undefined : ParseBoolean(status);
|
||||
|
||||
const incidents = await this.incidentsRepository.findByTeamId(
|
||||
teamId,
|
||||
startDate,
|
||||
parsedPage,
|
||||
parsedRowsPerPage,
|
||||
sortOrder,
|
||||
parsedStatus,
|
||||
monitorId,
|
||||
resolutionType
|
||||
);
|
||||
|
||||
const count = await this.incidentsRepository.countByTeamId(teamId, startDate, parsedStatus, monitorId, resolutionType);
|
||||
|
||||
return { incidents, count };
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
service: SERVICE_NAME,
|
||||
method: "getIncidentsByTeam",
|
||||
message: error.message,
|
||||
details: teamId,
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
getIncidentSummary = async (teamId: string, limit?: string) => {
|
||||
try {
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError(" team ID in request");
|
||||
}
|
||||
|
||||
const parsedLimit = limit && Number.isFinite(parseInt(limit, 10)) ? parseInt(limit, 10) : 10;
|
||||
const summary = await this.incidentsRepository.findSummaryByTeamId(teamId, parsedLimit);
|
||||
|
||||
return summary;
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
service: SERVICE_NAME,
|
||||
method: "getIncidentSummary",
|
||||
message: error.message,
|
||||
details: teamId,
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
getIncidentById = async ({ incidentId, teamId }: { incidentId: string; teamId: string }) => {
|
||||
try {
|
||||
if (!incidentId) {
|
||||
throw this.errorService.createBadRequestError("No incident ID in request");
|
||||
}
|
||||
|
||||
if (!teamId) {
|
||||
throw this.errorService.createBadRequestError("No team ID in request");
|
||||
}
|
||||
|
||||
const incident = await this.db.incidentModule.getIncidentById(incidentId);
|
||||
|
||||
if (!incident) {
|
||||
throw this.errorService.createNotFoundError("Incident not found");
|
||||
}
|
||||
|
||||
if (!incident.teamId.equals(teamId)) {
|
||||
throw this.errorService.createAuthorizationError();
|
||||
}
|
||||
|
||||
return incident;
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
service: SERVICE_NAME,
|
||||
method: "getIncidentById",
|
||||
message: error.message,
|
||||
details: incidentId,
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default IncidentService;
|
||||
@@ -2,6 +2,8 @@ const SERVICE_NAME = "JobQueueHelper";
|
||||
import type { Monitor } from "@/types/monitor.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js";
|
||||
import IncidentService from "@/service/business/incidentService.js";
|
||||
|
||||
class SuperSimpleQueueHelper {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
@@ -12,6 +14,7 @@ class SuperSimpleQueueHelper {
|
||||
private notificationsService: INotificationsService;
|
||||
private checkService: any;
|
||||
private buffer: any;
|
||||
private incidentService: IncidentService;
|
||||
|
||||
constructor({
|
||||
db,
|
||||
@@ -21,6 +24,7 @@ class SuperSimpleQueueHelper {
|
||||
notificationsService,
|
||||
checkService,
|
||||
buffer,
|
||||
incidentService,
|
||||
}: {
|
||||
db: any;
|
||||
logger: any;
|
||||
@@ -29,6 +33,7 @@ class SuperSimpleQueueHelper {
|
||||
notificationsService: INotificationsService;
|
||||
checkService: any;
|
||||
buffer: any;
|
||||
incidentService: IncidentService;
|
||||
}) {
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
@@ -37,6 +42,7 @@ class SuperSimpleQueueHelper {
|
||||
this.checkService = checkService;
|
||||
this.buffer = buffer;
|
||||
this.notificationsService = notificationsService;
|
||||
this.incidentService = incidentService;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
@@ -90,6 +96,11 @@ class SuperSimpleQueueHelper {
|
||||
stack: error.stack,
|
||||
});
|
||||
});
|
||||
|
||||
// Step 6. Handle incidents
|
||||
if (statusChangeResult.statusChanged) {
|
||||
await this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code);
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.warn({
|
||||
message: error.message,
|
||||
|
||||
@@ -7,21 +7,17 @@ class BufferService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
private BUFFER_TIMEOUT: number;
|
||||
private logger: any;
|
||||
private incidentService: any;
|
||||
private SERVICE_NAME: string;
|
||||
private buffer: any[];
|
||||
private incidentBuffer: any[];
|
||||
private bufferTimer: NodeJS.Timeout | null = null;
|
||||
private checksService: any;
|
||||
|
||||
constructor({ logger, incidentService, checkService }: { logger: any; incidentService: any; checkService: any }) {
|
||||
constructor({ logger, checkService }: { logger: any; checkService: any }) {
|
||||
this.BUFFER_TIMEOUT = config.NODE_ENV === "development" ? 10 : 1000 * 60 * 1; // 1 minute
|
||||
this.logger = logger;
|
||||
this.incidentService = incidentService;
|
||||
this.checksService = checkService;
|
||||
this.SERVICE_NAME = SERVICE_NAME;
|
||||
this.buffer = [];
|
||||
this.incidentBuffer = [];
|
||||
this.scheduleNextFlush();
|
||||
this.logger.info({
|
||||
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
|
||||
@@ -47,28 +43,6 @@ class BufferService {
|
||||
}
|
||||
}
|
||||
|
||||
addIncidentToBuffer({ monitor, check, action = "create" }: { monitor: any; check: Check; action?: string }) {
|
||||
try {
|
||||
if (!monitor || !check) {
|
||||
this.logger.warn({
|
||||
message: "Skipping incident buffer item: missing monitor or check",
|
||||
service: this.SERVICE_NAME,
|
||||
method: "addIncidentToBuffer",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
this.incidentBuffer.push({ monitor, check, action });
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "addIncidentToBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
removeCheckFromBuffer(checkToRemove: Check) {
|
||||
try {
|
||||
if (!checkToRemove) {
|
||||
@@ -141,40 +115,7 @@ class BufferService {
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.incidentBuffer.length > 0 && this.incidentService) {
|
||||
await this.flushIncidentBuffer();
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
|
||||
this.buffer = [];
|
||||
this.incidentBuffer = [];
|
||||
}
|
||||
|
||||
async flushIncidentBuffer() {
|
||||
if (!this.incidentService || this.incidentBuffer.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const itemsToProcess = [...this.incidentBuffer];
|
||||
await this.incidentService.processIncidentsFromBuffer(itemsToProcess);
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
message: `Error flushing incident buffer: ${error.message}`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushIncidentBuffer",
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -218,70 +218,6 @@ export class StatusService implements IStatusService {
|
||||
statusChanged = true;
|
||||
}
|
||||
|
||||
if (statusChanged) {
|
||||
this.logger.info({
|
||||
service: SERVICE_NAME,
|
||||
message: `${monitor.name} went from ${this.getStatusString(prevStatus)} to ${this.getStatusString(newStatus)}`,
|
||||
prevStatus,
|
||||
newStatus,
|
||||
});
|
||||
|
||||
if (newStatus === false) {
|
||||
await this.handleIncidentForCheck(check, monitor, "create", "status change to down");
|
||||
} else if (prevStatus === false) {
|
||||
await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up");
|
||||
}
|
||||
}
|
||||
|
||||
if (monitor.status === false && !statusChanged) {
|
||||
try {
|
||||
const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor.id);
|
||||
|
||||
let calculatedFailureRate = failureRate;
|
||||
|
||||
if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) {
|
||||
try {
|
||||
const checksAfterResolution = await CheckModel.find({
|
||||
monitorId: monitor.id,
|
||||
createdAt: { $gt: lastManuallyResolvedIncident.endTime },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.limit(monitor.statusWindowSize)
|
||||
.select("status")
|
||||
.lean();
|
||||
|
||||
if (checksAfterResolution.length > 0) {
|
||||
const checksStatuses = checksAfterResolution.map((c) => c.status);
|
||||
const failuresAfterResolution = checksStatuses.filter((s) => s === false).length;
|
||||
calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100;
|
||||
} else {
|
||||
calculatedFailureRate = 0;
|
||||
}
|
||||
} catch (checkQueryError: any) {
|
||||
this.logger.error({
|
||||
service: SERVICE_NAME,
|
||||
method: "updateStatus",
|
||||
message: `Failed to query checks after manual resolution: ${checkQueryError.message}`,
|
||||
monitorId: monitor.id,
|
||||
stack: checkQueryError.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (calculatedFailureRate >= monitor.statusWindowThreshold) {
|
||||
await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change");
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
service: SERVICE_NAME,
|
||||
method: "updateStatus",
|
||||
message: `Error handling threshold check without status change: ${error.message}`,
|
||||
monitorId: monitor.id,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
monitor.status = newStatus;
|
||||
const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
|
||||
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
// export type IncidentResolutionType = "automatic" | "manual" | null;
|
||||
|
||||
export const IncidentResolutionTypes = ["automatic", "manual", null] as const;
|
||||
export type IncidentResolutionType = (typeof IncidentResolutionTypes)[number];
|
||||
|
||||
export interface Incident {
|
||||
id: string;
|
||||
monitorId: string;
|
||||
teamId: string;
|
||||
startTime: string;
|
||||
endTime: string | null;
|
||||
status: boolean;
|
||||
message?: string | null;
|
||||
statusCode?: number | null;
|
||||
resolutionType: IncidentResolutionType;
|
||||
resolvedBy?: string | null;
|
||||
comment?: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface IncidentSummaryTopMonitor {
|
||||
monitorId: string;
|
||||
monitorName: string | null;
|
||||
incidentCount: number;
|
||||
}
|
||||
|
||||
export interface IncidentSummaryItem {
|
||||
id: string;
|
||||
monitorId: string;
|
||||
monitorName: string | null;
|
||||
status: boolean;
|
||||
startTime: string;
|
||||
endTime: string | null;
|
||||
resolutionType: IncidentResolutionType;
|
||||
message: string | null;
|
||||
statusCode: number | null;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface IncidentSummary {
|
||||
total: number;
|
||||
totalActive: number;
|
||||
totalManualResolutions: number;
|
||||
totalAutomaticResolutions: number;
|
||||
avgResolutionTimeHours: number;
|
||||
topMonitor: IncidentSummaryTopMonitor | null;
|
||||
latestIncidents: IncidentSummaryItem[];
|
||||
}
|
||||
@@ -9,3 +9,4 @@ export * from "@/types/recoveryToken.js";
|
||||
export * from "@/types/settings.js";
|
||||
export * from "@/types/notification.js";
|
||||
export * from "@/types/alert.js";
|
||||
export * from "@/types/incident.js";
|
||||
@@ -1,4 +1,4 @@
|
||||
const ParseBoolean = (value) => {
|
||||
export const ParseBoolean = (value: boolean | string | null | undefined) => {
|
||||
if (value === true || value === "true") {
|
||||
return true;
|
||||
} else if (value === false || value === "false" || value === null || value === undefined) {
|
||||
@@ -6,7 +6,7 @@ const ParseBoolean = (value) => {
|
||||
}
|
||||
};
|
||||
|
||||
const getTokenFromHeaders = (headers) => {
|
||||
export const getTokenFromHeaders = (headers: Record<string, string>) => {
|
||||
const authorizationHeader = headers.authorization;
|
||||
if (!authorizationHeader) throw new Error("No auth headers");
|
||||
|
||||
@@ -15,5 +15,3 @@ const getTokenFromHeaders = (headers) => {
|
||||
|
||||
return parts[1];
|
||||
};
|
||||
|
||||
export { ParseBoolean, getTokenFromHeaders };
|
||||
Reference in new issue
Block a user