2026-01-15 22:33:25 +00:00
|
|
|
import { IMonitorsRepository } from "@/repositories/index.js";
|
2026-01-12 19:32:23 +00:00
|
|
|
import MonitorStats from "../../db/models/MonitorStats.js";
|
2026-01-12 22:58:46 +00:00
|
|
|
import { CheckModel } from "@/db/models/index.js";
|
2026-01-16 18:11:04 +00:00
|
|
|
import type {
|
|
|
|
|
CheckErrorInfo,
|
|
|
|
|
Monitor,
|
|
|
|
|
MonitorStatusResponse,
|
|
|
|
|
StatusChangeResult,
|
|
|
|
|
Check,
|
|
|
|
|
HardwareStatusPayload,
|
|
|
|
|
PageSpeedStatusPayload,
|
|
|
|
|
} from "@/types/index.js";
|
2025-04-20 11:29:53 -07:00
|
|
|
const SERVICE_NAME = "StatusService";
|
|
|
|
|
|
|
|
|
|
class StatusService {
|
|
|
|
|
static SERVICE_NAME = SERVICE_NAME;
|
2026-01-15 22:33:25 +00:00
|
|
|
private db: any;
|
|
|
|
|
private logger: any;
|
|
|
|
|
private buffer: any;
|
|
|
|
|
private monitorsRepository: IMonitorsRepository;
|
|
|
|
|
|
2026-01-18 22:52:59 +00:00
|
|
|
constructor({ db, logger, buffer, monitorsRepository }: { db: any; logger: any; buffer: any; monitorsRepository: IMonitorsRepository }) {
|
2025-04-20 11:29:53 -07:00
|
|
|
this.db = db;
|
|
|
|
|
this.logger = logger;
|
|
|
|
|
this.buffer = buffer;
|
2026-01-13 20:20:00 +00:00
|
|
|
this.monitorsRepository = monitorsRepository;
|
2025-07-25 15:15:14 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
get serviceName() {
|
|
|
|
|
return StatusService.SERVICE_NAME;
|
2025-04-20 11:29:53 -07:00
|
|
|
}
|
|
|
|
|
|
2026-01-15 22:33:25 +00:00
|
|
|
async updateRunningStats({ monitor, networkResponse }: { monitor: Monitor; networkResponse: any }) {
|
2025-04-20 11:29:53 -07:00
|
|
|
try {
|
2026-01-13 20:20:00 +00:00
|
|
|
const monitorId = monitor.id;
|
2025-08-18 10:55:12 -07:00
|
|
|
const { responseTime, status } = networkResponse;
|
2025-04-20 11:29:53 -07:00
|
|
|
// Get stats
|
|
|
|
|
let stats = await MonitorStats.findOne({ monitorId });
|
|
|
|
|
if (!stats) {
|
|
|
|
|
stats = new MonitorStats({
|
|
|
|
|
monitorId,
|
|
|
|
|
avgResponseTime: 0,
|
|
|
|
|
totalChecks: 0,
|
|
|
|
|
totalUpChecks: 0,
|
|
|
|
|
totalDownChecks: 0,
|
|
|
|
|
uptimePercentage: 0,
|
|
|
|
|
lastCheck: null,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Update stats
|
|
|
|
|
|
|
|
|
|
// Last response time
|
|
|
|
|
stats.lastResponseTime = responseTime;
|
|
|
|
|
|
|
|
|
|
// Avg response time:
|
|
|
|
|
let avgResponseTime = stats.avgResponseTime;
|
|
|
|
|
if (typeof responseTime !== "undefined" && responseTime !== null) {
|
|
|
|
|
if (avgResponseTime === 0) {
|
|
|
|
|
avgResponseTime = responseTime;
|
|
|
|
|
} else {
|
2025-07-21 10:34:16 -07:00
|
|
|
avgResponseTime = (avgResponseTime * (stats.totalChecks - 1) + responseTime) / stats.totalChecks;
|
2025-04-20 11:29:53 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
stats.avgResponseTime = avgResponseTime;
|
|
|
|
|
|
|
|
|
|
// Total checks
|
|
|
|
|
stats.totalChecks++;
|
|
|
|
|
if (status === true) {
|
|
|
|
|
stats.totalUpChecks++;
|
|
|
|
|
// Update the timeSinceLastFailure if needed
|
|
|
|
|
if (stats.timeOfLastFailure === 0) {
|
|
|
|
|
stats.timeOfLastFailure = new Date().getTime();
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
stats.totalDownChecks++;
|
|
|
|
|
stats.timeOfLastFailure = 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Calculate uptime percentage
|
|
|
|
|
let uptimePercentage;
|
|
|
|
|
if (stats.totalChecks > 0) {
|
|
|
|
|
uptimePercentage = stats.totalUpChecks / stats.totalChecks;
|
|
|
|
|
} else {
|
|
|
|
|
uptimePercentage = status === true ? 100 : 0;
|
|
|
|
|
}
|
|
|
|
|
stats.uptimePercentage = uptimePercentage;
|
|
|
|
|
|
|
|
|
|
// latest check
|
|
|
|
|
stats.lastCheckTimestamp = new Date().getTime();
|
|
|
|
|
|
|
|
|
|
await stats.save();
|
|
|
|
|
return true;
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (error: any) {
|
2025-04-20 11:29:53 -07:00
|
|
|
this.logger.error({
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-04-20 11:29:53 -07:00
|
|
|
message: error.message,
|
|
|
|
|
method: "updateRunningStats",
|
|
|
|
|
stack: error.stack,
|
|
|
|
|
});
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-01-15 22:33:25 +00:00
|
|
|
getStatusString = (status: boolean | undefined) => {
|
2025-04-20 11:29:53 -07:00
|
|
|
if (status === true) return "up";
|
|
|
|
|
if (status === false) return "down";
|
|
|
|
|
return "unknown";
|
|
|
|
|
};
|
2025-11-18 16:01:47 -05:00
|
|
|
|
2026-01-15 22:33:25 +00:00
|
|
|
handleIncidentForCheck = async (check: any, monitor: Monitor, action: any, errorContext = "incident handling") => {
|
2025-11-18 16:01:47 -05:00
|
|
|
try {
|
|
|
|
|
let savedCheck = check;
|
|
|
|
|
|
|
|
|
|
if (!check._id) {
|
|
|
|
|
try {
|
2026-01-12 22:58:46 +00:00
|
|
|
const checkModel = new CheckModel(check);
|
2025-11-18 16:01:47 -05:00
|
|
|
savedCheck = await checkModel.save();
|
|
|
|
|
|
|
|
|
|
this.buffer.removeCheckFromBuffer(check);
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (checkError: any) {
|
2025-11-18 16:01:47 -05:00
|
|
|
this.logger.error({
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-11-18 16:01:47 -05:00
|
|
|
method: "handleIncidentForCheck",
|
|
|
|
|
message: `Failed to save check immediately for ${errorContext}: ${checkError.message}`,
|
2026-01-13 20:20:00 +00:00
|
|
|
monitorId: monitor.id,
|
2025-11-18 16:01:47 -05:00
|
|
|
stack: checkError.stack,
|
|
|
|
|
});
|
|
|
|
|
savedCheck = null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (savedCheck && savedCheck._id) {
|
|
|
|
|
try {
|
|
|
|
|
this.buffer.addIncidentToBuffer({ monitor, check: savedCheck, action });
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (incidentError: any) {
|
2025-11-18 16:01:47 -05:00
|
|
|
this.logger.error({
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-11-18 16:01:47 -05:00
|
|
|
method: "handleIncidentForCheck",
|
|
|
|
|
message: `Failed to add incident to buffer for ${errorContext}: ${incidentError.message}`,
|
2026-01-13 20:20:00 +00:00
|
|
|
monitorId: monitor.id,
|
2025-11-18 16:01:47 -05:00
|
|
|
action,
|
|
|
|
|
stack: incidentError.stack,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (error: any) {
|
2025-11-18 16:01:47 -05:00
|
|
|
this.logger.error({
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-11-18 16:01:47 -05:00
|
|
|
method: "handleIncidentForCheck",
|
|
|
|
|
message: `Error in ${errorContext}: ${error.message}`,
|
2026-01-13 20:20:00 +00:00
|
|
|
monitorId: monitor?.id,
|
2025-11-18 16:01:47 -05:00
|
|
|
stack: error.stack,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
};
|
2026-01-15 22:33:25 +00:00
|
|
|
|
2026-01-15 23:45:40 +00:00
|
|
|
updateMonitorStatus = async (
|
2026-01-16 18:11:04 +00:00
|
|
|
statusResponse: MonitorStatusResponse<PageSpeedStatusPayload | HardwareStatusPayload | undefined>,
|
|
|
|
|
check: Check
|
2026-01-15 23:45:40 +00:00
|
|
|
): Promise<StatusChangeResult> => {
|
2025-04-20 11:29:53 -07:00
|
|
|
try {
|
2026-01-15 23:45:40 +00:00
|
|
|
const { monitorId, teamId, status, code } = statusResponse;
|
2026-01-15 20:55:01 +00:00
|
|
|
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
|
2025-05-19 12:28:18 -07:00
|
|
|
|
2025-04-20 11:29:53 -07:00
|
|
|
// Update running stats
|
2026-01-15 23:45:40 +00:00
|
|
|
this.updateRunningStats({ monitor, networkResponse: statusResponse });
|
2025-05-19 12:28:18 -07:00
|
|
|
|
2025-08-18 15:34:15 -07:00
|
|
|
// If the status window size has changed, empty
|
|
|
|
|
while (monitor.statusWindow.length > monitor.statusWindowSize) {
|
|
|
|
|
monitor.statusWindow.shift();
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-18 13:40:56 -07:00
|
|
|
// Update status sliding window
|
|
|
|
|
monitor.statusWindow.push(status);
|
|
|
|
|
if (monitor.statusWindow.length > monitor.statusWindowSize) {
|
|
|
|
|
monitor.statusWindow.shift();
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-18 15:34:15 -07:00
|
|
|
if (monitor.status === undefined || monitor.status === null) {
|
2025-08-18 13:40:56 -07:00
|
|
|
monitor.status = status;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let newStatus = monitor.status;
|
|
|
|
|
let statusChanged = false;
|
|
|
|
|
const prevStatus = monitor.status;
|
|
|
|
|
|
|
|
|
|
// Return early if not enough data points
|
|
|
|
|
if (monitor.statusWindow.length < monitor.statusWindowSize) {
|
2026-01-15 18:33:03 +00:00
|
|
|
const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
|
2025-04-20 11:29:53 -07:00
|
|
|
return {
|
2026-01-13 20:20:00 +00:00
|
|
|
monitor: updated,
|
2025-04-20 11:29:53 -07:00
|
|
|
statusChanged: false,
|
2025-08-18 13:40:56 -07:00
|
|
|
prevStatus,
|
2025-04-20 11:29:53 -07:00
|
|
|
code,
|
2025-08-18 13:40:56 -07:00
|
|
|
timestamp: Date.now(),
|
2025-04-20 11:29:53 -07:00
|
|
|
};
|
2025-08-18 13:40:56 -07:00
|
|
|
}
|
2025-05-19 12:28:18 -07:00
|
|
|
|
2025-08-18 13:40:56 -07:00
|
|
|
// Check if threshold has been met
|
|
|
|
|
const failures = monitor.statusWindow.filter((s) => s === false).length;
|
2025-08-18 15:34:15 -07:00
|
|
|
const failureRate = (failures / monitor.statusWindow.length) * 100;
|
|
|
|
|
|
2025-08-18 13:40:56 -07:00
|
|
|
// If threshold has been met and the monitor is not already down, mark down:
|
2025-08-18 15:34:15 -07:00
|
|
|
if (failureRate >= monitor.statusWindowThreshold && monitor.status !== false) {
|
2025-08-18 13:40:56 -07:00
|
|
|
newStatus = false;
|
|
|
|
|
statusChanged = true;
|
|
|
|
|
}
|
|
|
|
|
// If the failure rate is below the threshold and the monitor is down, recover:
|
2025-08-30 16:19:17 -07:00
|
|
|
else if (failureRate < monitor.statusWindowThreshold && monitor.status === false) {
|
2025-08-18 13:40:56 -07:00
|
|
|
newStatus = true;
|
|
|
|
|
statusChanged = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (statusChanged) {
|
|
|
|
|
this.logger.info({
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-08-18 13:40:56 -07:00
|
|
|
message: `${monitor.name} went from ${this.getStatusString(prevStatus)} to ${this.getStatusString(newStatus)}`,
|
|
|
|
|
prevStatus,
|
|
|
|
|
newStatus,
|
|
|
|
|
});
|
2025-11-06 17:06:33 -05:00
|
|
|
|
2025-11-18 16:01:47 -05:00
|
|
|
if (newStatus === false) {
|
|
|
|
|
await this.handleIncidentForCheck(check, monitor, "create", "status change to down");
|
|
|
|
|
} else if (prevStatus === false) {
|
|
|
|
|
await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up");
|
2025-11-06 17:06:33 -05:00
|
|
|
}
|
2025-11-18 16:01:47 -05:00
|
|
|
}
|
2025-11-06 17:06:33 -05:00
|
|
|
|
2025-11-18 16:01:47 -05:00
|
|
|
if (monitor.status === false && !statusChanged) {
|
2025-11-06 17:06:33 -05:00
|
|
|
try {
|
2026-01-13 20:20:00 +00:00
|
|
|
const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor.id);
|
2025-11-18 16:01:47 -05:00
|
|
|
|
|
|
|
|
let calculatedFailureRate = failureRate;
|
|
|
|
|
|
|
|
|
|
if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) {
|
|
|
|
|
try {
|
2026-01-15 22:33:25 +00:00
|
|
|
const checksAfterResolution = await CheckModel.find({
|
2026-01-13 20:20:00 +00:00
|
|
|
monitorId: monitor.id,
|
2025-11-18 16:01:47 -05:00
|
|
|
createdAt: { $gt: lastManuallyResolvedIncident.endTime },
|
|
|
|
|
})
|
|
|
|
|
.sort({ createdAt: 1 })
|
|
|
|
|
.limit(monitor.statusWindowSize)
|
|
|
|
|
.select("status")
|
|
|
|
|
.lean();
|
|
|
|
|
|
|
|
|
|
if (checksAfterResolution.length > 0) {
|
|
|
|
|
const checksStatuses = checksAfterResolution.map((c) => c.status);
|
|
|
|
|
const failuresAfterResolution = checksStatuses.filter((s) => s === false).length;
|
|
|
|
|
calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100;
|
|
|
|
|
} else {
|
|
|
|
|
calculatedFailureRate = 0;
|
|
|
|
|
}
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (checkQueryError: any) {
|
2025-11-18 16:01:47 -05:00
|
|
|
this.logger.error({
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-11-18 16:01:47 -05:00
|
|
|
method: "updateStatus",
|
|
|
|
|
message: `Failed to query checks after manual resolution: ${checkQueryError.message}`,
|
2026-01-13 20:20:00 +00:00
|
|
|
monitorId: monitor.id,
|
2025-11-18 16:01:47 -05:00
|
|
|
stack: checkQueryError.stack,
|
|
|
|
|
});
|
|
|
|
|
}
|
2025-11-06 17:06:33 -05:00
|
|
|
}
|
2025-11-18 16:01:47 -05:00
|
|
|
|
|
|
|
|
if (calculatedFailureRate >= monitor.statusWindowThreshold) {
|
|
|
|
|
await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change");
|
|
|
|
|
}
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (error: any) {
|
2025-11-06 17:06:33 -05:00
|
|
|
this.logger.error({
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-11-06 17:06:33 -05:00
|
|
|
method: "updateStatus",
|
2025-11-18 16:01:47 -05:00
|
|
|
message: `Error handling threshold check without status change: ${error.message}`,
|
2026-01-13 20:20:00 +00:00
|
|
|
monitorId: monitor.id,
|
2025-11-18 16:01:47 -05:00
|
|
|
stack: error.stack,
|
2025-11-06 17:06:33 -05:00
|
|
|
});
|
|
|
|
|
}
|
2025-08-18 13:40:56 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
monitor.status = newStatus;
|
2026-01-15 18:33:03 +00:00
|
|
|
const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
|
2025-05-19 12:28:18 -07:00
|
|
|
|
2025-04-20 11:29:53 -07:00
|
|
|
return {
|
2026-01-13 20:20:00 +00:00
|
|
|
monitor: updated,
|
2025-08-18 13:40:56 -07:00
|
|
|
statusChanged,
|
2025-04-20 11:29:53 -07:00
|
|
|
prevStatus,
|
|
|
|
|
code,
|
|
|
|
|
timestamp: new Date().getTime(),
|
|
|
|
|
};
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (error: any) {
|
|
|
|
|
error.service = SERVICE_NAME;
|
2025-05-19 12:28:18 -07:00
|
|
|
error.method = "updateStatus";
|
2025-04-20 11:29:53 -07:00
|
|
|
throw error;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2026-01-16 18:11:04 +00:00
|
|
|
insertCheck = async (check: Check) => {
|
2025-04-20 11:29:53 -07:00
|
|
|
try {
|
|
|
|
|
if (typeof check === "undefined") {
|
|
|
|
|
this.logger.warn({
|
|
|
|
|
message: "Failed to build check",
|
2026-01-15 22:33:25 +00:00
|
|
|
service: SERVICE_NAME,
|
2025-04-20 11:29:53 -07:00
|
|
|
method: "insertCheck",
|
|
|
|
|
});
|
2025-08-18 10:55:12 -07:00
|
|
|
return false;
|
2025-04-20 11:29:53 -07:00
|
|
|
}
|
2025-08-18 10:55:12 -07:00
|
|
|
this.buffer.addToBuffer({ check });
|
|
|
|
|
return true;
|
2026-01-15 22:33:25 +00:00
|
|
|
} catch (error: any) {
|
2025-04-20 11:29:53 -07:00
|
|
|
this.logger.error({
|
|
|
|
|
message: error.message,
|
2026-01-15 22:33:25 +00:00
|
|
|
service: error.service || SERVICE_NAME,
|
2025-06-06 14:05:59 +08:00
|
|
|
method: error.method || "insertCheck",
|
2026-01-16 18:11:04 +00:00
|
|
|
details: error.details || `Error inserting check for monitor: ${check?.metadata.monitorId}`,
|
2025-04-20 11:29:53 -07:00
|
|
|
stack: error.stack,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
export default StatusService;
|