Files
Checkmate/server/src/services/infrastructure/JobGenerator.ts
T

216 lines
6.7 KiB
TypeScript
Raw Normal View History

2025-12-05 01:13:23 +00:00
import { IMonitor, ICheck, Incident } from "@/db/models/index.js";
2025-11-13 13:47:26 -08:00
import { INetworkService } from "./NetworkService.js";
import { ICheckService } from "../business/CheckService.js";
import { IMonitorStatsService } from "../business/MonitorStatsService.js";
import { IStatusService } from "./StatusService.js";
import { INotificationService } from "./NotificationService.js";
import { IMaintenanceService } from "../business/MaintenanceService.js";
import { IIncidentService } from "../business/IncidentService.js";
import ApiError from "@/utils/ApiError.js";
import { getChildLogger } from "@/logger/Logger.js";
const SERVICE_NAME = "JobGenerator";
const logger = getChildLogger(SERVICE_NAME);
export interface IJobGenerator {
generateJob: () => (Monitor: IMonitor) => Promise<void>;
generateCleanupJob: () => () => Promise<void>;
}
class JobGenerator implements IJobGenerator {
public SERVICE_NAME: string;
private networkService: INetworkService;
private checkService: ICheckService;
private monitorStatsService: IMonitorStatsService;
private statusService: IStatusService;
private notificationService: INotificationService;
private maintenanceService: IMaintenanceService;
private incidentService: IIncidentService;
constructor(
networkService: INetworkService,
checkService: ICheckService,
monitorStatsService: IMonitorStatsService,
statusService: IStatusService,
notificationService: INotificationService,
incidentService: IIncidentService,
maintenanceService: IMaintenanceService
) {
this.SERVICE_NAME = SERVICE_NAME;
this.networkService = networkService;
this.checkService = checkService;
this.monitorStatsService = monitorStatsService;
this.statusService = statusService;
this.notificationService = notificationService;
this.incidentService = incidentService;
this.maintenanceService = maintenanceService;
}
2025-12-05 01:13:23 +00:00
private evaluateThresholds = async (
monitor: IMonitor,
check: ICheck
) => {
try {
// Only for infrastructure monitors and only when service is up
if (monitor.type !== "infrastructure" || monitor.status !== "up") return;
if (!monitor.thresholds) return;
if (!check?.system) return;
const { thresholds: t } = monitor;
const breaches: string[] = [];
const cpu = check.system?.cpu?.usage_percent;
if (
typeof t?.cpu === "number" &&
typeof cpu === "number" &&
cpu > t.cpu
) {
breaches.push(`cpu ${cpu}% > ${t.cpu}%`);
}
const mem = check.system?.memory?.usage_percent;
if (
typeof t?.memory === "number" &&
typeof mem === "number" &&
mem > t.memory
) {
breaches.push(`memory ${mem}% > ${t.memory}%`);
}
const diskUsages: number[] = Array.isArray(check.system?.disk)
? check.system.disk
.map((d) => d?.usage_percent)
.filter((n): n is number => typeof n === "number")
: [];
const maxDisk = diskUsages.length ? Math.max(...diskUsages) : undefined;
if (
typeof t?.disk === "number" &&
typeof maxDisk === "number" &&
maxDisk > t.disk
) {
breaches.push(`disk ${maxDisk}% > ${t.disk}%`);
}
const temps: number[] = Array.isArray(check.system?.cpu?.temperature)
? check.system.cpu.temperature.filter(
(n): n is number => typeof n === "number"
)
: [];
const maxTemp = temps.length ? Math.max(...temps) : undefined;
if (
typeof t?.temperature === "number" &&
typeof maxTemp === "number" &&
maxTemp > t.temperature
) {
breaches.push(`temperature ${maxTemp}C > ${t.temperature}C`);
}
// If any breach → ensure one open incident exists; else resolve if one exists
const hasBreach = breaches.length > 0;
const existingOpen = await Incident.findOne({
monitorId: monitor._id,
teamId: monitor.teamId,
resolved: false,
});
if (hasBreach && !existingOpen) {
const incident = await this.incidentService.create(
monitor.teamId,
monitor._id,
check._id
);
// annotate with threshold note best-effort
try {
if (breaches.length) {
await Incident.updateOne(
{ _id: incident._id },
{
$set: {
resolutionNote: `threshold breach: ${breaches.join(", ")}`,
},
}
);
}
} catch {}
this.notificationService
.handleNotifications(monitor, incident)
.catch(() => {});
}
if (!hasBreach && existingOpen) {
await this.incidentService.resolve(
monitor.teamId.toString(),
existingOpen._id.toString(),
"auto",
check
);
}
} catch {
// best-effort; do not throw from evaluator
}
};
2025-11-13 13:47:26 -08:00
generateJob = () => {
return async (monitor: IMonitor) => {
try {
const monitorId = monitor._id.toString();
if (!monitorId) {
throw new ApiError("No monitorID for creating job", 400);
}
// Check for active maintenance window, if found, skip the check
const isInMaintenance = await this.maintenanceService.isInMaintenance(
monitorId
);
if (isInMaintenance) {
return;
}
const status = await this.networkService.requestStatus(monitor);
const check = await this.checkService.buildCheck(status, monitor.type);
await check.save();
const [updatedMonitor, statusChanged] =
await this.statusService.updateMonitorStatus(monitor, status);
if (statusChanged) {
const incident = await this.incidentService.handleStatusChange(
updatedMonitor,
check
);
// Best effort, don't wait, don't fail
if (incident) {
this.notificationService
.handleNotifications(updatedMonitor, incident)
.catch((error) => {
logger.warn(error);
});
}
}
await this.statusService.updateMonitorStats(
updatedMonitor,
status,
statusChanged
);
2025-12-05 01:13:23 +00:00
// Evaluate thresholds after saving the check and updating stats
await this.evaluateThresholds(updatedMonitor, check);
2025-11-13 13:47:26 -08:00
} catch (error) {
throw error;
}
};
};
generateCleanupJob = () => {
return async () => {
try {
await this.checkService.cleanupOrphanedChecks();
await this.monitorStatsService.cleanupOrphanedMonitorStats();
2025-11-21 20:18:34 +00:00
await this.incidentService.cleanupOrphanedIncidents();
2025-11-13 13:47:26 -08:00
} catch (error) {
throw error;
}
};
};
}
export default JobGenerator;