274 lines
9.7 KiB
TypeScript
Executable File
274 lines
9.7 KiB
TypeScript
Executable File
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
|
|
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
|
import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js";
|
|
import {
|
|
HardwareMetricKeys,
|
|
MonitorStatuses,
|
|
type HardwareBreaches,
|
|
type HardwareCounters,
|
|
type Monitor,
|
|
type MonitorStatus,
|
|
} from "@/domain/monitors/monitor.type.js";
|
|
import type { StatusChangeResult } from "@/types/network.js";
|
|
import { AppError } from "@/utils/AppError.js";
|
|
import { ILogger } from "@/utils/logger.js";
|
|
import type { HardwareStatusMetrics } from "@/types/network.js";
|
|
import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js";
|
|
import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js";
|
|
import { findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
|
|
|
const SERVICE_NAME = "StatusService";
|
|
const HARDWARE_ALERT_COUNTER_START = 5;
|
|
|
|
export interface IStatusService {
|
|
updateRunningStats(check: Check, monitor: Monitor): Promise<boolean>;
|
|
updateMonitorStatus(check: Check, monitor: Monitor): Promise<StatusChangeResult>;
|
|
}
|
|
|
|
export class StatusService implements IStatusService {
|
|
static SERVICE_NAME = SERVICE_NAME;
|
|
private logger: ILogger;
|
|
private monitorsRepository: IMonitorsRepository;
|
|
private monitorStatsRepository: IMonitorStatsRepository;
|
|
|
|
constructor(logger: ILogger, monitorsRepository: IMonitorsRepository, monitorStatsRepository: IMonitorStatsRepository) {
|
|
this.logger = logger;
|
|
this.monitorsRepository = monitorsRepository;
|
|
this.monitorStatsRepository = monitorStatsRepository;
|
|
}
|
|
|
|
async updateRunningStats(check: Check, monitor: Monitor) {
|
|
try {
|
|
await this.monitorStatsRepository.updateByMonitorId(monitor.id, {
|
|
status: check.status === true,
|
|
responseTime: check.responseTime ?? 0,
|
|
now: Date.now(),
|
|
});
|
|
return true;
|
|
} catch (error: unknown) {
|
|
this.logger.error({
|
|
service: SERVICE_NAME,
|
|
message: error instanceof Error ? error.message : "Unknown error",
|
|
method: "updateRunningStats",
|
|
stack: error instanceof Error ? error.stack : undefined,
|
|
});
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private tryUpdateRunningStats = async (check: Check, monitor: Monitor) => {
|
|
const statsOk = await this.updateRunningStats(check, monitor);
|
|
if (!statsOk) {
|
|
this.logger.warn({
|
|
service: SERVICE_NAME,
|
|
method: "updateMonitorStatus",
|
|
message: `Stats update failed for monitor ${monitor.id}`,
|
|
});
|
|
}
|
|
};
|
|
|
|
private computeReachability = (
|
|
currentStatus: MonitorStatus,
|
|
window: Array<boolean>,
|
|
threshold: number
|
|
): { nextStatus: "up" | "down"; transitioned: boolean } => {
|
|
const failures = window.filter((status) => status === false).length;
|
|
const failureRate = (failures / window.length) * 100;
|
|
|
|
if (failureRate >= threshold && currentStatus !== "down") {
|
|
return { nextStatus: "down", transitioned: true };
|
|
}
|
|
|
|
if (failureRate < threshold && currentStatus === "down") {
|
|
return { nextStatus: "up", transitioned: true };
|
|
}
|
|
return { nextStatus: "up", transitioned: false };
|
|
};
|
|
|
|
private computeHardwareStatus = (params: {
|
|
currentStatus: MonitorStatus;
|
|
reachabilityDown: boolean;
|
|
metrics: HardwareStatusMetrics;
|
|
thresholds: { cpu: number; memory: number; disk: number; temp: number };
|
|
counters: HardwareCounters;
|
|
}): {
|
|
nextStatus: MonitorStatus;
|
|
transitioned: boolean;
|
|
breaches: HardwareBreaches;
|
|
nextCounters: HardwareCounters;
|
|
} => {
|
|
const { metrics, thresholds, counters, currentStatus, reachabilityDown } = params;
|
|
|
|
const cpuUsage = metrics.cpu?.usage_percent ?? -1;
|
|
const memoryUsage = metrics.memory?.usage_percent ?? -1;
|
|
const temps = metrics.cpu?.temperature ?? [];
|
|
|
|
const breaches: HardwareBreaches = {
|
|
cpu: cpuUsage !== -1 && cpuUsage > thresholds.cpu / 100,
|
|
memory: memoryUsage !== -1 && memoryUsage > thresholds.memory / 100,
|
|
disk: metrics.disk
|
|
? metrics.disk.some((d: CheckDiskInfo) => d != null && typeof d.usage_percent === "number" && d.usage_percent > thresholds.disk / 100)
|
|
: false,
|
|
temp: temps.some((temp: number) => temp > thresholds.temp),
|
|
};
|
|
|
|
// Update counters: decrement (floored at 0) if breached, reset to start otherwise.
|
|
const nextCounters = { ...counters };
|
|
for (const key of HardwareMetricKeys) {
|
|
nextCounters[key] = breaches[key] ? Math.max(0, counters[key] - 1) : HARDWARE_ALERT_COUNTER_START;
|
|
}
|
|
|
|
// Status transition: reachability "down" takes precedence; hardware can only drive
|
|
// transitions into "breached" and back out to "up". Any other case leaves status untouched.
|
|
let nextStatus: MonitorStatus = currentStatus;
|
|
let transitioned = false;
|
|
|
|
if (!reachabilityDown) {
|
|
// A counter can only reach zero via the decrement path, which only runs when that
|
|
// metric is currently breaching — so anyCounterZero already implies anyBreached.
|
|
const anyCounterZero = HardwareMetricKeys.some((k) => nextCounters[k] === 0);
|
|
const allNormal = HardwareMetricKeys.every((k) => !breaches[k]);
|
|
|
|
if (anyCounterZero && currentStatus !== "breached") {
|
|
nextStatus = "breached";
|
|
transitioned = true;
|
|
} else if (allNormal && currentStatus === "breached") {
|
|
nextStatus = "up";
|
|
transitioned = true;
|
|
}
|
|
}
|
|
|
|
return { nextStatus, transitioned, breaches, nextCounters };
|
|
};
|
|
|
|
private computeDockerStatus = (params: { currentStatus: MonitorStatus; reachabilityDown: boolean; breaching: boolean }) => {
|
|
const { currentStatus, reachabilityDown, breaching } = params;
|
|
let nextStatus: MonitorStatus = currentStatus;
|
|
let transitioned = false;
|
|
|
|
if (!reachabilityDown) {
|
|
if (breaching && currentStatus !== "breached") {
|
|
nextStatus = "breached";
|
|
transitioned = true;
|
|
} else if (!breaching && currentStatus === "breached") {
|
|
nextStatus = "up";
|
|
transitioned = true;
|
|
}
|
|
}
|
|
return { nextStatus, transitioned };
|
|
};
|
|
|
|
updateMonitorStatus = async (check: Check, monitor: Monitor): Promise<StatusChangeResult> => {
|
|
try {
|
|
const { status } = check;
|
|
|
|
// Update running stats
|
|
await this.tryUpdateRunningStats(check, monitor);
|
|
|
|
const prevStatus = monitor.status;
|
|
const checkSnapshot = toCheckSnapshot(check);
|
|
|
|
// Project the window as it will look after updating DB
|
|
// This is done because we need the updated status window to compute new status, but we don't
|
|
// want an an extra DB write just to get the window.
|
|
const projectedWindow = [...(monitor.statusWindow || []), check.status].slice(-monitor.statusWindowSize);
|
|
|
|
// Build the status patch — computed against the projected window
|
|
const patch: Partial<Monitor> = {};
|
|
|
|
// Resolve "initializing" and "maintenance" up front, incidents should be created on initialization && down
|
|
// Unknown values (e.g. legacy boolean statuses from old versions) are resolved the same way,
|
|
// otherwise computeReachability can never transition them and they stay stuck forever.
|
|
const isUnknownStatus = !MonitorStatuses.includes(monitor.status);
|
|
let newStatus: MonitorStatus = monitor.status;
|
|
let statusChanged = false;
|
|
if (monitor.status === "initializing" || monitor.status === "maintenance" || isUnknownStatus) {
|
|
newStatus = status ? "up" : "down";
|
|
patch.status = newStatus;
|
|
statusChanged = newStatus === "down";
|
|
}
|
|
|
|
let thresholdBreaches: HardwareBreaches | undefined;
|
|
|
|
if (projectedWindow.length >= monitor.statusWindowSize) {
|
|
// First evaluate reachability status changes, which apply to all monitor types
|
|
// and take precedence over hardware breaches.
|
|
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
|
|
if (reachabilityResult.transitioned) {
|
|
newStatus = reachabilityResult.nextStatus;
|
|
statusChanged = true;
|
|
}
|
|
|
|
// Evaluate hardware threshold breaches
|
|
if (monitor.type === "hardware") {
|
|
const hardware = this.computeHardwareStatus({
|
|
currentStatus: newStatus,
|
|
reachabilityDown: newStatus === "down",
|
|
metrics: check,
|
|
thresholds: {
|
|
cpu: monitor.cpuAlertThreshold,
|
|
memory: monitor.memoryAlertThreshold,
|
|
disk: monitor.diskAlertThreshold,
|
|
temp: monitor.tempAlertThreshold,
|
|
},
|
|
counters: {
|
|
cpu: monitor.cpuAlertCounter,
|
|
memory: monitor.memoryAlertCounter,
|
|
disk: monitor.diskAlertCounter,
|
|
temp: monitor.tempAlertCounter,
|
|
},
|
|
});
|
|
|
|
patch.cpuAlertCounter = hardware.nextCounters.cpu;
|
|
patch.memoryAlertCounter = hardware.nextCounters.memory;
|
|
patch.diskAlertCounter = hardware.nextCounters.disk;
|
|
patch.tempAlertCounter = hardware.nextCounters.temp;
|
|
thresholdBreaches = hardware.breaches;
|
|
if (hardware.transitioned) {
|
|
newStatus = hardware.nextStatus;
|
|
statusChanged = true;
|
|
}
|
|
}
|
|
|
|
if (monitor.type === "docker" && check.containers) {
|
|
const breaches = findContainerBreaches(monitor, check.containers);
|
|
const docker = this.computeDockerStatus({
|
|
currentStatus: newStatus,
|
|
reachabilityDown: newStatus === "down",
|
|
breaching: breaches.length > 0,
|
|
});
|
|
if (docker.transitioned) {
|
|
newStatus = docker.nextStatus;
|
|
statusChanged = true;
|
|
}
|
|
}
|
|
patch.status = newStatus;
|
|
}
|
|
|
|
// Single atomic write: push arrays + set status/counters
|
|
const updated = await this.monitorsRepository.updateStatusWindowAndChecks(
|
|
monitor.id,
|
|
monitor.teamId,
|
|
check.status,
|
|
checkSnapshot,
|
|
monitor.statusWindowSize,
|
|
MAX_RECENT_CHECKS,
|
|
patch
|
|
);
|
|
|
|
return {
|
|
monitor: updated,
|
|
statusChanged,
|
|
prevStatus,
|
|
thresholdBreaches,
|
|
};
|
|
} catch (error: unknown) {
|
|
throw new AppError({
|
|
message: `Failed to update monitor with id ${check.metadata.monitorId} with status: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
service: SERVICE_NAME,
|
|
method: "updateMonitorStatus",
|
|
});
|
|
}
|
|
};
|
|
}
|