Files
Checkmate/server/src/service/infrastructure/statusService.ts
T

288 lines
9.1 KiB
TypeScript
Raw Normal View History

2026-02-12 22:37:40 +00:00
import { IChecksRepository, IMonitorsRepository, IMonitorStatsRepository } from "@/repositories/index.js";
2026-01-16 18:11:04 +00:00
import type {
Monitor,
2026-02-12 18:30:44 +00:00
MonitorStatus,
2026-01-16 18:11:04 +00:00
MonitorStatusResponse,
StatusChangeResult,
Check,
HardwareStatusPayload,
PageSpeedStatusPayload,
2026-03-03 19:34:28 +00:00
PingStatusPayload,
HttpStatusPayload,
DockerStatusPayload,
PortStatusPayload,
GameStatusPayload,
GrpcStatusPayload,
2026-01-27 18:26:31 +00:00
CheckSnapshot,
2026-03-03 23:01:35 +00:00
CheckDiskInfo,
2026-01-16 18:11:04 +00:00
} from "@/types/index.js";
2026-03-03 23:01:35 +00:00
import { AppError } from "@/utils/AppError.js";
2026-02-12 22:37:40 +00:00
import { ILogger } from "@/utils/logger.js";
2026-03-03 23:01:35 +00:00
import { IBufferService } from "./bufferService.js";
2025-04-20 11:29:53 -07:00
const SERVICE_NAME = "StatusService";
2026-01-18 23:38:40 +00:00
export interface IStatusService {
2026-03-03 23:01:35 +00:00
updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise<boolean>;
2026-01-18 23:38:40 +00:00
updateMonitorStatus(
2026-03-03 19:34:28 +00:00
statusResponse: MonitorStatusResponse<
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
>,
2026-01-18 23:38:40 +00:00
check: Check
): Promise<StatusChangeResult>;
}
export class StatusService implements IStatusService {
2025-04-20 11:29:53 -07:00
static SERVICE_NAME = SERVICE_NAME;
2026-03-03 23:01:35 +00:00
private logger: ILogger;
private buffer: IBufferService;
2026-01-15 22:33:25 +00:00
private monitorsRepository: IMonitorsRepository;
2026-02-12 22:37:40 +00:00
private monitorStatsRepository: IMonitorStatsRepository;
private checksRepository: IChecksRepository;
2026-01-15 22:33:25 +00:00
2026-02-12 22:37:40 +00:00
constructor(
logger: ILogger,
2026-03-03 23:01:35 +00:00
buffer: IBufferService,
2026-02-12 22:37:40 +00:00
monitorsRepository: IMonitorsRepository,
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository
) {
2025-04-20 11:29:53 -07:00
this.logger = logger;
this.buffer = buffer;
2026-01-13 20:20:00 +00:00
this.monitorsRepository = monitorsRepository;
2026-02-12 22:37:40 +00:00
this.monitorStatsRepository = monitorStatsRepository;
this.checksRepository = checksRepository;
2025-07-25 15:15:14 -07:00
}
get serviceName() {
return StatusService.SERVICE_NAME;
2025-04-20 11:29:53 -07:00
}
2026-03-03 23:01:35 +00:00
async updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse) {
2025-04-20 11:29:53 -07:00
try {
await this.monitorStatsRepository.updateByMonitorId(monitor.id, {
status: networkResponse.status === true,
responseTime: networkResponse.responseTime ?? 0,
now: Date.now(),
});
2025-04-20 11:29:53 -07:00
return true;
2026-03-03 23:01:35 +00:00
} catch (error: unknown) {
2025-04-20 11:29:53 -07:00
this.logger.error({
2026-01-15 22:33:25 +00:00
service: SERVICE_NAME,
2026-03-03 23:01:35 +00:00
message: error instanceof Error ? error.message : "Unknown error",
2025-04-20 11:29:53 -07:00
method: "updateRunningStats",
2026-03-03 23:01:35 +00:00
stack: error instanceof Error ? error.stack : undefined,
2025-04-20 11:29:53 -07:00
});
return false;
}
}
2026-01-15 23:45:40 +00:00
updateMonitorStatus = async (
2026-03-03 19:34:28 +00:00
statusResponse: MonitorStatusResponse<
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
>,
2026-01-16 18:11:04 +00:00
check: Check
2026-01-15 23:45:40 +00:00
): Promise<StatusChangeResult> => {
2025-04-20 11:29:53 -07:00
try {
2026-01-15 23:45:40 +00:00
const { monitorId, teamId, status, code } = statusResponse;
2026-01-15 20:55:01 +00:00
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
2025-05-19 12:28:18 -07:00
2025-04-20 11:29:53 -07:00
// Update running stats
const statsOk = await this.updateRunningStats(monitor, statusResponse);
if (!statsOk) {
this.logger.warn({
service: SERVICE_NAME,
method: "updateMonitorStatus",
message: `Stats update failed for monitor ${monitor.id}`,
});
}
2025-05-19 12:28:18 -07:00
2026-01-18 23:38:40 +00:00
monitor.statusWindow = monitor.statusWindow || [];
monitor.statusWindow.push(status);
2026-01-18 23:38:40 +00:00
while (monitor.statusWindow.length > monitor.statusWindowSize) {
monitor.statusWindow.shift();
}
2026-01-27 18:26:31 +00:00
const checkSnapshot: CheckSnapshot = {
id: check.id,
status: check.status,
responseTime: check.responseTime,
timings: check.timings,
statusCode: check.statusCode,
message: check.message,
cpu: check.cpu,
memory: check.memory,
disk: check.disk,
host: check.host,
errors: check.errors,
capture: check.capture,
net: check.net,
accessibility: check.accessibility,
bestPractices: check.bestPractices,
seo: check.seo,
performance: check.performance,
audits: check.audits,
createdAt: check.createdAt,
};
monitor.recentChecks = monitor.recentChecks || [];
monitor.recentChecks.push(checkSnapshot);
const maxRecentChecks = 25;
while (monitor.recentChecks.length > maxRecentChecks) {
monitor.recentChecks.shift();
}
2026-01-18 23:38:40 +00:00
const prevStatus = monitor.status;
2026-02-12 18:30:44 +00:00
let newStatus: MonitorStatus = status === true ? "up" : "down";
let statusChanged = false;
// Return early if not enough data points
if (monitor.statusWindow.length < monitor.statusWindowSize) {
2026-02-12 18:30:44 +00:00
monitor.status = newStatus;
2026-01-15 18:33:03 +00:00
const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
2025-04-20 11:29:53 -07:00
return {
2026-01-13 20:20:00 +00:00
monitor: updated,
2025-04-20 11:29:53 -07:00
statusChanged: false,
prevStatus,
2025-04-20 11:29:53 -07:00
code,
timestamp: Date.now(),
2025-04-20 11:29:53 -07:00
};
}
2025-05-19 12:28:18 -07:00
2026-04-14 12:07:51 -04:00
// With a full window, a single raw check must not change monitor.status; only the sliding-window threshold can trigger a transition.
newStatus = monitor.status;
// Check if threshold has been met
const failures = monitor.statusWindow.filter((s) => s === false).length;
const failureRate = (failures / monitor.statusWindow.length) * 100;
// If threshold has been met and the monitor is not already down, mark down:
if (failureRate >= monitor.statusWindowThreshold && monitor.status !== "down") {
newStatus = "down";
statusChanged = true;
}
// If the failure rate is below the threshold and the monitor is down, recover:
else if (failureRate < monitor.statusWindowThreshold && monitor.status === "down") {
newStatus = "up";
statusChanged = true;
}
2026-02-14 22:14:51 +00:00
// Evaluate hardware threshold breaches (only for hardware monitors)
let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined;
if (monitor.type === "hardware" && statusResponse.payload) {
const payload = statusResponse.payload as HardwareStatusPayload;
2026-04-08 10:31:24 -07:00
const metrics = payload.data;
2026-02-14 22:14:51 +00:00
if (metrics) {
// Evaluate threshold breaches
const cpuUsage = metrics.cpu?.usage_percent ?? -1;
const cpuBreach = cpuUsage !== -1 && cpuUsage > monitor.cpuAlertThreshold / 100;
const memoryUsage = metrics.memory?.usage_percent ?? -1;
const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100;
2026-04-08 10:31:24 -07:00
const diskBreach = metrics.disk
? metrics.disk.some(
(d: CheckDiskInfo) => d != null && typeof d.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100
)
: false;
2026-02-14 22:14:51 +00:00
const temps = metrics.cpu?.temperature ?? [];
const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold);
thresholdBreaches = {
cpu: cpuBreach,
memory: memoryBreach,
disk: diskBreach,
temp: tempBreach,
};
// Update counters: decrement if breached, reset to 5 if not breached
if (cpuBreach) {
monitor.cpuAlertCounter = Math.max(0, monitor.cpuAlertCounter - 1);
} else {
monitor.cpuAlertCounter = 5;
}
if (memoryBreach) {
monitor.memoryAlertCounter = Math.max(0, monitor.memoryAlertCounter - 1);
} else {
monitor.memoryAlertCounter = 5;
}
if (diskBreach) {
monitor.diskAlertCounter = Math.max(0, monitor.diskAlertCounter - 1);
} else {
monitor.diskAlertCounter = 5;
}
if (tempBreach) {
monitor.tempAlertCounter = Math.max(0, monitor.tempAlertCounter - 1);
} else {
monitor.tempAlertCounter = 5;
}
2026-02-14 22:52:14 +00:00
// Check if any counter has reached zero (initial breach)
const anyCounterZero =
monitor.cpuAlertCounter === 0 || monitor.memoryAlertCounter === 0 || monitor.diskAlertCounter === 0 || monitor.tempAlertCounter === 0;
const anyThresholdBreached = cpuBreach || memoryBreach || diskBreach || tempBreach;
const allThresholdsNormal = !cpuBreach && !memoryBreach && !diskBreach && !tempBreach;
// Update monitor status based on threshold breach state
if (newStatus !== "down") {
// Don't override "down" status - service unreachable takes precedence
// Check current monitor status, not newStatus for comparison
if (anyCounterZero && anyThresholdBreached && monitor.status !== "breached") {
// Initial breach: counter hit zero, change status to breached
newStatus = "breached";
statusChanged = true;
} else if (anyCounterZero && anyThresholdBreached && monitor.status === "breached") {
// Already breached, keep status but don't mark as changed
newStatus = "breached";
// statusChanged remains false
} else if (allThresholdsNormal && monitor.status === "breached") {
// All thresholds returned to normal, recover from breached state
newStatus = "up";
statusChanged = true;
}
}
2026-02-14 22:14:51 +00:00
}
}
2026-02-14 22:52:14 +00:00
// Apply the final status
monitor.status = newStatus;
2026-01-15 18:33:03 +00:00
const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
2025-05-19 12:28:18 -07:00
2025-04-20 11:29:53 -07:00
return {
2026-01-13 20:20:00 +00:00
monitor: updated,
statusChanged,
2025-04-20 11:29:53 -07:00
prevStatus,
code,
timestamp: new Date().getTime(),
2026-02-14 22:14:51 +00:00
thresholdBreaches,
2025-04-20 11:29:53 -07:00
};
2026-03-03 23:01:35 +00:00
} catch (error: unknown) {
throw new AppError({
2026-03-12 11:12:46 -07:00
message: `Failed to update monitor with id ${check.metadata.monitorId} with status: ${error instanceof Error ? error.message : "Unknown error"}`,
2026-03-03 23:01:35 +00:00
service: SERVICE_NAME,
method: "updateMonitorStatus",
});
2025-04-20 11:29:53 -07:00
}
};
}