refactor out rebuilding of status response
This commit is contained in:
1 parent
849ea26cf9
commit
0d75629dcb
18 files changed
+436
-786
No files matched your search
@@ -13,7 +13,6 @@ const SERVICE_NAME = "checkService";
|
||||
export interface ICheckService {
|
||||
createChecks(checks: Check[]): Promise<Check[]>;
|
||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||
getChecksByMonitor(params: {
|
||||
monitorId: string;
|
||||
teamId: string;
|
||||
@@ -135,32 +134,6 @@ export class CheckService implements ICheckService {
|
||||
return check;
|
||||
};
|
||||
|
||||
private toPayload = (check: Check): HardwareStatusPayload | DockerStatusPayload | undefined => {
|
||||
switch (check.metadata.type) {
|
||||
case "hardware":
|
||||
return { data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload;
|
||||
case "docker":
|
||||
return check.containers ? { containers: check.containers, summary: check.containerSummary } : undefined;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
toStatusResponse = (check: Check) => {
|
||||
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
|
||||
monitorId: check.metadata.monitorId,
|
||||
teamId: check.metadata.teamId,
|
||||
type: check.metadata.type,
|
||||
status: check.status,
|
||||
code: check.statusCode,
|
||||
message: check.message,
|
||||
responseTime: check.responseTime,
|
||||
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
|
||||
payload: this.toPayload(check),
|
||||
};
|
||||
return statusResponse;
|
||||
};
|
||||
|
||||
getChecksByMonitor = async ({
|
||||
monitorId,
|
||||
teamId,
|
||||
|
||||
@@ -45,12 +45,14 @@ const mapMemory = (memory?: CheckMemoryInfo): SnapshotMemoryInfo | undefined =>
|
||||
};
|
||||
|
||||
const mapDisk = (disk?: CheckDiskInfo[]): SnapshotDiskInfo[] | undefined =>
|
||||
disk?.map((entry) => ({
|
||||
device: entry.device,
|
||||
total_bytes: entry.total_bytes,
|
||||
used_bytes: entry.used_bytes,
|
||||
usage_percent: entry.usage_percent,
|
||||
}));
|
||||
disk
|
||||
?.filter((entry) => entry != null)
|
||||
.map((entry) => ({
|
||||
device: entry.device,
|
||||
total_bytes: entry.total_bytes,
|
||||
used_bytes: entry.used_bytes,
|
||||
usage_percent: entry.usage_percent,
|
||||
}));
|
||||
|
||||
const mapHost = (host?: CheckHostInfo): SnapshotHostInfo | undefined =>
|
||||
host && {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
const SERVICE_NAME = "incidentService";
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { Check } from "@/domain/checks/check.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { getDateForRange } from "@/utils/dataUtils.js";
|
||||
import type { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
|
||||
@@ -15,12 +15,7 @@ import { DateRange } from "@/types/query.js";
|
||||
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
||||
|
||||
export interface IIncidentService {
|
||||
handleIncident(
|
||||
monitor: Monitor,
|
||||
code: number,
|
||||
decision: MonitorActionDecision,
|
||||
monitorStatusResponse?: MonitorStatusResponse
|
||||
): Promise<Incident | null>;
|
||||
handleIncident(monitor: Monitor, code: number, decision: MonitorActionDecision, check: Check): Promise<Incident | null>;
|
||||
resolveIncident(incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string): Promise<Incident>;
|
||||
getIncidentsByTeam(
|
||||
teamId: string,
|
||||
@@ -59,12 +54,7 @@ export class IncidentService implements IIncidentService {
|
||||
this.notificationMessageBuilder = notificationMessageBuilder;
|
||||
}
|
||||
|
||||
handleIncident = async (
|
||||
monitor: Monitor,
|
||||
code: number,
|
||||
decision: MonitorActionDecision,
|
||||
monitorStatusResponse?: MonitorStatusResponse
|
||||
): Promise<Incident | null> => {
|
||||
handleIncident = async (monitor: Monitor, code: number, decision: MonitorActionDecision, check: Check): Promise<Incident | null> => {
|
||||
if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) {
|
||||
return null;
|
||||
}
|
||||
@@ -81,7 +71,7 @@ export class IncidentService implements IIncidentService {
|
||||
// For threshold breaches, use 9999 status code and build descriptive message
|
||||
if (decision.incidentReason === "threshold_breach") {
|
||||
statusCode = 9999;
|
||||
message = this.buildThresholdBreachMessage(monitor, monitorStatusResponse);
|
||||
message = this.buildThresholdBreachMessage(monitor, check);
|
||||
}
|
||||
|
||||
const incident = {
|
||||
@@ -106,18 +96,14 @@ export class IncidentService implements IIncidentService {
|
||||
return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
|
||||
};
|
||||
|
||||
private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string {
|
||||
private buildThresholdBreachMessage(monitor: Monitor, check: Check): string {
|
||||
if (monitor.type === "docker") {
|
||||
const containers = (monitorStatusResponse?.payload as DockerStatusPayload | undefined)?.containers ?? [];
|
||||
const containers = check.containers ?? [];
|
||||
const breaches = findContainerBreaches(monitor, containers);
|
||||
return breaches.length > 0 ? breaches.map(describeContainerBreach).join(", ") : "Container alert";
|
||||
}
|
||||
|
||||
if (!monitorStatusResponse) {
|
||||
return "Threshold breach detected";
|
||||
}
|
||||
|
||||
const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse);
|
||||
const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, check);
|
||||
|
||||
if (breaches.length === 0) {
|
||||
return "Threshold breach detected";
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { DockerStatusPayload, HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
import type {
|
||||
NotificationMessage,
|
||||
@@ -9,15 +8,11 @@ import type {
|
||||
NotificationContent,
|
||||
} from "@/domain/notifications/notification.type.js";
|
||||
import { describeContainerBreach, findContainerBreaches } from "@/domain/docker/docker-alert.js";
|
||||
import type { Check } from "@/domain/checks/check.type.js";
|
||||
|
||||
export interface INotificationMessageBuilder {
|
||||
buildMessage(
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
decision: MonitorActionDecision,
|
||||
clientHost: string
|
||||
): NotificationMessage;
|
||||
extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[];
|
||||
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage;
|
||||
extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[];
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "NotificationMessageBuilder";
|
||||
@@ -25,15 +20,10 @@ const SERVICE_NAME = "NotificationMessageBuilder";
|
||||
export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
buildMessage(
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
decision: MonitorActionDecision,
|
||||
clientHost: string
|
||||
): NotificationMessage {
|
||||
buildMessage(monitor: Monitor, check: Check, decision: MonitorActionDecision, clientHost: string): NotificationMessage {
|
||||
const type = this.determineNotificationType(decision, monitor);
|
||||
const severity = this.determineSeverity(type);
|
||||
const content = this.buildContent(type, monitor, monitorStatusResponse);
|
||||
const content = this.buildContent(type, monitor, check);
|
||||
|
||||
return {
|
||||
type,
|
||||
@@ -92,18 +82,18 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
private buildContent(type: NotificationType, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent {
|
||||
private buildContent(type: NotificationType, monitor: Monitor, check: Check): NotificationContent {
|
||||
switch (type) {
|
||||
case "monitor_down":
|
||||
return this.buildMonitorDownContent(monitor, monitorStatusResponse);
|
||||
return this.buildMonitorDownContent(monitor, check);
|
||||
case "monitor_up":
|
||||
return this.buildMonitorUpContent(monitor);
|
||||
case "threshold_breach":
|
||||
return this.buildThresholdBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<HardwareStatusPayload>);
|
||||
return this.buildThresholdBreachContent(monitor, check);
|
||||
case "threshold_resolved":
|
||||
return this.buildThresholdResolvedContent(monitor);
|
||||
case "container_breach":
|
||||
return this.buildContainerBreachContent(monitor, monitorStatusResponse as MonitorStatusResponse<DockerStatusPayload>);
|
||||
return this.buildContainerBreachContent(monitor, check);
|
||||
case "container_resolved":
|
||||
return this.buildContainersRecoveredContent(monitor);
|
||||
default:
|
||||
@@ -111,19 +101,19 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
private buildMonitorDownContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): NotificationContent {
|
||||
private buildMonitorDownContent(monitor: Monitor, check: Check): NotificationContent {
|
||||
const title = `Monitor Down: ${monitor.name}`;
|
||||
const summary = `Monitor "${monitor.name}" is currently down and unreachable.`;
|
||||
const details = [`URL: ${monitor.url}`, `Status: Down`, `Type: ${monitor.type}`];
|
||||
|
||||
// Add response code if available
|
||||
if (monitorStatusResponse.code) {
|
||||
details.push(`Response Code: ${monitorStatusResponse.code}`);
|
||||
if (check.statusCode) {
|
||||
details.push(`Response Code: ${check.statusCode}`);
|
||||
}
|
||||
|
||||
// Add error message if available
|
||||
if (monitorStatusResponse.message) {
|
||||
details.push(`Error: ${monitorStatusResponse.message}`);
|
||||
if (check.message) {
|
||||
details.push(`Error: ${check.message}`);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -147,12 +137,12 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
};
|
||||
}
|
||||
|
||||
private buildThresholdBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<HardwareStatusPayload>): NotificationContent {
|
||||
private buildThresholdBreachContent(monitor: Monitor, check: Check): NotificationContent {
|
||||
const title = `Threshold Exceeded: ${monitor.name}`;
|
||||
const summary = `Monitor "${monitor.name}" has exceeded one or more thresholds.`;
|
||||
const details = [`URL: ${monitor.url}`, `Status: Threshold exceeded`, `Type: ${monitor.type}`];
|
||||
|
||||
const thresholds = this.extractThresholdBreaches(monitor, monitorStatusResponse);
|
||||
const thresholds = this.extractThresholdBreaches(monitor, check);
|
||||
|
||||
return {
|
||||
title,
|
||||
@@ -176,9 +166,8 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
};
|
||||
}
|
||||
|
||||
private buildContainerBreachContent(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<DockerStatusPayload>): NotificationContent {
|
||||
const payload = monitorStatusResponse.payload;
|
||||
const containers = payload && typeof payload !== "string" ? payload.containers : [];
|
||||
private buildContainerBreachContent(monitor: Monitor, check: Check): NotificationContent {
|
||||
const containers = check.containers ?? [];
|
||||
const breaches = findContainerBreaches(monitor, containers);
|
||||
const title = `Container Alert: ${monitor.name}`;
|
||||
const summary = `${breaches.length} container(s) on "${monitor.name}" need attention.`;
|
||||
@@ -202,25 +191,17 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
};
|
||||
}
|
||||
|
||||
public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse<HardwareStatusPayload>): ThresholdBreach[] {
|
||||
public extractThresholdBreaches(monitor: Monitor, check: Check): ThresholdBreach[] {
|
||||
const breaches: ThresholdBreach[] = [];
|
||||
|
||||
// Check if this is a hardware monitor with threshold data
|
||||
if (monitor.type !== "hardware" || !monitorStatusResponse.payload || typeof monitorStatusResponse.payload === "string") {
|
||||
return breaches;
|
||||
}
|
||||
|
||||
// Cast to HardwareStatusPayload type
|
||||
const payload = monitorStatusResponse.payload;
|
||||
const hardware = payload.data;
|
||||
|
||||
if (!hardware) {
|
||||
if (monitor.type !== "hardware") {
|
||||
return breaches;
|
||||
}
|
||||
|
||||
// Note: usage_percent values in hardware payload are decimals (0-1)
|
||||
if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && hardware.cpu?.usage_percent !== undefined) {
|
||||
const cpuUsageDecimal = hardware.cpu.usage_percent;
|
||||
if (monitor.cpuAlertThreshold !== undefined && monitor.cpuAlertThreshold !== null && check.cpu?.usage_percent !== undefined) {
|
||||
const cpuUsageDecimal = check.cpu.usage_percent;
|
||||
const cpuPercent = cpuUsageDecimal * 100;
|
||||
const threshold = monitor.cpuAlertThreshold;
|
||||
if (cpuPercent > threshold) {
|
||||
@@ -235,8 +216,8 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
}
|
||||
|
||||
// Memory threshold breach
|
||||
if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && hardware.memory?.usage_percent !== undefined) {
|
||||
const memoryUsageDecimal = hardware.memory.usage_percent;
|
||||
if (monitor.memoryAlertThreshold !== undefined && monitor.memoryAlertThreshold !== null && check.memory?.usage_percent !== undefined) {
|
||||
const memoryUsageDecimal = check.memory.usage_percent;
|
||||
const memoryPercent = memoryUsageDecimal * 100;
|
||||
const threshold = monitor.memoryAlertThreshold;
|
||||
if (memoryPercent > threshold) {
|
||||
@@ -251,10 +232,10 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
}
|
||||
|
||||
// Disk threshold breach
|
||||
if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(hardware.disk)) {
|
||||
if (monitor.diskAlertThreshold !== undefined && monitor.diskAlertThreshold !== null && Array.isArray(check.disk)) {
|
||||
// Find the highest disk usage
|
||||
let maxDiskUsageDecimal = 0;
|
||||
for (const disk of hardware.disk) {
|
||||
for (const disk of check.disk) {
|
||||
if (disk.usage_percent !== undefined && disk.usage_percent > maxDiskUsageDecimal) {
|
||||
maxDiskUsageDecimal = disk.usage_percent;
|
||||
}
|
||||
@@ -273,9 +254,9 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
}
|
||||
|
||||
// Temperature threshold breach
|
||||
if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && hardware.cpu?.temperature) {
|
||||
if (monitor.tempAlertThreshold !== undefined && monitor.tempAlertThreshold !== null && check.cpu?.temperature) {
|
||||
// Temperature is an array in cpu.temperature
|
||||
const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature];
|
||||
const temps = Array.isArray(check.cpu.temperature) ? check.cpu.temperature : [check.cpu.temperature];
|
||||
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
|
||||
const threshold = monitor.tempAlertThreshold;
|
||||
if (maxTemp > threshold) {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { Notification } from "@/domain/notifications/notification.type.js";
|
||||
import type { MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { NotificationMessage } from "@/domain/notifications/notification.type.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
|
||||
@@ -10,6 +9,7 @@ import type { ISettingsService } from "@/domain/app-settings/app-settings.servic
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
|
||||
import type { NotificationChannel } from "@/domain/notifications/notification.type.js";
|
||||
import type { Check } from "@/domain/checks/check.type.js";
|
||||
|
||||
export type NotificationProviderRegistry = Record<NotificationChannel, INotificationProvider>;
|
||||
|
||||
@@ -19,7 +19,7 @@ export interface INotificationsService {
|
||||
findNotificationsByTeamId: (teamId: string) => Promise<Notification[]>;
|
||||
updateById(id: string, teamId: string, updateData: Partial<Notification>): Promise<Notification>;
|
||||
deleteById: (id: string, teamId: string) => Promise<Notification>;
|
||||
handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise<boolean>;
|
||||
handleNotifications: (monitor: Monitor, check: Check, decision: MonitorActionDecision) => Promise<boolean>;
|
||||
|
||||
sendTestNotification: (notification: Partial<Notification>) => Promise<boolean>;
|
||||
testAllNotifications: (notificationIds: string[]) => Promise<boolean>;
|
||||
@@ -63,7 +63,7 @@ export class NotificationsService implements INotificationsService {
|
||||
private send = async (
|
||||
notification: Notification,
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
check: Check,
|
||||
decision: MonitorActionDecision,
|
||||
notificationMessage: NotificationMessage | undefined
|
||||
): Promise<boolean> => {
|
||||
@@ -89,16 +89,16 @@ export class NotificationsService implements INotificationsService {
|
||||
return await provider.sendMessage(notification, notificationMessage);
|
||||
};
|
||||
|
||||
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
|
||||
private sendNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => {
|
||||
const notificationIds = monitor.notifications ?? [];
|
||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||
|
||||
// Build notification message once for all notifications
|
||||
const settings = this.settingsService.getSettings();
|
||||
const clientHost = settings.clientHost || "Host not defined";
|
||||
const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, monitorStatusResponse, decision, clientHost);
|
||||
const notificationMessage = this.notificationMessageBuilder.buildMessage(monitor, check, decision, clientHost);
|
||||
|
||||
const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage));
|
||||
const tasks = notifications.map((notification) => this.send(notification, monitor, check, decision, notificationMessage));
|
||||
|
||||
const outcomes = await Promise.all(tasks);
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
@@ -114,13 +114,13 @@ export class NotificationsService implements INotificationsService {
|
||||
return succeeded === notifications.length;
|
||||
};
|
||||
|
||||
handleNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => {
|
||||
handleNotifications = async (monitor: Monitor, check: Check, decision: MonitorActionDecision) => {
|
||||
if (!decision.shouldSendNotification) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Send notifications based on decision
|
||||
return await this.sendNotifications(monitor, monitorStatusResponse, decision);
|
||||
return await this.sendNotifications(monitor, check, decision);
|
||||
};
|
||||
|
||||
sendTestNotification = async (notification: Partial<Notification>) => {
|
||||
|
||||
@@ -2,18 +2,7 @@ import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.re
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js";
|
||||
import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js";
|
||||
import type {
|
||||
DockerStatusPayload,
|
||||
GameStatusPayload,
|
||||
GrpcStatusPayload,
|
||||
HardwareStatusPayload,
|
||||
HttpStatusPayload,
|
||||
MonitorStatusResponse,
|
||||
PageSpeedStatusPayload,
|
||||
PingStatusPayload,
|
||||
PortStatusPayload,
|
||||
StatusChangeResult,
|
||||
} from "@/types/network.js";
|
||||
import type { StatusChangeResult } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import type { HardwareStatusMetrics } from "@/types/network.js";
|
||||
@@ -29,22 +18,8 @@ type HardwareBreaches = Record<HardwareMetricKey, boolean>;
|
||||
type HardwareCounters = Record<HardwareMetricKey, number>;
|
||||
|
||||
export interface IStatusService {
|
||||
updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse): Promise<boolean>;
|
||||
updateMonitorStatus(
|
||||
statusResponse: MonitorStatusResponse<
|
||||
| PingStatusPayload
|
||||
| HttpStatusPayload
|
||||
| PageSpeedStatusPayload
|
||||
| HardwareStatusPayload
|
||||
| DockerStatusPayload
|
||||
| PortStatusPayload
|
||||
| GameStatusPayload
|
||||
| GrpcStatusPayload
|
||||
| undefined
|
||||
>,
|
||||
check: Check,
|
||||
monitor: Monitor
|
||||
): Promise<StatusChangeResult>;
|
||||
updateRunningStats(check: Check, monitor: Monitor): Promise<boolean>;
|
||||
updateMonitorStatus(check: Check, monitor: Monitor): Promise<StatusChangeResult>;
|
||||
}
|
||||
|
||||
export class StatusService implements IStatusService {
|
||||
@@ -59,11 +34,11 @@ export class StatusService implements IStatusService {
|
||||
this.monitorStatsRepository = monitorStatsRepository;
|
||||
}
|
||||
|
||||
async updateRunningStats(monitor: Monitor, networkResponse: MonitorStatusResponse) {
|
||||
async updateRunningStats(check: Check, monitor: Monitor) {
|
||||
try {
|
||||
await this.monitorStatsRepository.updateByMonitorId(monitor.id, {
|
||||
status: networkResponse.status === true,
|
||||
responseTime: networkResponse.responseTime ?? 0,
|
||||
status: check.status === true,
|
||||
responseTime: check.responseTime ?? 0,
|
||||
now: Date.now(),
|
||||
});
|
||||
return true;
|
||||
@@ -78,8 +53,8 @@ export class StatusService implements IStatusService {
|
||||
}
|
||||
}
|
||||
|
||||
private tryUpdateRunningStats = async (monitor: Monitor, statusResponse: MonitorStatusResponse) => {
|
||||
const statsOk = await this.updateRunningStats(monitor, statusResponse);
|
||||
private tryUpdateRunningStats = async (check: Check, monitor: Monitor) => {
|
||||
const statsOk = await this.updateRunningStats(check, monitor);
|
||||
if (!statsOk) {
|
||||
this.logger.warn({
|
||||
service: SERVICE_NAME,
|
||||
@@ -180,26 +155,12 @@ export class StatusService implements IStatusService {
|
||||
return { nextStatus, transitioned };
|
||||
};
|
||||
|
||||
updateMonitorStatus = async (
|
||||
statusResponse: MonitorStatusResponse<
|
||||
| PingStatusPayload
|
||||
| HttpStatusPayload
|
||||
| PageSpeedStatusPayload
|
||||
| HardwareStatusPayload
|
||||
| DockerStatusPayload
|
||||
| PortStatusPayload
|
||||
| GameStatusPayload
|
||||
| GrpcStatusPayload
|
||||
| undefined
|
||||
>,
|
||||
check: Check,
|
||||
monitor: Monitor
|
||||
): Promise<StatusChangeResult> => {
|
||||
updateMonitorStatus = async (check: Check, monitor: Monitor): Promise<StatusChangeResult> => {
|
||||
try {
|
||||
const { status, code } = statusResponse;
|
||||
const { status, statusCode } = check;
|
||||
|
||||
// Update running stats
|
||||
await this.tryUpdateRunningStats(monitor, statusResponse);
|
||||
await this.tryUpdateRunningStats(check, monitor);
|
||||
|
||||
const prevStatus = monitor.status;
|
||||
const checkSnapshot = toCheckSnapshot(check);
|
||||
@@ -240,7 +201,7 @@ export class StatusService implements IStatusService {
|
||||
monitor: updated,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
code,
|
||||
code: statusCode,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
@@ -253,14 +214,13 @@ export class StatusService implements IStatusService {
|
||||
statusChanged = true;
|
||||
}
|
||||
|
||||
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
|
||||
// Evaluate hardware threshold breaches
|
||||
let thresholdBreaches: HardwareBreaches | undefined;
|
||||
const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined;
|
||||
if (monitor.type === "hardware" && hardwarePayload?.data) {
|
||||
if (monitor.type === "hardware") {
|
||||
const hardware = this.computeHardwareStatus({
|
||||
currentStatus: newStatus,
|
||||
reachabilityDown: newStatus === "down",
|
||||
metrics: hardwarePayload.data,
|
||||
metrics: check,
|
||||
thresholds: {
|
||||
cpu: monitor.cpuAlertThreshold,
|
||||
memory: monitor.memoryAlertThreshold,
|
||||
@@ -286,9 +246,8 @@ export class StatusService implements IStatusService {
|
||||
}
|
||||
}
|
||||
|
||||
const dockerPayload = statusResponse.payload as DockerStatusPayload | undefined;
|
||||
if (monitor.type === "docker" && dockerPayload?.containers) {
|
||||
const breaches = findContainerBreaches(monitor, dockerPayload.containers);
|
||||
if (monitor.type === "docker" && check.containers) {
|
||||
const breaches = findContainerBreaches(monitor, check.containers);
|
||||
const docker = this.computeDockerStatus({
|
||||
currentStatus: newStatus,
|
||||
reachabilityDown: newStatus === "down",
|
||||
@@ -317,7 +276,7 @@ export class StatusService implements IStatusService {
|
||||
monitor: updated,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
code,
|
||||
code: statusCode,
|
||||
timestamp: new Date().getTime(),
|
||||
thresholdBreaches,
|
||||
};
|
||||
|
||||
@@ -8,6 +8,6 @@ export class IncidentReactor implements IMonitorReactor {
|
||||
constructor(private incidentService: IIncidentService) {}
|
||||
|
||||
react = async (evaluation: MonitorEvaluation) => {
|
||||
await this.incidentService.handleIncident(evaluation.monitor, evaluation.statusChange.code, evaluation.decision, evaluation.status);
|
||||
await this.incidentService.handleIncident(evaluation.monitor, evaluation.statusChange.code, evaluation.decision, evaluation.check);
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,6 @@ export class NotificationReactor implements IMonitorReactor {
|
||||
|
||||
react = async (evaluation: MonitorEvaluation) => {
|
||||
if (!evaluation.decision.shouldSendNotification) return;
|
||||
await this.notificationService.handleNotifications(evaluation.monitor, evaluation.status, evaluation.decision);
|
||||
await this.notificationService.handleNotifications(evaluation.monitor, evaluation.check, evaluation.decision);
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { Job, JobType } from "@/domain/jobs/job.type.js";
|
||||
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { StatusChangeResult } from "@/types/network.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
import type { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
|
||||
@@ -23,7 +23,6 @@ export interface MonitorActionDecision {
|
||||
|
||||
export type MonitorEvaluation = {
|
||||
monitor: Monitor;
|
||||
status: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>; // raw result from networkService.requestStatus
|
||||
check: Check;
|
||||
statusChange: StatusChangeResult; // from statusService.updateMonitorStatus
|
||||
decision: MonitorActionDecision;
|
||||
|
||||
@@ -240,8 +240,7 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
|
||||
let current = monitor;
|
||||
for (const check of checks) {
|
||||
const status = this.deps.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.evaluateCheck(status, check, current);
|
||||
const evaluation = await this.evaluateCheck(check, current);
|
||||
await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
if (!leaseHeld) {
|
||||
@@ -272,7 +271,7 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
// A check taken while the instance had no outbound connectivity says nothing about the target.
|
||||
// It is left out of the status window, running stats and monitor status entirely, so that no incident
|
||||
// opens for it and no spurious "resolved"/"up" fires once egress returns.
|
||||
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
|
||||
private skipDegradedEgressCheck = (check: Check, monitor: Monitor): MonitorEvaluation => {
|
||||
this.deps.logger.debug({
|
||||
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
|
||||
service: SERVICE_NAME,
|
||||
@@ -287,22 +286,21 @@ export class WorkerPipeline implements IWorkerPipeline {
|
||||
};
|
||||
return {
|
||||
monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: check.statusCode, timestamp: Date.now() },
|
||||
decision,
|
||||
};
|
||||
};
|
||||
|
||||
evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
|
||||
evaluateCheck = async (check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
|
||||
if (check.egressStatus === "degraded") {
|
||||
return this.skipDegradedEgressCheck(status, check, monitor);
|
||||
return this.skipDegradedEgressCheck(check, monitor);
|
||||
}
|
||||
|
||||
const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor);
|
||||
const statusChange = await this.deps.statusService.updateMonitorStatus(check, monitor);
|
||||
|
||||
const decision = this.decide(statusChange);
|
||||
return { monitor: statusChange.monitor, status, check, statusChange, decision };
|
||||
return { monitor: statusChange.monitor, check, statusChange, decision };
|
||||
};
|
||||
|
||||
private decide = (statusChange: StatusChangeResult): MonitorActionDecision => {
|
||||
|
||||
Reference in new issue
Block a user