2026-02-12 22:37:40 +00:00
import { IChecksRepository , IMonitorsRepository , IMonitorStatsRepository } from "@/repositories/index.js" ;
2026-01-16 18:11:04 +00:00
import type {
Monitor ,
2026-02-12 18:30:44 +00:00
MonitorStatus ,
2026-01-16 18:11:04 +00:00
MonitorStatusResponse ,
StatusChangeResult ,
Check ,
HardwareStatusPayload ,
PageSpeedStatusPayload ,
2026-03-03 19:34:28 +00:00
PingStatusPayload ,
HttpStatusPayload ,
DockerStatusPayload ,
PortStatusPayload ,
GameStatusPayload ,
GrpcStatusPayload ,
2026-01-27 18:26:31 +00:00
CheckSnapshot ,
2026-03-03 23:01:35 +00:00
CheckDiskInfo ,
2026-01-16 18:11:04 +00:00
} from "@/types/index.js" ;
2026-03-03 23:01:35 +00:00
import { AppError } from "@/utils/AppError.js" ;
2026-02-12 22:37:40 +00:00
import { ILogger } from "@/utils/logger.js" ;
2026-03-03 23:01:35 +00:00
import { IBufferService } from "./bufferService.js" ;
2026-04-14 10:28:43 -07:00
import type { HardwareStatusMetrics } from "@/types/network.js" ;
2025-04-20 11:29:53 -07:00
const SERVICE_NAME = "StatusService" ;
2026-04-14 10:28:43 -07:00
const MAX_RECENT_CHECKS = 25 ;
const HARDWARE_ALERT_COUNTER_START = 5 ;
const HARDWARE_METRIC_KEYS = [ "cpu" , "memory" , "disk" , "temp" ] as const ;
type HardwareMetricKey = ( typeof HARDWARE_METRIC_KEYS )[ number ];
type HardwareBreaches = Record < HardwareMetricKey , boolean >;
type HardwareCounters = Record < HardwareMetricKey , number >;
2025-04-20 11:29:53 -07:00
2026-01-18 23:38:40 +00:00
export interface IStatusService {
2026-03-03 23:01:35 +00:00
updateRunningStats ( monitor : Monitor , networkResponse : MonitorStatusResponse ) : Promise < boolean >;
2026-01-18 23:38:40 +00:00
updateMonitorStatus (
2026-03-03 19:34:28 +00:00
statusResponse : MonitorStatusResponse <
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
> ,
2026-01-18 23:38:40 +00:00
check : Check
) : Promise < StatusChangeResult >;
}
export class StatusService implements IStatusService {
2025-04-20 11:29:53 -07:00
static SERVICE_NAME = SERVICE_NAME ;
2026-03-03 23:01:35 +00:00
private logger : ILogger ;
private buffer : IBufferService ;
2026-01-15 22:33:25 +00:00
private monitorsRepository : IMonitorsRepository ;
2026-02-12 22:37:40 +00:00
private monitorStatsRepository : IMonitorStatsRepository ;
private checksRepository : IChecksRepository ;
2026-01-15 22:33:25 +00:00
2026-02-12 22:37:40 +00:00
constructor (
logger : ILogger ,
2026-03-03 23:01:35 +00:00
buffer : IBufferService ,
2026-02-12 22:37:40 +00:00
monitorsRepository : IMonitorsRepository ,
monitorStatsRepository : IMonitorStatsRepository ,
checksRepository : IChecksRepository
) {
2025-04-20 11:29:53 -07:00
this . logger = logger ;
this . buffer = buffer ;
2026-01-13 20:20:00 +00:00
this . monitorsRepository = monitorsRepository ;
2026-02-12 22:37:40 +00:00
this . monitorStatsRepository = monitorStatsRepository ;
this . checksRepository = checksRepository ;
2025-07-25 15:15:14 -07:00
}
get serviceName() {
return StatusService . SERVICE_NAME ;
2025-04-20 11:29:53 -07:00
}
2026-03-03 23:01:35 +00:00
async updateRunningStats ( monitor : Monitor , networkResponse : MonitorStatusResponse ) {
2025-04-20 11:29:53 -07:00
try {
2026-04-13 11:39:29 -07:00
await this . monitorStatsRepository . updateByMonitorId ( monitor . id , {
status : networkResponse.status === true ,
responseTime : networkResponse.responseTime ?? 0 ,
now : Date.now (),
});
2025-04-20 11:29:53 -07:00
return true ;
2026-03-03 23:01:35 +00:00
} catch ( error : unknown ) {
2025-04-20 11:29:53 -07:00
this . logger . error ({
2026-01-15 22:33:25 +00:00
service : SERVICE_NAME ,
2026-03-03 23:01:35 +00:00
message : error instanceof Error ? error . message : "Unknown error" ,
2025-04-20 11:29:53 -07:00
method : "updateRunningStats" ,
2026-03-03 23:01:35 +00:00
stack : error instanceof Error ? error.stack : undefined ,
2025-04-20 11:29:53 -07:00
});
return false ;
}
}
2026-04-14 10:28:43 -07:00
private tryUpdateRunningStats = async ( monitor : Monitor , statusResponse : MonitorStatusResponse ) => {
const statsOk = await this . updateRunningStats ( monitor , statusResponse );
if ( ! statsOk ) {
this . logger . warn ({
service : SERVICE_NAME ,
method : "updateMonitorStatus" ,
message : `Stats update failed for monitor ${ monitor . id } ` ,
});
}
};
private appendToWindow = ( monitor : Monitor , status : boolean ) => {
monitor . statusWindow = monitor . statusWindow || [];
monitor . statusWindow . push ( status );
while ( monitor . statusWindow . length > monitor . statusWindowSize ) {
monitor . statusWindow . shift ();
}
};
private toCheckSnapshot = ( check : Check ) : CheckSnapshot => {
return {
id : check.id ,
status : check.status ,
responseTime : check.responseTime ,
timings : check.timings ,
statusCode : check.statusCode ,
message : check.message ,
cpu : check.cpu ,
memory : check.memory ,
disk : check.disk ,
host : check.host ,
errors : check.errors ,
capture : check.capture ,
net : check.net ,
accessibility : check.accessibility ,
bestPractices : check.bestPractices ,
seo : check.seo ,
performance : check.performance ,
audits : check.audits ,
createdAt : check.createdAt ,
};
};
private appendToRecentChecks = ( monitor : Monitor , check : Check ) => {
monitor . recentChecks = monitor . recentChecks || [];
const checkSnapshot = this . toCheckSnapshot ( check );
monitor . recentChecks . push ( checkSnapshot );
while ( monitor . recentChecks . length > MAX_RECENT_CHECKS ) {
monitor . recentChecks . shift ();
}
};
private computeReachability = (
currentStatus : MonitorStatus ,
window : Array < boolean >,
threshold : number
) : { nextStatus : "up" | "down" ; transitioned : boolean } => {
const failures = window . filter (( status ) => status === false ). length ;
const failureRate = ( failures / window . length ) * 100 ;
if ( failureRate >= threshold && currentStatus !== "down" ) {
return { nextStatus : "down" , transitioned : true };
}
if ( failureRate < threshold && currentStatus === "down" ) {
return { nextStatus : "up" , transitioned : true };
}
return { nextStatus : "up" , transitioned : false };
};
private computeHardwareStatus = ( params : {
currentStatus : MonitorStatus ;
reachabilityDown : boolean ;
2026-04-14 10:41:55 -07:00
metrics : HardwareStatusMetrics ;
2026-04-14 10:28:43 -07:00
thresholds : { cpu : number ; memory : number ; disk : number ; temp : number };
counters : HardwareCounters ;
}) : {
nextStatus : MonitorStatus ;
transitioned : boolean ;
breaches : HardwareBreaches ;
nextCounters : HardwareCounters ;
2026-04-14 10:41:55 -07:00
} => {
2026-04-14 10:28:43 -07:00
const { metrics , thresholds , counters , currentStatus , reachabilityDown } = params ;
const cpuUsage = metrics . cpu ? . usage_percent ?? - 1 ;
const memoryUsage = metrics . memory ? . usage_percent ?? - 1 ;
const temps = metrics . cpu ? . temperature ?? [];
const breaches : HardwareBreaches = {
cpu : cpuUsage !== - 1 && cpuUsage > thresholds . cpu / 100 ,
memory : memoryUsage !== - 1 && memoryUsage > thresholds . memory / 100 ,
disk : metrics.disk
? metrics . disk . some (( d : CheckDiskInfo ) => d != null && typeof d . usage_percent === "number" && d . usage_percent > thresholds . disk / 100 )
: false ,
temp : temps.some (( temp : number ) => temp > thresholds . temp ),
};
// Update counters: decrement (floored at 0) if breached, reset to start otherwise.
const nextCounters = { ... counters };
for ( const key of HARDWARE_METRIC_KEYS ) {
nextCounters [ key ] = breaches [ key ] ? Math . max ( 0 , counters [ key ] - 1 ) : HARDWARE_ALERT_COUNTER_START ;
}
2026-04-14 10:41:55 -07:00
// Status transition: reachability "down" takes precedence; hardware can only drive
// transitions into "breached" and back out to "up". Any other case leaves status untouched.
2026-04-14 10:28:43 -07:00
let nextStatus : MonitorStatus = currentStatus ;
let transitioned = false ;
if ( ! reachabilityDown ) {
2026-04-14 10:41:55 -07:00
// A counter can only reach zero via the decrement path, which only runs when that
// metric is currently breaching — so anyCounterZero already implies anyBreached.
2026-04-14 10:28:43 -07:00
const anyCounterZero = HARDWARE_METRIC_KEYS . some (( k ) => nextCounters [ k ] === 0 );
2026-04-14 10:41:55 -07:00
const allNormal = HARDWARE_METRIC_KEYS . every (( k ) => ! breaches [ k ]);
2026-04-14 10:28:43 -07:00
2026-04-14 10:41:55 -07:00
if ( anyCounterZero && currentStatus !== "breached" ) {
2026-04-14 10:28:43 -07:00
nextStatus = "breached" ;
transitioned = true ;
} else if ( allNormal && currentStatus === "breached" ) {
nextStatus = "up" ;
transitioned = true ;
}
}
return { nextStatus , transitioned , breaches , nextCounters };
};
2026-01-15 23:45:40 +00:00
updateMonitorStatus = async (
2026-03-03 19:34:28 +00:00
statusResponse : MonitorStatusResponse <
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
> ,
2026-01-16 18:11:04 +00:00
check : Check
2026-01-15 23:45:40 +00:00
) : Promise < StatusChangeResult > => {
2025-04-20 11:29:53 -07:00
try {
2026-01-15 23:45:40 +00:00
const { monitorId , teamId , status , code } = statusResponse ;
2026-01-15 20:55:01 +00:00
const monitor = await this . monitorsRepository . findById ( monitorId , teamId );
2025-05-19 12:28:18 -07:00
2025-04-20 11:29:53 -07:00
// Update running stats
2026-04-14 10:28:43 -07:00
await this . tryUpdateRunningStats ( monitor , statusResponse );
2025-08-18 13:40:56 -07:00
2026-04-14 10:28:43 -07:00
// Update the sliding window and recent checks
this . appendToWindow ( monitor , status );
this . appendToRecentChecks ( monitor , check );
2026-01-27 18:26:31 +00:00
2026-01-18 23:38:40 +00:00
const prevStatus = monitor . status ;
2025-08-18 13:40:56 -07:00
// Return early if not enough data points
if ( monitor . statusWindow . length < monitor . statusWindowSize ) {
2026-04-14 10:28:43 -07:00
monitor . status = status === true ? "up" : "down" ;
2026-01-15 18:33:03 +00:00
const updated = await this . monitorsRepository . updateById ( monitor . id , monitor . teamId , monitor );
2025-04-20 11:29:53 -07:00
return {
2026-01-13 20:20:00 +00:00
monitor : updated ,
2025-04-20 11:29:53 -07:00
statusChanged : false ,
2025-08-18 13:40:56 -07:00
prevStatus ,
2025-04-20 11:29:53 -07:00
code ,
2025-08-18 13:40:56 -07:00
timestamp : Date.now (),
2025-04-20 11:29:53 -07:00
};
2025-08-18 13:40:56 -07:00
}
2025-05-19 12:28:18 -07:00
2026-04-15 10:54:43 -07:00
// With a full window, a single raw check must not change UNLESS we are initializing. Otherwise, only the sliding-window threshold can trigger a transition.
let newStatus : MonitorStatus ;
if ( monitor . status === "initializing" ) {
newStatus = status === true ? "up" : "down" ;
} else {
newStatus = monitor . status ;
}
2026-04-14 10:28:43 -07:00
let statusChanged = false ;
2025-08-18 15:34:15 -07:00
2026-04-14 10:28:43 -07:00
// First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches.
2026-04-15 10:54:43 -07:00
const reachabilityResult = this . computeReachability ( newStatus , monitor . statusWindow , monitor . statusWindowThreshold );
2026-04-14 10:28:43 -07:00
if ( reachabilityResult . transitioned ) {
newStatus = reachabilityResult . nextStatus ;
2025-08-18 13:40:56 -07:00
statusChanged = true ;
}
2026-04-14 10:41:55 -07:00
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
2026-04-14 10:28:43 -07:00
let thresholdBreaches : HardwareBreaches | undefined ;
2026-04-14 10:41:55 -07:00
const hardwarePayload = statusResponse . payload as HardwareStatusPayload | undefined ;
if ( monitor . type === "hardware" && hardwarePayload ? . data ) {
2026-04-14 10:28:43 -07:00
const hardware = this . computeHardwareStatus ({
2026-04-15 10:54:43 -07:00
currentStatus : newStatus ,
2026-04-14 10:28:43 -07:00
reachabilityDown : newStatus === "down" ,
2026-04-14 10:41:55 -07:00
metrics : hardwarePayload.data ,
2026-04-14 10:28:43 -07:00
thresholds : {
cpu : monitor.cpuAlertThreshold ,
memory : monitor.memoryAlertThreshold ,
disk : monitor.diskAlertThreshold ,
temp : monitor.tempAlertThreshold ,
},
counters : {
cpu : monitor.cpuAlertCounter ,
memory : monitor.memoryAlertCounter ,
disk : monitor.diskAlertCounter ,
temp : monitor.tempAlertCounter ,
},
});
2026-02-14 22:52:14 +00:00
2026-04-14 10:41:55 -07:00
monitor . cpuAlertCounter = hardware . nextCounters . cpu ;
monitor . memoryAlertCounter = hardware . nextCounters . memory ;
monitor . diskAlertCounter = hardware . nextCounters . disk ;
monitor . tempAlertCounter = hardware . nextCounters . temp ;
thresholdBreaches = hardware . breaches ;
if ( hardware . transitioned ) {
newStatus = hardware . nextStatus ;
statusChanged = true ;
2026-02-14 22:14:51 +00:00
}
}
2026-02-14 22:52:14 +00:00
// Apply the final status
monitor . status = newStatus ;
2026-01-15 18:33:03 +00:00
const updated = await this . monitorsRepository . updateById ( monitor . id , monitor . teamId , monitor );
2025-05-19 12:28:18 -07:00
2025-04-20 11:29:53 -07:00
return {
2026-01-13 20:20:00 +00:00
monitor : updated ,
2025-08-18 13:40:56 -07:00
statusChanged ,
2025-04-20 11:29:53 -07:00
prevStatus ,
code ,
timestamp : new Date (). getTime (),
2026-02-14 22:14:51 +00:00
thresholdBreaches ,
2025-04-20 11:29:53 -07:00
};
2026-03-03 23:01:35 +00:00
} catch ( error : unknown ) {
throw new AppError ({
2026-03-12 11:12:46 -07:00
message : `Failed to update monitor with id ${ check . metadata . monitorId } with status: ${ error instanceof Error ? error . message : "Unknown error" } ` ,
2026-03-03 23:01:35 +00:00
service : SERVICE_NAME ,
method : "updateMonitorStatus" ,
});
2025-04-20 11:29:53 -07:00
}
};
}