2026-02-12 22:37:40 +00:00
import { IChecksRepository , IMonitorsRepository , IMonitorStatsRepository } from "@/repositories/index.js" ;
2026-01-16 18:11:04 +00:00
import type {
Monitor ,
2026-02-12 18:30:44 +00:00
MonitorStatus ,
2026-01-16 18:11:04 +00:00
MonitorStatusResponse ,
StatusChangeResult ,
Check ,
HardwareStatusPayload ,
PageSpeedStatusPayload ,
2026-03-03 19:34:28 +00:00
PingStatusPayload ,
HttpStatusPayload ,
DockerStatusPayload ,
PortStatusPayload ,
GameStatusPayload ,
GrpcStatusPayload ,
2026-01-27 18:26:31 +00:00
CheckSnapshot ,
2026-03-03 23:01:35 +00:00
CheckDiskInfo ,
2026-01-16 18:11:04 +00:00
} from "@/types/index.js" ;
2026-03-03 23:01:35 +00:00
import { AppError } from "@/utils/AppError.js" ;
2026-02-12 22:37:40 +00:00
import { ILogger } from "@/utils/logger.js" ;
2026-03-03 23:01:35 +00:00
import { IBufferService } from "./bufferService.js" ;
2026-04-14 10:28:43 -07:00
import type { HardwareStatusMetrics } from "@/types/network.js" ;
2025-04-20 11:29:53 -07:00
const SERVICE_NAME = "StatusService" ;
2026-04-14 10:28:43 -07:00
const MAX_RECENT_CHECKS = 25 ;
const HARDWARE_ALERT_COUNTER_START = 5 ;
const HARDWARE_METRIC_KEYS = [ "cpu" , "memory" , "disk" , "temp" ] as const ;
type HardwareMetricKey = ( typeof HARDWARE_METRIC_KEYS )[ number ];
type HardwareBreaches = Record < HardwareMetricKey , boolean >;
type HardwareCounters = Record < HardwareMetricKey , number >;
2025-04-20 11:29:53 -07:00
2026-01-18 23:38:40 +00:00
export interface IStatusService {
2026-03-03 23:01:35 +00:00
updateRunningStats ( monitor : Monitor , networkResponse : MonitorStatusResponse ) : Promise < boolean >;
2026-01-18 23:38:40 +00:00
updateMonitorStatus (
2026-03-03 19:34:28 +00:00
statusResponse : MonitorStatusResponse <
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
> ,
2026-01-18 23:38:40 +00:00
check : Check
) : Promise < StatusChangeResult >;
}
export class StatusService implements IStatusService {
2025-04-20 11:29:53 -07:00
static SERVICE_NAME = SERVICE_NAME ;
2026-03-03 23:01:35 +00:00
private logger : ILogger ;
private buffer : IBufferService ;
2026-01-15 22:33:25 +00:00
private monitorsRepository : IMonitorsRepository ;
2026-02-12 22:37:40 +00:00
private monitorStatsRepository : IMonitorStatsRepository ;
private checksRepository : IChecksRepository ;
2026-01-15 22:33:25 +00:00
2026-02-12 22:37:40 +00:00
constructor (
logger : ILogger ,
2026-03-03 23:01:35 +00:00
buffer : IBufferService ,
2026-02-12 22:37:40 +00:00
monitorsRepository : IMonitorsRepository ,
monitorStatsRepository : IMonitorStatsRepository ,
checksRepository : IChecksRepository
) {
2025-04-20 11:29:53 -07:00
this . logger = logger ;
this . buffer = buffer ;
2026-01-13 20:20:00 +00:00
this . monitorsRepository = monitorsRepository ;
2026-02-12 22:37:40 +00:00
this . monitorStatsRepository = monitorStatsRepository ;
this . checksRepository = checksRepository ;
2025-07-25 15:15:14 -07:00
}
get serviceName() {
return StatusService . SERVICE_NAME ;
2025-04-20 11:29:53 -07:00
}
2026-03-03 23:01:35 +00:00
async updateRunningStats ( monitor : Monitor , networkResponse : MonitorStatusResponse ) {
2025-04-20 11:29:53 -07:00
try {
2026-04-13 11:39:29 -07:00
await this . monitorStatsRepository . updateByMonitorId ( monitor . id , {
status : networkResponse.status === true ,
responseTime : networkResponse.responseTime ?? 0 ,
now : Date.now (),
});
2025-04-20 11:29:53 -07:00
return true ;
2026-03-03 23:01:35 +00:00
} catch ( error : unknown ) {
2025-04-20 11:29:53 -07:00
this . logger . error ({
2026-01-15 22:33:25 +00:00
service : SERVICE_NAME ,
2026-03-03 23:01:35 +00:00
message : error instanceof Error ? error . message : "Unknown error" ,
2025-04-20 11:29:53 -07:00
method : "updateRunningStats" ,
2026-03-03 23:01:35 +00:00
stack : error instanceof Error ? error.stack : undefined ,
2025-04-20 11:29:53 -07:00
});
return false ;
}
}
2026-04-14 10:28:43 -07:00
private tryUpdateRunningStats = async ( monitor : Monitor , statusResponse : MonitorStatusResponse ) => {
const statsOk = await this . updateRunningStats ( monitor , statusResponse );
if ( ! statsOk ) {
this . logger . warn ({
service : SERVICE_NAME ,
method : "updateMonitorStatus" ,
message : `Stats update failed for monitor ${ monitor . id } ` ,
});
}
};
private toCheckSnapshot = ( check : Check ) : CheckSnapshot => {
return {
id : check.id ,
status : check.status ,
responseTime : check.responseTime ,
timings : check.timings ,
statusCode : check.statusCode ,
message : check.message ,
cpu : check.cpu ,
memory : check.memory ,
disk : check.disk ,
host : check.host ,
errors : check.errors ,
capture : check.capture ,
net : check.net ,
accessibility : check.accessibility ,
bestPractices : check.bestPractices ,
seo : check.seo ,
performance : check.performance ,
audits : check.audits ,
createdAt : check.createdAt ,
};
};
private computeReachability = (
currentStatus : MonitorStatus ,
window : Array < boolean >,
threshold : number
) : { nextStatus : "up" | "down" ; transitioned : boolean } => {
const failures = window . filter (( status ) => status === false ). length ;
const failureRate = ( failures / window . length ) * 100 ;
if ( failureRate >= threshold && currentStatus !== "down" ) {
return { nextStatus : "down" , transitioned : true };
}
if ( failureRate < threshold && currentStatus === "down" ) {
return { nextStatus : "up" , transitioned : true };
}
return { nextStatus : "up" , transitioned : false };
};
private computeHardwareStatus = ( params : {
currentStatus : MonitorStatus ;
reachabilityDown : boolean ;
2026-04-14 10:41:55 -07:00
metrics : HardwareStatusMetrics ;
2026-04-14 10:28:43 -07:00
thresholds : { cpu : number ; memory : number ; disk : number ; temp : number };
counters : HardwareCounters ;
}) : {
nextStatus : MonitorStatus ;
transitioned : boolean ;
breaches : HardwareBreaches ;
nextCounters : HardwareCounters ;
2026-04-14 10:41:55 -07:00
} => {
2026-04-14 10:28:43 -07:00
const { metrics , thresholds , counters , currentStatus , reachabilityDown } = params ;
const cpuUsage = metrics . cpu ? . usage_percent ?? - 1 ;
const memoryUsage = metrics . memory ? . usage_percent ?? - 1 ;
const temps = metrics . cpu ? . temperature ?? [];
const breaches : HardwareBreaches = {
cpu : cpuUsage !== - 1 && cpuUsage > thresholds . cpu / 100 ,
memory : memoryUsage !== - 1 && memoryUsage > thresholds . memory / 100 ,
disk : metrics.disk
? metrics . disk . some (( d : CheckDiskInfo ) => d != null && typeof d . usage_percent === "number" && d . usage_percent > thresholds . disk / 100 )
: false ,
temp : temps.some (( temp : number ) => temp > thresholds . temp ),
};
// Update counters: decrement (floored at 0) if breached, reset to start otherwise.
const nextCounters = { ... counters };
for ( const key of HARDWARE_METRIC_KEYS ) {
nextCounters [ key ] = breaches [ key ] ? Math . max ( 0 , counters [ key ] - 1 ) : HARDWARE_ALERT_COUNTER_START ;
}
2026-04-14 10:41:55 -07:00
// Status transition: reachability "down" takes precedence; hardware can only drive
// transitions into "breached" and back out to "up". Any other case leaves status untouched.
2026-04-14 10:28:43 -07:00
let nextStatus : MonitorStatus = currentStatus ;
let transitioned = false ;
if ( ! reachabilityDown ) {
2026-04-14 10:41:55 -07:00
// A counter can only reach zero via the decrement path, which only runs when that
// metric is currently breaching — so anyCounterZero already implies anyBreached.
2026-04-14 10:28:43 -07:00
const anyCounterZero = HARDWARE_METRIC_KEYS . some (( k ) => nextCounters [ k ] === 0 );
2026-04-14 10:41:55 -07:00
const allNormal = HARDWARE_METRIC_KEYS . every (( k ) => ! breaches [ k ]);
2026-04-14 10:28:43 -07:00
2026-04-14 10:41:55 -07:00
if ( anyCounterZero && currentStatus !== "breached" ) {
2026-04-14 10:28:43 -07:00
nextStatus = "breached" ;
transitioned = true ;
} else if ( allNormal && currentStatus === "breached" ) {
nextStatus = "up" ;
transitioned = true ;
}
}
return { nextStatus , transitioned , breaches , nextCounters };
};
2026-04-16 10:41:00 -07:00
private updateStatusWindowAndRecentChecks = async ( monitor : Monitor , check : Check ) => {
const checkSnapshot = this . toCheckSnapshot ( check );
return await this . monitorsRepository . updateStatusWindowAndChecks (
monitor . id ,
monitor . teamId ,
check . status ,
checkSnapshot ,
monitor . statusWindowSize ,
MAX_RECENT_CHECKS
);
};
2026-04-16 10:50:30 -07:00
updateMonitorStatus = async (
2026-04-16 10:41:00 -07:00
statusResponse : MonitorStatusResponse <
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
> ,
check : Check
) : Promise < StatusChangeResult > => {
try {
const { monitorId , teamId , status , code } = statusResponse ;
const monitor = await this . monitorsRepository . findById ( monitorId , teamId );
// Update running stats
await this . tryUpdateRunningStats ( monitor , statusResponse );
const updatedMonitor = await this . updateStatusWindowAndRecentChecks ( monitor , check );
const prevStatus = updatedMonitor . status ;
// Return early if not enough data points
if ( updatedMonitor . statusWindow . length < updatedMonitor . statusWindowSize ) {
updatedMonitor . status = status === true ? "up" : "down" ;
const updated = await this . monitorsRepository . updateById ( updatedMonitor . id , updatedMonitor . teamId , { status : updatedMonitor.status });
return {
monitor : updated ,
statusChanged : false ,
prevStatus ,
code ,
timestamp : Date.now (),
};
}
// With a full window, a single raw check must not change UNLESS we are initializing. Otherwise, only the sliding-window threshold can trigger a transition.
let newStatus : MonitorStatus ;
if ( updatedMonitor . status === "initializing" ) {
newStatus = status === true ? "up" : "down" ;
} else {
newStatus = updatedMonitor . status ;
}
let statusChanged = false ;
// First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches.
const reachabilityResult = this . computeReachability ( newStatus , updatedMonitor . statusWindow , updatedMonitor . statusWindowThreshold );
if ( reachabilityResult . transitioned ) {
newStatus = reachabilityResult . nextStatus ;
statusChanged = true ;
}
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
let thresholdBreaches : HardwareBreaches | undefined ;
const hardwarePayload = statusResponse . payload as HardwareStatusPayload | undefined ;
if ( updatedMonitor . type === "hardware" && hardwarePayload ? . data ) {
const hardware = this . computeHardwareStatus ({
currentStatus : newStatus ,
reachabilityDown : newStatus === "down" ,
metrics : hardwarePayload.data ,
thresholds : {
cpu : updatedMonitor.cpuAlertThreshold ,
memory : updatedMonitor.memoryAlertThreshold ,
disk : updatedMonitor.diskAlertThreshold ,
temp : updatedMonitor.tempAlertThreshold ,
},
counters : {
cpu : updatedMonitor.cpuAlertCounter ,
memory : updatedMonitor.memoryAlertCounter ,
disk : updatedMonitor.diskAlertCounter ,
temp : updatedMonitor.tempAlertCounter ,
},
});
updatedMonitor . cpuAlertCounter = hardware . nextCounters . cpu ;
updatedMonitor . memoryAlertCounter = hardware . nextCounters . memory ;
updatedMonitor . diskAlertCounter = hardware . nextCounters . disk ;
updatedMonitor . tempAlertCounter = hardware . nextCounters . temp ;
thresholdBreaches = hardware . breaches ;
if ( hardware . transitioned ) {
newStatus = hardware . nextStatus ;
statusChanged = true ;
}
}
2026-04-16 10:50:30 -07:00
// Apply the final status — only write fields computed by application logic,
// not statusWindow/recentChecks which are owned by the atomic push.
const patch : Partial < Monitor > = { status : newStatus };
if ( updatedMonitor . type === "hardware" && hardwarePayload ? . data ) {
patch . cpuAlertCounter = updatedMonitor . cpuAlertCounter ;
patch . memoryAlertCounter = updatedMonitor . memoryAlertCounter ;
patch . diskAlertCounter = updatedMonitor . diskAlertCounter ;
patch . tempAlertCounter = updatedMonitor . tempAlertCounter ;
2026-02-14 22:14:51 +00:00
}
2026-04-16 10:50:30 -07:00
const updated = await this . monitorsRepository . updateById ( updatedMonitor . id , updatedMonitor . teamId , patch );
2025-05-19 12:28:18 -07:00
2025-04-20 11:29:53 -07:00
return {
2026-01-13 20:20:00 +00:00
monitor : updated ,
2025-08-18 13:40:56 -07:00
statusChanged ,
2025-04-20 11:29:53 -07:00
prevStatus ,
code ,
timestamp : new Date (). getTime (),
2026-02-14 22:14:51 +00:00
thresholdBreaches ,
2025-04-20 11:29:53 -07:00
};
2026-03-03 23:01:35 +00:00
} catch ( error : unknown ) {
throw new AppError ({
2026-03-12 11:12:46 -07:00
message : `Failed to update monitor with id ${ check . metadata . monitorId } with status: ${ error instanceof Error ? error . message : "Unknown error" } ` ,
2026-03-03 23:01:35 +00:00
service : SERVICE_NAME ,
method : "updateMonitorStatus" ,
});
2025-04-20 11:29:53 -07:00
}
};
}