atomic update

This commit is contained in:
Alex Holliday committed 2026-04-16 10:41:00 -07:00
1 parent 273aa7d01a
commit c7ef412d7a
3 files changed
+153 -1

No files matched your search

@@ -217,6 +217,125 @@ export class StatusService implements IStatusService {
return { nextStatus, transitioned, breaches, nextCounters };
};
private updateStatusWindowAndRecentChecks = async (monitor: Monitor, check: Check) => {
const checkSnapshot = this.toCheckSnapshot(check);
return await this.monitorsRepository.updateStatusWindowAndChecks(
monitor.id,
monitor.teamId,
check.status,
checkSnapshot,
monitor.statusWindowSize,
MAX_RECENT_CHECKS
);
};
newUpdateMonitorStatus = async (
statusResponse: MonitorStatusResponse<
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
>,
check: Check
): Promise<StatusChangeResult> => {
try {
const { monitorId, teamId, status, code } = statusResponse;
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
// Update running stats
await this.tryUpdateRunningStats(monitor, statusResponse);
const updatedMonitor = await this.updateStatusWindowAndRecentChecks(monitor, check);
const prevStatus = updatedMonitor.status;
// Return early if not enough data points
if (updatedMonitor.statusWindow.length < updatedMonitor.statusWindowSize) {
updatedMonitor.status = status === true ? "up" : "down";
const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, { status: updatedMonitor.status });
return {
monitor: updated,
statusChanged: false,
prevStatus,
code,
timestamp: Date.now(),
};
}
// With a full window, a single raw check must not change UNLESS we are initializing. Otherwise, only the sliding-window threshold can trigger a transition.
let newStatus: MonitorStatus;
if (updatedMonitor.status === "initializing") {
newStatus = status === true ? "up" : "down";
} else {
newStatus = updatedMonitor.status;
}
let statusChanged = false;
// First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches.
const reachabilityResult = this.computeReachability(newStatus, updatedMonitor.statusWindow, updatedMonitor.statusWindowThreshold);
if (reachabilityResult.transitioned) {
newStatus = reachabilityResult.nextStatus;
statusChanged = true;
}
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
let thresholdBreaches: HardwareBreaches | undefined;
const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined;
if (updatedMonitor.type === "hardware" && hardwarePayload?.data) {
const hardware = this.computeHardwareStatus({
currentStatus: newStatus,
reachabilityDown: newStatus === "down",
metrics: hardwarePayload.data,
thresholds: {
cpu: updatedMonitor.cpuAlertThreshold,
memory: updatedMonitor.memoryAlertThreshold,
disk: updatedMonitor.diskAlertThreshold,
temp: updatedMonitor.tempAlertThreshold,
},
counters: {
cpu: updatedMonitor.cpuAlertCounter,
memory: updatedMonitor.memoryAlertCounter,
disk: updatedMonitor.diskAlertCounter,
temp: updatedMonitor.tempAlertCounter,
},
});
updatedMonitor.cpuAlertCounter = hardware.nextCounters.cpu;
updatedMonitor.memoryAlertCounter = hardware.nextCounters.memory;
updatedMonitor.diskAlertCounter = hardware.nextCounters.disk;
updatedMonitor.tempAlertCounter = hardware.nextCounters.temp;
thresholdBreaches = hardware.breaches;
if (hardware.transitioned) {
newStatus = hardware.nextStatus;
statusChanged = true;
}
}
// Apply the final status
updatedMonitor.status = newStatus;
const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, updatedMonitor);
return {
monitor: updated,
statusChanged,
prevStatus,
code,
timestamp: new Date().getTime(),
thresholdBreaches,
};
} catch (error: unknown) {
throw new AppError({
message: `Failed to update monitor with id ${check.metadata.monitorId} with status: ${error instanceof Error ? error.message : "Unknown error"}`,
service: SERVICE_NAME,
method: "updateMonitorStatus",
});
}
};
updateMonitorStatus = async (
statusResponse: MonitorStatusResponse<
| PingStatusPayload