atomic update

This commit is contained in:
Alex Holliday committed 2026-04-16 10:41:00 -07:00
1 parent 273aa7d01a
commit c7ef412d7a
3 files changed
+153 -1

No files matched your search

@@ -1,4 +1,4 @@
import { type MonitorType, type Monitor, type MonitorsSummary } from "@/types/index.js";
import { type MonitorType, type Monitor, type MonitorsSummary, CheckSnapshot } from "@/types/index.js";
export interface TeamQueryConfig {
limit?: number;
@@ -29,6 +29,14 @@ export interface IMonitorsRepository {
// update
updateById(monitorId: string, teamId: string, updates: Partial<Monitor>): Promise<Monitor>;
updateStatusWindowAndChecks(
monitorId: string,
teamId: string,
status: boolean,
checkSnapshot: CheckSnapshot,
windowSize: number,
maxRecentChecks: number
): Promise<Monitor>;
togglePauseById(monitorId: string, teamId: string): Promise<Monitor>;
// delete
deleteById(monitorId: string, teamId: string): Promise<Monitor>;
@@ -182,6 +182,31 @@ class MongoMonitorsRepository implements IMonitorsRepository {
return this.toEntity(updatedMonitor);
};
updateStatusWindowAndChecks = async (
monitorId: string,
teamId: string,
status: boolean,
checkSnapshot: CheckSnapshot,
windowSize: number,
maxRecentChecks: number
): Promise<Monitor> => {
const updatedMonitor = await MonitorModel.findOneAndUpdate(
{ _id: monitorId, teamId },
{
$push: {
statusWindow: { $each: [status], $slice: -windowSize },
recentChecks: { $each: [checkSnapshot], $slice: -maxRecentChecks },
},
},
{ returnDocument: "after" }
);
if (!updatedMonitor) {
throw new AppError({ message: `Failed to update status and checks for monitor with id ${monitorId}`, status: 500 });
}
return this.toEntity(updatedMonitor);
};
togglePauseById = async (monitorId: string, teamId: string) => {
const monitor = await MonitorModel.findOneAndUpdate(
{ _id: monitorId, teamId },
@@ -217,6 +217,125 @@ export class StatusService implements IStatusService {
return { nextStatus, transitioned, breaches, nextCounters };
};
private updateStatusWindowAndRecentChecks = async (monitor: Monitor, check: Check) => {
const checkSnapshot = this.toCheckSnapshot(check);
return await this.monitorsRepository.updateStatusWindowAndChecks(
monitor.id,
monitor.teamId,
check.status,
checkSnapshot,
monitor.statusWindowSize,
MAX_RECENT_CHECKS
);
};
newUpdateMonitorStatus = async (
statusResponse: MonitorStatusResponse<
| PingStatusPayload
| HttpStatusPayload
| PageSpeedStatusPayload
| HardwareStatusPayload
| DockerStatusPayload
| PortStatusPayload
| GameStatusPayload
| GrpcStatusPayload
| undefined
>,
check: Check
): Promise<StatusChangeResult> => {
try {
const { monitorId, teamId, status, code } = statusResponse;
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
// Update running stats
await this.tryUpdateRunningStats(monitor, statusResponse);
const updatedMonitor = await this.updateStatusWindowAndRecentChecks(monitor, check);
const prevStatus = updatedMonitor.status;
// Return early if not enough data points
if (updatedMonitor.statusWindow.length < updatedMonitor.statusWindowSize) {
updatedMonitor.status = status === true ? "up" : "down";
const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, { status: updatedMonitor.status });
return {
monitor: updated,
statusChanged: false,
prevStatus,
code,
timestamp: Date.now(),
};
}
// With a full window, a single raw check must not change UNLESS we are initializing. Otherwise, only the sliding-window threshold can trigger a transition.
let newStatus: MonitorStatus;
if (updatedMonitor.status === "initializing") {
newStatus = status === true ? "up" : "down";
} else {
newStatus = updatedMonitor.status;
}
let statusChanged = false;
// First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches.
const reachabilityResult = this.computeReachability(newStatus, updatedMonitor.statusWindow, updatedMonitor.statusWindowThreshold);
if (reachabilityResult.transitioned) {
newStatus = reachabilityResult.nextStatus;
statusChanged = true;
}
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
let thresholdBreaches: HardwareBreaches | undefined;
const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined;
if (updatedMonitor.type === "hardware" && hardwarePayload?.data) {
const hardware = this.computeHardwareStatus({
currentStatus: newStatus,
reachabilityDown: newStatus === "down",
metrics: hardwarePayload.data,
thresholds: {
cpu: updatedMonitor.cpuAlertThreshold,
memory: updatedMonitor.memoryAlertThreshold,
disk: updatedMonitor.diskAlertThreshold,
temp: updatedMonitor.tempAlertThreshold,
},
counters: {
cpu: updatedMonitor.cpuAlertCounter,
memory: updatedMonitor.memoryAlertCounter,
disk: updatedMonitor.diskAlertCounter,
temp: updatedMonitor.tempAlertCounter,
},
});
updatedMonitor.cpuAlertCounter = hardware.nextCounters.cpu;
updatedMonitor.memoryAlertCounter = hardware.nextCounters.memory;
updatedMonitor.diskAlertCounter = hardware.nextCounters.disk;
updatedMonitor.tempAlertCounter = hardware.nextCounters.temp;
thresholdBreaches = hardware.breaches;
if (hardware.transitioned) {
newStatus = hardware.nextStatus;
statusChanged = true;
}
}
// Apply the final status
updatedMonitor.status = newStatus;
const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, updatedMonitor);
return {
monitor: updated,
statusChanged,
prevStatus,
code,
timestamp: new Date().getTime(),
thresholdBreaches,
};
} catch (error: unknown) {
throw new AppError({
message: `Failed to update monitor with id ${check.metadata.monitorId} with status: ${error instanceof Error ? error.message : "Unknown error"}`,
service: SERVICE_NAME,
method: "updateMonitorStatus",
});
}
};
updateMonitorStatus = async (
statusResponse: MonitorStatusResponse<
| PingStatusPayload