atomic update
This commit is contained in:
1 parent
273aa7d01a
commit
c7ef412d7a
3 files changed
+153
-1
No files matched your search
@@ -1,4 +1,4 @@
|
||||
import { type MonitorType, type Monitor, type MonitorsSummary } from "@/types/index.js";
|
||||
import { type MonitorType, type Monitor, type MonitorsSummary, CheckSnapshot } from "@/types/index.js";
|
||||
|
||||
export interface TeamQueryConfig {
|
||||
limit?: number;
|
||||
@@ -29,6 +29,14 @@ export interface IMonitorsRepository {
|
||||
|
||||
// update
|
||||
updateById(monitorId: string, teamId: string, updates: Partial<Monitor>): Promise<Monitor>;
|
||||
updateStatusWindowAndChecks(
|
||||
monitorId: string,
|
||||
teamId: string,
|
||||
status: boolean,
|
||||
checkSnapshot: CheckSnapshot,
|
||||
windowSize: number,
|
||||
maxRecentChecks: number
|
||||
): Promise<Monitor>;
|
||||
togglePauseById(monitorId: string, teamId: string): Promise<Monitor>;
|
||||
// delete
|
||||
deleteById(monitorId: string, teamId: string): Promise<Monitor>;
|
||||
|
||||
@@ -182,6 +182,31 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
return this.toEntity(updatedMonitor);
|
||||
};
|
||||
|
||||
updateStatusWindowAndChecks = async (
|
||||
monitorId: string,
|
||||
teamId: string,
|
||||
status: boolean,
|
||||
checkSnapshot: CheckSnapshot,
|
||||
windowSize: number,
|
||||
maxRecentChecks: number
|
||||
): Promise<Monitor> => {
|
||||
const updatedMonitor = await MonitorModel.findOneAndUpdate(
|
||||
{ _id: monitorId, teamId },
|
||||
{
|
||||
$push: {
|
||||
statusWindow: { $each: [status], $slice: -windowSize },
|
||||
recentChecks: { $each: [checkSnapshot], $slice: -maxRecentChecks },
|
||||
},
|
||||
},
|
||||
{ returnDocument: "after" }
|
||||
);
|
||||
|
||||
if (!updatedMonitor) {
|
||||
throw new AppError({ message: `Failed to update status and checks for monitor with id ${monitorId}`, status: 500 });
|
||||
}
|
||||
return this.toEntity(updatedMonitor);
|
||||
};
|
||||
|
||||
togglePauseById = async (monitorId: string, teamId: string) => {
|
||||
const monitor = await MonitorModel.findOneAndUpdate(
|
||||
{ _id: monitorId, teamId },
|
||||
|
||||
@@ -217,6 +217,125 @@ export class StatusService implements IStatusService {
|
||||
return { nextStatus, transitioned, breaches, nextCounters };
|
||||
};
|
||||
|
||||
private updateStatusWindowAndRecentChecks = async (monitor: Monitor, check: Check) => {
|
||||
const checkSnapshot = this.toCheckSnapshot(check);
|
||||
return await this.monitorsRepository.updateStatusWindowAndChecks(
|
||||
monitor.id,
|
||||
monitor.teamId,
|
||||
check.status,
|
||||
checkSnapshot,
|
||||
monitor.statusWindowSize,
|
||||
MAX_RECENT_CHECKS
|
||||
);
|
||||
};
|
||||
|
||||
newUpdateMonitorStatus = async (
|
||||
statusResponse: MonitorStatusResponse<
|
||||
| PingStatusPayload
|
||||
| HttpStatusPayload
|
||||
| PageSpeedStatusPayload
|
||||
| HardwareStatusPayload
|
||||
| DockerStatusPayload
|
||||
| PortStatusPayload
|
||||
| GameStatusPayload
|
||||
| GrpcStatusPayload
|
||||
| undefined
|
||||
>,
|
||||
check: Check
|
||||
): Promise<StatusChangeResult> => {
|
||||
try {
|
||||
const { monitorId, teamId, status, code } = statusResponse;
|
||||
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
|
||||
|
||||
// Update running stats
|
||||
await this.tryUpdateRunningStats(monitor, statusResponse);
|
||||
const updatedMonitor = await this.updateStatusWindowAndRecentChecks(monitor, check);
|
||||
const prevStatus = updatedMonitor.status;
|
||||
|
||||
// Return early if not enough data points
|
||||
if (updatedMonitor.statusWindow.length < updatedMonitor.statusWindowSize) {
|
||||
updatedMonitor.status = status === true ? "up" : "down";
|
||||
const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, { status: updatedMonitor.status });
|
||||
return {
|
||||
monitor: updated,
|
||||
statusChanged: false,
|
||||
prevStatus,
|
||||
code,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// With a full window, a single raw check must not change UNLESS we are initializing. Otherwise, only the sliding-window threshold can trigger a transition.
|
||||
let newStatus: MonitorStatus;
|
||||
if (updatedMonitor.status === "initializing") {
|
||||
newStatus = status === true ? "up" : "down";
|
||||
} else {
|
||||
newStatus = updatedMonitor.status;
|
||||
}
|
||||
|
||||
let statusChanged = false;
|
||||
|
||||
// First evaluate reachability-based status changes, which apply to all monitor types and take precedence over hardware breaches.
|
||||
const reachabilityResult = this.computeReachability(newStatus, updatedMonitor.statusWindow, updatedMonitor.statusWindowThreshold);
|
||||
if (reachabilityResult.transitioned) {
|
||||
newStatus = reachabilityResult.nextStatus;
|
||||
statusChanged = true;
|
||||
}
|
||||
|
||||
// Evaluate hardware threshold breaches (only for hardware monitors with metrics payload)
|
||||
let thresholdBreaches: HardwareBreaches | undefined;
|
||||
const hardwarePayload = statusResponse.payload as HardwareStatusPayload | undefined;
|
||||
if (updatedMonitor.type === "hardware" && hardwarePayload?.data) {
|
||||
const hardware = this.computeHardwareStatus({
|
||||
currentStatus: newStatus,
|
||||
reachabilityDown: newStatus === "down",
|
||||
metrics: hardwarePayload.data,
|
||||
thresholds: {
|
||||
cpu: updatedMonitor.cpuAlertThreshold,
|
||||
memory: updatedMonitor.memoryAlertThreshold,
|
||||
disk: updatedMonitor.diskAlertThreshold,
|
||||
temp: updatedMonitor.tempAlertThreshold,
|
||||
},
|
||||
counters: {
|
||||
cpu: updatedMonitor.cpuAlertCounter,
|
||||
memory: updatedMonitor.memoryAlertCounter,
|
||||
disk: updatedMonitor.diskAlertCounter,
|
||||
temp: updatedMonitor.tempAlertCounter,
|
||||
},
|
||||
});
|
||||
|
||||
updatedMonitor.cpuAlertCounter = hardware.nextCounters.cpu;
|
||||
updatedMonitor.memoryAlertCounter = hardware.nextCounters.memory;
|
||||
updatedMonitor.diskAlertCounter = hardware.nextCounters.disk;
|
||||
updatedMonitor.tempAlertCounter = hardware.nextCounters.temp;
|
||||
thresholdBreaches = hardware.breaches;
|
||||
if (hardware.transitioned) {
|
||||
newStatus = hardware.nextStatus;
|
||||
statusChanged = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply the final status
|
||||
updatedMonitor.status = newStatus;
|
||||
|
||||
const updated = await this.monitorsRepository.updateById(updatedMonitor.id, updatedMonitor.teamId, updatedMonitor);
|
||||
|
||||
return {
|
||||
monitor: updated,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
code,
|
||||
timestamp: new Date().getTime(),
|
||||
thresholdBreaches,
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
throw new AppError({
|
||||
message: `Failed to update monitor with id ${check.metadata.monitorId} with status: ${error instanceof Error ? error.message : "Unknown error"}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "updateMonitorStatus",
|
||||
});
|
||||
}
|
||||
};
|
||||
updateMonitorStatus = async (
|
||||
statusResponse: MonitorStatusResponse<
|
||||
| PingStatusPayload
|
||||
|
||||
Reference in new issue
Block a user