diff --git a/server/src/db/models/Monitor.js b/server/src/db/models/Monitor.js index 8d4ff9468..0ad0e0a78 100755 --- a/server/src/db/models/Monitor.js +++ b/server/src/db/models/Monitor.js @@ -28,6 +28,18 @@ const MonitorSchema = mongoose.Schema( type: Boolean, default: undefined, }, + statusWindow: { + type: [Boolean], + default: [], + }, + statusWindowSize: { + type: Number, + default: 5, + }, + statusWindowThreshold: { + type: Number, + default: 0.6, + }, type: { type: String, required: true, diff --git a/server/src/service/infrastructure/bufferService.js b/server/src/service/infrastructure/bufferService.js index f55e25037..ed48cc654 100755 --- a/server/src/service/infrastructure/bufferService.js +++ b/server/src/service/infrastructure/bufferService.js @@ -3,7 +3,7 @@ const SERVICE_NAME = "BufferService"; class BufferService { static SERVICE_NAME = SERVICE_NAME; constructor({ db, logger, envSettings }) { - this.BUFFER_TIMEOUT = envSettings.nodeEnv === "development" ? 5000 : 1000 * 60 * 1; // 1 minute + this.BUFFER_TIMEOUT = envSettings.nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute this.db = db; this.logger = logger; this.SERVICE_NAME = SERVICE_NAME; diff --git a/server/src/service/infrastructure/statusService.js b/server/src/service/infrastructure/statusService.js index af2c82fa9..0c3434df0 100755 --- a/server/src/service/infrastructure/statusService.js +++ b/server/src/service/infrastructure/statusService.js @@ -117,31 +117,62 @@ class StatusService { // Update running stats this.updateRunningStats({ monitor, networkResponse }); - // No change in monitor status, return early - if (monitor.status === status) + // Update status sliding window + monitor.statusWindow.push(status); + if (monitor.statusWindow.length > monitor.statusWindowSize) { + monitor.statusWindow.shift(); + } + + if (!monitor.status) { + monitor.status = status; + } + + let newStatus = monitor.status; + let statusChanged = false; + const prevStatus = monitor.status; + + // Return early if not enough data points + if (monitor.statusWindow.length < monitor.statusWindowSize) { + await monitor.save(); return { monitor, statusChanged: false, - prevStatus: monitor.status, + prevStatus, code, - timestamp: new Date().getTime(), + timestamp: Date.now(), }; + } - // Monitor status changed, save prev status and update monitor - this.logger.info({ - service: this.SERVICE_NAME, - message: `${monitor.name} went from ${this.getStatusString(monitor.status)} to ${this.getStatusString(status)}`, - prevStatus: monitor.status, - newStatus: status, - }); + // Check if threshold has been met + const failures = monitor.statusWindow.filter((s) => s === false).length; + const failureRate = failures / monitor.statusWindow.length; - const prevStatus = monitor.status; - monitor.status = status; + // If threshold has been met and the monitor is not already down, mark down: + if (failureRate > monitor.statusWindowThreshold && monitor.status !== false) { + newStatus = false; + statusChanged = true; + } + // If the failure rate is below the threshold and the monitor is down, recover: + else if (failureRate <= monitor.statusWindowThreshold && monitor.status === false) { + newStatus = true; + statusChanged = true; + } + + if (statusChanged) { + this.logger.info({ + service: this.SERVICE_NAME, + message: `${monitor.name} went from ${this.getStatusString(prevStatus)} to ${this.getStatusString(newStatus)}`, + prevStatus, + newStatus, + }); + } + + monitor.status = newStatus; await monitor.save(); return { monitor, - statusChanged: true, + statusChanged, prevStatus, code, timestamp: new Date().getTime(),