From a23e4de1d3335599463a3cb411655cca155eb22c Mon Sep 17 00:00:00 2001 From: gorkem-bwl Date: Fri, 20 Feb 2026 00:57:46 -0500 Subject: [PATCH] refactor: wire backfill pipeline and remove per-location Check documents ## Changes - Remove location field from CheckMetadata, MonitorStatusResponse, Check model metadata schema, and location index - Remove location from checkService.buildCheck() and MongoChecksRepository toDocument/mapMetadata - Rewrite SuperSimpleQueueHelper Step 2b: instead of creating separate Check documents per Globalping location, backfill locationResults into the same Check document after the local check is buffered - Rewrite findLocationChecksByMonitorId to use $unwind on locationResults subdocument array instead of querying by metadata.location - Remove metadata.location filter from findUptimeDateRangeChecks - Pass locationResults through in toEntity ## Benefits Single Check document per monitoring cycle now contains both local results (top-level fields) and per-location Globalping results (locationResults array). This eliminates document proliferation and simplifies queries. --- server/src/db/models/Check.ts | 5 -- .../checks/MongoChecksRepistory.ts | 25 ++++---- server/src/service/business/checkService.ts | 3 +- .../SuperSimpleQueueHelper.ts | 59 ++++++++++--------- server/src/types/check.ts | 1 - server/src/types/network.ts | 1 - 6 files changed, 45 insertions(+), 49 deletions(-) diff --git a/server/src/db/models/Check.ts b/server/src/db/models/Check.ts index 3545960f2..ca94d9542 100644 --- a/server/src/db/models/Check.ts +++ b/server/src/db/models/Check.ts @@ -20,7 +20,6 @@ type CheckMetadataDocument = Omit & { monitorId: Types.ObjectId; teamId: Types.ObjectId; type: MonitorType; - location?: string; }; type CheckDocumentBase = Omit & { @@ -210,9 +209,6 @@ const metadataSchema = new Schema( required: true, index: true, }, - location: { - type: String, - }, }, { _id: false } ); @@ -319,7 +315,6 @@ CheckSchema.index({ createdAt: 1 }); CheckSchema.index({ "metadata.teamId": 1, createdAt: -1 }); CheckSchema.index({ "metadata.monitorId": 1, "metadata.type": 1, createdAt: -1 }); CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 }); -CheckSchema.index({ "metadata.monitorId": 1, "metadata.location": 1, createdAt: -1 }); const CheckModel = model("Check", CheckSchema); diff --git a/server/src/repositories/checks/MongoChecksRepistory.ts b/server/src/repositories/checks/MongoChecksRepistory.ts index ccd17b4a5..d8809da5d 100644 --- a/server/src/repositories/checks/MongoChecksRepistory.ts +++ b/server/src/repositories/checks/MongoChecksRepistory.ts @@ -170,7 +170,6 @@ class MongoChecksRepository implements IChecksRepository { monitorId: toStringId(metadata.monitorId), teamId: toStringId(metadata.teamId), type: metadata.type, - ...(metadata.location ? { location: metadata.location } : {}), }); return { @@ -196,6 +195,7 @@ class MongoChecksRepository implements IChecksRepository { seo: doc.seo, performance: doc.performance, audits: mapAudits(doc.audits), + locationResults: (doc as any).locationResults, __v: doc.__v ?? 0, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), @@ -221,7 +221,6 @@ class MongoChecksRepository implements IChecksRepository { monitorId: new mongoose.Types.ObjectId(metadata.monitorId), teamId: new mongoose.Types.ObjectId(metadata.teamId), type: metadata.type, - ...(metadata.location ? { location: metadata.location } : {}), }, ...rest, } as unknown as CheckDocument; @@ -429,24 +428,25 @@ class MongoChecksRepository implements IChecksRepository { findLocationChecksByMonitorId = async (monitorId: string, startDate: Date, endDate: Date, dateString: string): Promise> => { const monitorObjectId = new mongoose.Types.ObjectId(monitorId); - const matchStage = { - "metadata.monitorId": monitorObjectId, - "metadata.location": { $exists: true, $ne: null }, - createdAt: { $gte: startDate, $lte: endDate }, - }; const results = await CheckModel.aggregate([ - { $match: matchStage }, - { $sort: { createdAt: 1 } }, + { + $match: { + "metadata.monitorId": monitorObjectId, + createdAt: { $gte: startDate, $lte: endDate }, + locationResults: { $type: "array", $ne: [] }, + }, + }, + { $unwind: "$locationResults" }, { $group: { _id: { - location: "$metadata.location", + location: "$locationResults.location", bucket: { $dateToString: { format: dateString, date: "$createdAt" } }, }, - avgResponseTime: { $avg: "$responseTime" }, + avgResponseTime: { $avg: "$locationResults.responseTime" }, totalChecks: { $sum: 1 }, - upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } }, + upChecks: { $sum: { $cond: [{ $eq: ["$locationResults.status", true] }, 1, 0] } }, }, }, { $sort: { "_id.bucket": 1 } }, @@ -490,7 +490,6 @@ class MongoChecksRepository implements IChecksRepository { const matchStage: Record = { "metadata.monitorId": monitorObjectId, createdAt: { $gte: startDate, $lte: endDate }, - "metadata.location": { $exists: false }, }; const [result] = await CheckModel.aggregate([ { $match: matchStage }, diff --git a/server/src/service/business/checkService.ts b/server/src/service/business/checkService.ts index 7643645a3..5564d1d7f 100644 --- a/server/src/service/business/checkService.ts +++ b/server/src/service/business/checkService.ts @@ -37,7 +37,7 @@ class CheckService { }; buildCheck = (statusResponse: MonitorStatusResponse) => { - const { monitorId, teamId, type, status, responseTime, code, message, payload, timings, location } = statusResponse; + const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse; const check: Partial = { id: new Types.ObjectId().toString(), @@ -46,7 +46,6 @@ class CheckService { monitorId, teamId, type, - ...(location ? { location } : {}), }, status, statusCode: code, diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index 2a5a93443..cae18304a 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -127,33 +127,6 @@ class SuperSimpleQueueHelper { throw new Error("No network response"); } - // Step 2b. Run Globalping checks (fire-and-forget, doesn't affect local check pipeline) - if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping")) { - this.globalpingService - .runChecks(monitor) - .then((locationStatuses) => { - for (const locStatus of locationStatuses) { - const locCheck = this.checkService.buildCheck(locStatus); - if (locCheck) { - this.buffer.addToBuffer(locCheck); - } else { - this.logger.warn({ - message: `Failed to build Globalping check for monitor ${monitorId}, location: ${locStatus.location}`, - service: SERVICE_NAME, - method: "getMonitorJob", - }); - } - } - }) - .catch((err: any) => { - this.logger.warn({ - message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`, - service: SERVICE_NAME, - method: "getMonitorJob", - }); - }); - } - // Step 3. Build check const check = await this.checkService.buildCheck(status); if (!check) { @@ -167,6 +140,38 @@ class SuperSimpleQueueHelper { } // Step 4. Add check to buffer this.buffer.addToBuffer(check); + + // Step 4b. Run Globalping checks and backfill into the same check document. + // Fire-and-forget: Globalping results are supplementary and must not block the + // local check pipeline. The dual-write strategy below is safe against mid-flush + // races because flushBuffer() replaces the buffer array reference atomically + // (this.buffer = []) — if findInBuffer returns a hit, the check is still in the + // current buffer and our mutation will be included in the upcoming flush; if it + // returns undefined the check has already been flushed, so we fall back to a + // direct DB update. + if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping") && check.id) { + const checkId = check.id; + this.globalpingService + .runChecks(monitor) + .then(async (locationResults) => { + if (locationResults.length > 0) { + const buffered = this.buffer.findInBuffer(checkId); + if (buffered) { + buffered.locationResults = locationResults; + } else { + await this.checksRepository.updateLocationResults(checkId, locationResults); + } + } + }) + .catch((err: any) => { + this.logger.warn({ + message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`, + service: SERVICE_NAME, + method: "getMonitorJob", + }); + }); + } + // Step 5. Update monitor status const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); diff --git a/server/src/types/check.ts b/server/src/types/check.ts index e85b53323..82fa6b20c 100644 --- a/server/src/types/check.ts +++ b/server/src/types/check.ts @@ -6,7 +6,6 @@ export interface CheckMetadata { monitorId: string; teamId: string; type: MonitorType; - location?: string; } export interface LocationResult { diff --git a/server/src/types/network.ts b/server/src/types/network.ts index 3191350af..105917334 100644 --- a/server/src/types/network.ts +++ b/server/src/types/network.ts @@ -34,7 +34,6 @@ export interface MonitorStatusResponse { matchMethod?: MonitorMatchMethod; expectedValue?: string; extracted?: unknown; - location?: string; } export interface GlobalpingCheckResult {