refactor: wire backfill pipeline and remove per-location Check documents

## Changes
- Remove location field from CheckMetadata, MonitorStatusResponse, Check model metadata schema, and location index
- Remove location from checkService.buildCheck() and MongoChecksRepository toDocument/mapMetadata
- Rewrite SuperSimpleQueueHelper Step 2b: instead of creating separate Check documents per Globalping location, backfill locationResults into the same Check document after the local check is buffered
- Rewrite findLocationChecksByMonitorId to use $unwind on locationResults subdocument array instead of querying by metadata.location
- Remove metadata.location filter from findUptimeDateRangeChecks
- Pass locationResults through in toEntity

## Benefits
Single Check document per monitoring cycle now contains both local results
(top-level fields) and per-location Globalping results (locationResults array).
This eliminates document proliferation and simplifies queries.
This commit is contained in:
gorkem-bwl committed 2026-02-20 08:20:16 -05:00
1 parent 3b858de2b5
commit a23e4de1d3
6 files changed
+45 -49

No files matched your search

-5
View File
@@ -20,7 +20,6 @@ type CheckMetadataDocument = Omit<CheckMetadata, "monitorId" | "teamId"> & {
monitorId: Types.ObjectId;
teamId: Types.ObjectId;
type: MonitorType;
location?: string;
};
type CheckDocumentBase = Omit<Check, "id" | "metadata" | "expiry" | "ackAt" | "createdAt" | "updatedAt"> & {
@@ -210,9 +209,6 @@ const metadataSchema = new Schema<CheckMetadataDocument>(
required: true,
index: true,
},
location: {
type: String,
},
},
{ _id: false }
);
@@ -319,7 +315,6 @@ CheckSchema.index({ createdAt: 1 });
CheckSchema.index({ "metadata.teamId": 1, createdAt: -1 });
CheckSchema.index({ "metadata.monitorId": 1, "metadata.type": 1, createdAt: -1 });
CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
CheckSchema.index({ "metadata.monitorId": 1, "metadata.location": 1, createdAt: -1 });
const CheckModel = model<CheckDocument>("Check", CheckSchema);
@@ -170,7 +170,6 @@ class MongoChecksRepository implements IChecksRepository {
monitorId: toStringId(metadata.monitorId),
teamId: toStringId(metadata.teamId),
type: metadata.type,
...(metadata.location ? { location: metadata.location } : {}),
});
return {
@@ -196,6 +195,7 @@ class MongoChecksRepository implements IChecksRepository {
seo: doc.seo,
performance: doc.performance,
audits: mapAudits(doc.audits),
locationResults: (doc as any).locationResults,
__v: doc.__v ?? 0,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
@@ -221,7 +221,6 @@ class MongoChecksRepository implements IChecksRepository {
monitorId: new mongoose.Types.ObjectId(metadata.monitorId),
teamId: new mongoose.Types.ObjectId(metadata.teamId),
type: metadata.type,
...(metadata.location ? { location: metadata.location } : {}),
},
...rest,
} as unknown as CheckDocument;
@@ -429,24 +428,25 @@ class MongoChecksRepository implements IChecksRepository {
findLocationChecksByMonitorId = async (monitorId: string, startDate: Date, endDate: Date, dateString: string): Promise<Record<string, any>> => {
const monitorObjectId = new mongoose.Types.ObjectId(monitorId);
const matchStage = {
"metadata.monitorId": monitorObjectId,
"metadata.location": { $exists: true, $ne: null },
createdAt: { $gte: startDate, $lte: endDate },
};
const results = await CheckModel.aggregate([
{ $match: matchStage },
{ $sort: { createdAt: 1 } },
{
$match: {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
locationResults: { $type: "array", $ne: [] },
},
},
{ $unwind: "$locationResults" },
{
$group: {
_id: {
location: "$metadata.location",
location: "$locationResults.location",
bucket: { $dateToString: { format: dateString, date: "$createdAt" } },
},
avgResponseTime: { $avg: "$responseTime" },
avgResponseTime: { $avg: "$locationResults.responseTime" },
totalChecks: { $sum: 1 },
upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
upChecks: { $sum: { $cond: [{ $eq: ["$locationResults.status", true] }, 1, 0] } },
},
},
{ $sort: { "_id.bucket": 1 } },
@@ -490,7 +490,6 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage: Record<string, any> = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
"metadata.location": { $exists: false },
};
const [result] = await CheckModel.aggregate([
{ $match: matchStage },
+1 -2
View File
@@ -37,7 +37,7 @@ class CheckService {
};
buildCheck = (statusResponse: MonitorStatusResponse<PageSpeedStatusPayload | HardwareStatusPayload | undefined>) => {
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings, location } = statusResponse;
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
const check: Partial<Check> = {
id: new Types.ObjectId().toString(),
@@ -46,7 +46,6 @@ class CheckService {
monitorId,
teamId,
type,
...(location ? { location } : {}),
},
status,
statusCode: code,
@@ -127,33 +127,6 @@ class SuperSimpleQueueHelper {
throw new Error("No network response");
}
// Step 2b. Run Globalping checks (fire-and-forget, doesn't affect local check pipeline)
if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping")) {
this.globalpingService
.runChecks(monitor)
.then((locationStatuses) => {
for (const locStatus of locationStatuses) {
const locCheck = this.checkService.buildCheck(locStatus);
if (locCheck) {
this.buffer.addToBuffer(locCheck);
} else {
this.logger.warn({
message: `Failed to build Globalping check for monitor ${monitorId}, location: ${locStatus.location}`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
}
}
})
.catch((err: any) => {
this.logger.warn({
message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
});
}
// Step 3. Build check
const check = await this.checkService.buildCheck(status);
if (!check) {
@@ -167,6 +140,38 @@ class SuperSimpleQueueHelper {
}
// Step 4. Add check to buffer
this.buffer.addToBuffer(check);
// Step 4b. Run Globalping checks and backfill into the same check document.
// Fire-and-forget: Globalping results are supplementary and must not block the
// local check pipeline. The dual-write strategy below is safe against mid-flush
// races because flushBuffer() replaces the buffer array reference atomically
// (this.buffer = []) — if findInBuffer returns a hit, the check is still in the
// current buffer and our mutation will be included in the upcoming flush; if it
// returns undefined the check has already been flushed, so we fall back to a
// direct DB update.
if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping") && check.id) {
const checkId = check.id;
this.globalpingService
.runChecks(monitor)
.then(async (locationResults) => {
if (locationResults.length > 0) {
const buffered = this.buffer.findInBuffer(checkId);
if (buffered) {
buffered.locationResults = locationResults;
} else {
await this.checksRepository.updateLocationResults(checkId, locationResults);
}
}
})
.catch((err: any) => {
this.logger.warn({
message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
});
}
// Step 5. Update monitor status
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
-1
View File
@@ -6,7 +6,6 @@ export interface CheckMetadata {
monitorId: string;
teamId: string;
type: MonitorType;
location?: string;
}
export interface LocationResult {
-1
View File
@@ -34,7 +34,6 @@ export interface MonitorStatusResponse<T = any> {
matchMethod?: MonitorMatchMethod;
expectedValue?: string;
extracted?: unknown;
location?: string;
}
export interface GlobalpingCheckResult {