refactor: wire backfill pipeline and remove per-location Check documents
## Changes - Remove location field from CheckMetadata, MonitorStatusResponse, Check model metadata schema, and location index - Remove location from checkService.buildCheck() and MongoChecksRepository toDocument/mapMetadata - Rewrite SuperSimpleQueueHelper Step 2b: instead of creating separate Check documents per Globalping location, backfill locationResults into the same Check document after the local check is buffered - Rewrite findLocationChecksByMonitorId to use $unwind on locationResults subdocument array instead of querying by metadata.location - Remove metadata.location filter from findUptimeDateRangeChecks - Pass locationResults through in toEntity ## Benefits Single Check document per monitoring cycle now contains both local results (top-level fields) and per-location Globalping results (locationResults array). This eliminates document proliferation and simplifies queries.
This commit is contained in:
1 parent
3b858de2b5
commit
a23e4de1d3
6 files changed
+45
-49
No files matched your search
@@ -20,7 +20,6 @@ type CheckMetadataDocument = Omit<CheckMetadata, "monitorId" | "teamId"> & {
|
||||
monitorId: Types.ObjectId;
|
||||
teamId: Types.ObjectId;
|
||||
type: MonitorType;
|
||||
location?: string;
|
||||
};
|
||||
|
||||
type CheckDocumentBase = Omit<Check, "id" | "metadata" | "expiry" | "ackAt" | "createdAt" | "updatedAt"> & {
|
||||
@@ -210,9 +209,6 @@ const metadataSchema = new Schema<CheckMetadataDocument>(
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
location: {
|
||||
type: String,
|
||||
},
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
@@ -319,7 +315,6 @@ CheckSchema.index({ createdAt: 1 });
|
||||
CheckSchema.index({ "metadata.teamId": 1, createdAt: -1 });
|
||||
CheckSchema.index({ "metadata.monitorId": 1, "metadata.type": 1, createdAt: -1 });
|
||||
CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
|
||||
CheckSchema.index({ "metadata.monitorId": 1, "metadata.location": 1, createdAt: -1 });
|
||||
|
||||
const CheckModel = model<CheckDocument>("Check", CheckSchema);
|
||||
|
||||
|
||||
@@ -170,7 +170,6 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
monitorId: toStringId(metadata.monitorId),
|
||||
teamId: toStringId(metadata.teamId),
|
||||
type: metadata.type,
|
||||
...(metadata.location ? { location: metadata.location } : {}),
|
||||
});
|
||||
|
||||
return {
|
||||
@@ -196,6 +195,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
seo: doc.seo,
|
||||
performance: doc.performance,
|
||||
audits: mapAudits(doc.audits),
|
||||
locationResults: (doc as any).locationResults,
|
||||
__v: doc.__v ?? 0,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
@@ -221,7 +221,6 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
monitorId: new mongoose.Types.ObjectId(metadata.monitorId),
|
||||
teamId: new mongoose.Types.ObjectId(metadata.teamId),
|
||||
type: metadata.type,
|
||||
...(metadata.location ? { location: metadata.location } : {}),
|
||||
},
|
||||
...rest,
|
||||
} as unknown as CheckDocument;
|
||||
@@ -429,24 +428,25 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
|
||||
findLocationChecksByMonitorId = async (monitorId: string, startDate: Date, endDate: Date, dateString: string): Promise<Record<string, any>> => {
|
||||
const monitorObjectId = new mongoose.Types.ObjectId(monitorId);
|
||||
const matchStage = {
|
||||
"metadata.monitorId": monitorObjectId,
|
||||
"metadata.location": { $exists: true, $ne: null },
|
||||
createdAt: { $gte: startDate, $lte: endDate },
|
||||
};
|
||||
|
||||
const results = await CheckModel.aggregate([
|
||||
{ $match: matchStage },
|
||||
{ $sort: { createdAt: 1 } },
|
||||
{
|
||||
$match: {
|
||||
"metadata.monitorId": monitorObjectId,
|
||||
createdAt: { $gte: startDate, $lte: endDate },
|
||||
locationResults: { $type: "array", $ne: [] },
|
||||
},
|
||||
},
|
||||
{ $unwind: "$locationResults" },
|
||||
{
|
||||
$group: {
|
||||
_id: {
|
||||
location: "$metadata.location",
|
||||
location: "$locationResults.location",
|
||||
bucket: { $dateToString: { format: dateString, date: "$createdAt" } },
|
||||
},
|
||||
avgResponseTime: { $avg: "$responseTime" },
|
||||
avgResponseTime: { $avg: "$locationResults.responseTime" },
|
||||
totalChecks: { $sum: 1 },
|
||||
upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
|
||||
upChecks: { $sum: { $cond: [{ $eq: ["$locationResults.status", true] }, 1, 0] } },
|
||||
},
|
||||
},
|
||||
{ $sort: { "_id.bucket": 1 } },
|
||||
@@ -490,7 +490,6 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const matchStage: Record<string, any> = {
|
||||
"metadata.monitorId": monitorObjectId,
|
||||
createdAt: { $gte: startDate, $lte: endDate },
|
||||
"metadata.location": { $exists: false },
|
||||
};
|
||||
const [result] = await CheckModel.aggregate([
|
||||
{ $match: matchStage },
|
||||
|
||||
@@ -37,7 +37,7 @@ class CheckService {
|
||||
};
|
||||
|
||||
buildCheck = (statusResponse: MonitorStatusResponse<PageSpeedStatusPayload | HardwareStatusPayload | undefined>) => {
|
||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings, location } = statusResponse;
|
||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
||||
|
||||
const check: Partial<Check> = {
|
||||
id: new Types.ObjectId().toString(),
|
||||
@@ -46,7 +46,6 @@ class CheckService {
|
||||
monitorId,
|
||||
teamId,
|
||||
type,
|
||||
...(location ? { location } : {}),
|
||||
},
|
||||
status,
|
||||
statusCode: code,
|
||||
|
||||
@@ -127,33 +127,6 @@ class SuperSimpleQueueHelper {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// Step 2b. Run Globalping checks (fire-and-forget, doesn't affect local check pipeline)
|
||||
if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping")) {
|
||||
this.globalpingService
|
||||
.runChecks(monitor)
|
||||
.then((locationStatuses) => {
|
||||
for (const locStatus of locationStatuses) {
|
||||
const locCheck = this.checkService.buildCheck(locStatus);
|
||||
if (locCheck) {
|
||||
this.buffer.addToBuffer(locCheck);
|
||||
} else {
|
||||
this.logger.warn({
|
||||
message: `Failed to build Globalping check for monitor ${monitorId}, location: ${locStatus.location}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
});
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch((err: any) => {
|
||||
this.logger.warn({
|
||||
message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Step 3. Build check
|
||||
const check = await this.checkService.buildCheck(status);
|
||||
if (!check) {
|
||||
@@ -167,6 +140,38 @@ class SuperSimpleQueueHelper {
|
||||
}
|
||||
// Step 4. Add check to buffer
|
||||
this.buffer.addToBuffer(check);
|
||||
|
||||
// Step 4b. Run Globalping checks and backfill into the same check document.
|
||||
// Fire-and-forget: Globalping results are supplementary and must not block the
|
||||
// local check pipeline. The dual-write strategy below is safe against mid-flush
|
||||
// races because flushBuffer() replaces the buffer array reference atomically
|
||||
// (this.buffer = []) — if findInBuffer returns a hit, the check is still in the
|
||||
// current buffer and our mutation will be included in the upcoming flush; if it
|
||||
// returns undefined the check has already been flushed, so we fall back to a
|
||||
// direct DB update.
|
||||
if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping") && check.id) {
|
||||
const checkId = check.id;
|
||||
this.globalpingService
|
||||
.runChecks(monitor)
|
||||
.then(async (locationResults) => {
|
||||
if (locationResults.length > 0) {
|
||||
const buffered = this.buffer.findInBuffer(checkId);
|
||||
if (buffered) {
|
||||
buffered.locationResults = locationResults;
|
||||
} else {
|
||||
await this.checksRepository.updateLocationResults(checkId, locationResults);
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch((err: any) => {
|
||||
this.logger.warn({
|
||||
message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Step 5. Update monitor status
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ export interface CheckMetadata {
|
||||
monitorId: string;
|
||||
teamId: string;
|
||||
type: MonitorType;
|
||||
location?: string;
|
||||
}
|
||||
|
||||
export interface LocationResult {
|
||||
|
||||
@@ -34,7 +34,6 @@ export interface MonitorStatusResponse<T = any> {
|
||||
matchMethod?: MonitorMatchMethod;
|
||||
expectedValue?: string;
|
||||
extracted?: unknown;
|
||||
location?: string;
|
||||
}
|
||||
|
||||
export interface GlobalpingCheckResult {
|
||||
|
||||
Reference in new issue
Block a user