Compare commits

..
Author SHA1 Message Date
gorkem-bwl a23e4de1d3 refactor: wire backfill pipeline and remove per-location Check documents
## Changes
- Remove location field from CheckMetadata, MonitorStatusResponse, Check model metadata schema, and location index
- Remove location from checkService.buildCheck() and MongoChecksRepository toDocument/mapMetadata
- Rewrite SuperSimpleQueueHelper Step 2b: instead of creating separate Check documents per Globalping location, backfill locationResults into the same Check document after the local check is buffered
- Rewrite findLocationChecksByMonitorId to use $unwind on locationResults subdocument array instead of querying by metadata.location
- Remove metadata.location filter from findUptimeDateRangeChecks
- Pass locationResults through in toEntity

## Benefits
Single Check document per monitoring cycle now contains both local results
(top-level fields) and per-location Globalping results (locationResults array).
This eliminates document proliferation and simplifies queries.
2026-02-20 08:20:16 -05:00
gorkem-bwl 3b858de2b5 feat: add buffer lookup and location results update to services
## Changes
- Add findInBuffer() to BufferService for looking up checks still in the write buffer
- Add updateLocationResults() to IChecksRepository interface and MongoChecksRepository
- Update GlobalpingService to return GlobalpingCheckResult[] instead of MonitorStatusResponse[]

## Benefits
Non-destructive service additions that prepare the backfill pipeline.
Existing check-writing and Globalping flows are unchanged.
2026-02-20 01:31:37 -05:00
9 changed files with 65 additions and 63 deletions

No files matched your search

-5
View File
@@ -20,7 +20,6 @@ type CheckMetadataDocument = Omit<CheckMetadata, "monitorId" | "teamId"> & {
monitorId: Types.ObjectId;
teamId: Types.ObjectId;
type: MonitorType;
location?: string;
};
type CheckDocumentBase = Omit<Check, "id" | "metadata" | "expiry" | "ackAt" | "createdAt" | "updatedAt"> & {
@@ -210,9 +209,6 @@ const metadataSchema = new Schema<CheckMetadataDocument>(
required: true,
index: true,
},
location: {
type: String,
},
},
{ _id: false }
);
@@ -319,7 +315,6 @@ CheckSchema.index({ createdAt: 1 });
CheckSchema.index({ "metadata.teamId": 1, createdAt: -1 });
CheckSchema.index({ "metadata.monitorId": 1, "metadata.type": 1, createdAt: -1 });
CheckSchema.index({ "metadata.teamId": 1, status: 1, createdAt: -1 });
CheckSchema.index({ "metadata.monitorId": 1, "metadata.location": 1, createdAt: -1 });
const CheckModel = model<CheckDocument>("Check", CheckSchema);
@@ -2,6 +2,7 @@ import type {
Check,
ChecksQueryResult,
ChecksSummary,
LocationResult,
MonitorType,
PageSpeedChecksResult,
HardwareChecksResult,
@@ -37,6 +38,7 @@ export interface IChecksRepository {
findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
findLocationChecksByMonitorId(monitorId: string, startDate: Date, endDate: Date, dateString: string): Promise<Record<string, UptimeChecksResult>>;
// update
updateLocationResults(checkId: string, locationResults: LocationResult[]): Promise<void>;
//delete
deleteByMonitorId(monitorId: string): Promise<number>;
deleteByTeamId(teamId: string): Promise<number>;
@@ -11,6 +11,7 @@ import type {
CheckMetadata,
CheckNetworkInterfaceInfo,
GotTimings,
LocationResult,
MonitorType,
} from "@/types/index.js";
import { CheckModel, MonitorModel, type CheckDocument } from "@/db/models/index.js";
@@ -169,7 +170,6 @@ class MongoChecksRepository implements IChecksRepository {
monitorId: toStringId(metadata.monitorId),
teamId: toStringId(metadata.teamId),
type: metadata.type,
...(metadata.location ? { location: metadata.location } : {}),
});
return {
@@ -195,6 +195,7 @@ class MongoChecksRepository implements IChecksRepository {
seo: doc.seo,
performance: doc.performance,
audits: mapAudits(doc.audits),
locationResults: (doc as any).locationResults,
__v: doc.__v ?? 0,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
@@ -220,7 +221,6 @@ class MongoChecksRepository implements IChecksRepository {
monitorId: new mongoose.Types.ObjectId(metadata.monitorId),
teamId: new mongoose.Types.ObjectId(metadata.teamId),
type: metadata.type,
...(metadata.location ? { location: metadata.location } : {}),
},
...rest,
} as unknown as CheckDocument;
@@ -406,6 +406,10 @@ class MongoChecksRepository implements IChecksRepository {
};
};
updateLocationResults = async (checkId: string, locationResults: LocationResult[]): Promise<void> => {
await CheckModel.updateOne({ _id: new mongoose.Types.ObjectId(checkId) }, { $set: { locationResults } });
};
deleteByMonitorId = async (monitorId: string): Promise<number> => {
const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
return result.deletedCount;
@@ -424,24 +428,25 @@ class MongoChecksRepository implements IChecksRepository {
findLocationChecksByMonitorId = async (monitorId: string, startDate: Date, endDate: Date, dateString: string): Promise<Record<string, any>> => {
const monitorObjectId = new mongoose.Types.ObjectId(monitorId);
const matchStage = {
"metadata.monitorId": monitorObjectId,
"metadata.location": { $exists: true, $ne: null },
createdAt: { $gte: startDate, $lte: endDate },
};
const results = await CheckModel.aggregate([
{ $match: matchStage },
{ $sort: { createdAt: 1 } },
{
$match: {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
locationResults: { $type: "array", $ne: [] },
},
},
{ $unwind: "$locationResults" },
{
$group: {
_id: {
location: "$metadata.location",
location: "$locationResults.location",
bucket: { $dateToString: { format: dateString, date: "$createdAt" } },
},
avgResponseTime: { $avg: "$responseTime" },
avgResponseTime: { $avg: "$locationResults.responseTime" },
totalChecks: { $sum: 1 },
upChecks: { $sum: { $cond: [{ $eq: ["$status", true] }, 1, 0] } },
upChecks: { $sum: { $cond: [{ $eq: ["$locationResults.status", true] }, 1, 0] } },
},
},
{ $sort: { "_id.bucket": 1 } },
@@ -485,7 +490,6 @@ class MongoChecksRepository implements IChecksRepository {
const matchStage: Record<string, any> = {
"metadata.monitorId": monitorObjectId,
createdAt: { $gte: startDate, $lte: endDate },
"metadata.location": { $exists: false },
};
const [result] = await CheckModel.aggregate([
{ $match: matchStage },
+1 -2
View File
@@ -37,7 +37,7 @@ class CheckService {
};
buildCheck = (statusResponse: MonitorStatusResponse<PageSpeedStatusPayload | HardwareStatusPayload | undefined>) => {
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings, location } = statusResponse;
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
const check: Partial<Check> = {
id: new Types.ObjectId().toString(),
@@ -46,7 +46,6 @@ class CheckService {
monitorId,
teamId,
type,
...(location ? { location } : {}),
},
status,
statusCode: code,
@@ -127,33 +127,6 @@ class SuperSimpleQueueHelper {
throw new Error("No network response");
}
// Step 2b. Run Globalping checks (fire-and-forget, doesn't affect local check pipeline)
if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping")) {
this.globalpingService
.runChecks(monitor)
.then((locationStatuses) => {
for (const locStatus of locationStatuses) {
const locCheck = this.checkService.buildCheck(locStatus);
if (locCheck) {
this.buffer.addToBuffer(locCheck);
} else {
this.logger.warn({
message: `Failed to build Globalping check for monitor ${monitorId}, location: ${locStatus.location}`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
}
}
})
.catch((err: any) => {
this.logger.warn({
message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
});
}
// Step 3. Build check
const check = await this.checkService.buildCheck(status);
if (!check) {
@@ -167,6 +140,38 @@ class SuperSimpleQueueHelper {
}
// Step 4. Add check to buffer
this.buffer.addToBuffer(check);
// Step 4b. Run Globalping checks and backfill into the same check document.
// Fire-and-forget: Globalping results are supplementary and must not block the
// local check pipeline. The dual-write strategy below is safe against mid-flush
// races because flushBuffer() replaces the buffer array reference atomically
// (this.buffer = []) — if findInBuffer returns a hit, the check is still in the
// current buffer and our mutation will be included in the upcoming flush; if it
// returns undefined the check has already been flushed, so we fall back to a
// direct DB update.
if (monitor.globalpingEnabled && (monitor.type === "http" || monitor.type === "ping") && check.id) {
const checkId = check.id;
this.globalpingService
.runChecks(monitor)
.then(async (locationResults) => {
if (locationResults.length > 0) {
const buffered = this.buffer.findInBuffer(checkId);
if (buffered) {
buffered.locationResults = locationResults;
} else {
await this.checksRepository.updateLocationResults(checkId, locationResults);
}
}
})
.catch((err: any) => {
this.logger.warn({
message: `Globalping checks failed for monitor ${monitorId}: ${err.message}`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
});
}
// Step 5. Update monitor status
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
@@ -5,6 +5,7 @@ const SERVICE_NAME = "BufferService";
export interface IBufferService {
addToBuffer(check: Check): void;
removeCheckFromBuffer(check: Check): boolean;
findInBuffer(checkId: string): Check | undefined;
scheduleNextFlush(): void;
flushBuffer(): Promise<void>;
}
@@ -49,6 +50,10 @@ class BufferService implements IBufferService {
}
}
findInBuffer(checkId: string): Check | undefined {
return this.buffer.find((c: Check) => c.id?.toString() === checkId);
}
removeCheckFromBuffer(checkToRemove: Check) {
try {
if (!checkToRemove) {
@@ -1,7 +1,7 @@
import { Globalping } from "globalping";
import type { ISettingsService } from "@/service/system/settingsService.js";
import type { Monitor } from "@/types/monitor.js";
import type { MonitorStatusResponse } from "@/types/network.js";
import type { GlobalpingCheckResult } from "@/types/network.js";
const SERVICE_NAME = "GlobalpingService";
@@ -130,7 +130,7 @@ class GlobalpingService {
return { client: this.client, locationIds };
}
async runChecks(monitor: Monitor): Promise<MonitorStatusResponse[]> {
async runChecks(monitor: Monitor): Promise<GlobalpingCheckResult[]> {
if (monitor.type !== "http" && monitor.type !== "ping") {
return [];
}
@@ -148,7 +148,7 @@ class GlobalpingService {
}
}
private async executeChecks(monitor: Monitor): Promise<MonitorStatusResponse[]> {
private async executeChecks(monitor: Monitor): Promise<GlobalpingCheckResult[]> {
const { client, locationIds } = await this.getClientAndLocations();
if (!client || locationIds.length === 0) {
return [];
@@ -224,7 +224,7 @@ class GlobalpingService {
}
const measurement = awaitResult.data as any;
const results: MonitorStatusResponse[] = [];
const results: GlobalpingCheckResult[] = [];
for (const item of measurement.results || []) {
const probe = item.probe;
@@ -233,27 +233,21 @@ class GlobalpingService {
if (measurementType === "http") {
results.push({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
location: locationId,
status: result.statusCode >= 200 && result.statusCode < 400,
code: result.statusCode || 0,
statusCode: result.statusCode || 0,
message: result.statusCodeName || result.rawOutput || "",
responseTime: result.timings?.total ?? 0,
location: locationId,
});
} else {
// Ping: loss < 100% is considered up (consistent with local ping checks)
const stats = result.stats;
results.push({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
location: locationId,
status: stats?.loss !== undefined ? stats.loss < 100 : false,
code: stats?.loss === 0 ? 200 : 5000,
statusCode: stats?.loss === 0 ? 200 : 5000,
message: stats?.loss === 0 ? "Ping successful" : `Packet loss: ${stats?.loss}%`,
responseTime: stats?.avg ?? 0,
location: locationId,
});
}
}
-1
View File
@@ -6,7 +6,6 @@ export interface CheckMetadata {
monitorId: string;
teamId: string;
type: MonitorType;
location?: string;
}
export interface LocationResult {
-1
View File
@@ -34,7 +34,6 @@ export interface MonitorStatusResponse<T = any> {
matchMethod?: MonitorMatchMethod;
expectedValue?: string;
extracted?: unknown;
location?: string;
}
export interface GlobalpingCheckResult {