From ed54fa4668e25d922660b6b38a6df30bfe66fba0 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 29 Jul 2026 12:30:18 -0700 Subject: [PATCH] first byte to total repsonse --- .../0009_recomputeResponseTimeFromTimings.ts | 209 ++++++++++++++++++ server/src/db/migration/index.ts | 2 + server/src/service/network/HttpProvider.ts | 4 +- 3 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 server/src/db/migration/0009_recomputeResponseTimeFromTimings.ts diff --git a/server/src/db/migration/0009_recomputeResponseTimeFromTimings.ts b/server/src/db/migration/0009_recomputeResponseTimeFromTimings.ts new file mode 100644 index 000000000..e65812530 --- /dev/null +++ b/server/src/db/migration/0009_recomputeResponseTimeFromTimings.ts @@ -0,0 +1,209 @@ +import mongoose from "mongoose"; +import type { Document } from "mongodb"; +import type { ILogger } from "@/utils/logger.js"; + +const SERVICE_NAME = "Migration:RecomputeResponseTimeFromTimings"; +const CHECKS_COLLECTION = "checks"; +const BACKUP_COLLECTION = "checks_backup_response_time"; +const FAILED_DOCS_COLLECTION = "checks_response_time_migration_failed"; +const BATCH_SIZE = 1000; +const NAMESPACE_EXISTS_CODE = 48; + +interface MigrationStats { + totalSource: number; + totalMigrated: number; + totalFailed: number; +} + +const getDb = () => { + const db = mongoose.connection.db; + if (!db) { + throw new Error("Database connection is not initialized"); + } + return db; +}; + +// `checks` is a time-series collection (0002): measurement fields cannot be updated in place, +// so recomputing responseTime requires a backup → drop → recreate → re-insert rewrite. +const createTimeSeriesCollection = async () => { + const db = getDb(); + const existing = await db.listCollections({ name: CHECKS_COLLECTION }).toArray(); + if (existing.length === 0) { + try { + await db.createCollection(CHECKS_COLLECTION, { + timeseries: { + timeField: "createdAt", + metaField: "metadata", + granularity: "seconds", + }, + }); + } catch (error) { + // Mongoose's autoCreate can create `checks` concurrently with this migration + if ((error as { code?: number })?.code !== NAMESPACE_EXISTS_CODE) { + throw error; + } + } + } + + await db + .collection(CHECKS_COLLECTION) + .createIndexes([ + { key: { createdAt: 1 } }, + { key: { "metadata.monitorId": 1, createdAt: 1 } }, + { key: { "metadata.monitorId": 1, createdAt: -1 } }, + { key: { "metadata.teamId": 1, createdAt: -1 } }, + { key: { "metadata.monitorId": 1, "metadata.type": 1, createdAt: -1 } }, + { key: { "metadata.teamId": 1, status: 1, createdAt: -1 } }, + ]); +}; + +const stripId = (doc: Document): Document => { + const { _id, ...rest } = doc; + void _id; + return rest; +}; + +const recomputeResponseTime = (doc: Document): Document => { + const rest = stripId(doc); + const total = doc.timings?.phases?.total; + if (typeof total === "number") { + rest.responseTime = total; + } + return rest; +}; + +const storeFailedDocument = async (doc: Document, reason: string) => { + const db = getDb(); + await db.collection(FAILED_DOCS_COLLECTION).insertOne({ + originalDoc: doc, + reason, + failedAt: new Date(), + }); +}; + +const streamBackupIntoChecks = async (transform: (doc: Document) => Document, collectFailures: boolean): Promise => { + const db = getDb(); + const source = db.collection(BACKUP_COLLECTION); + const target = db.collection(CHECKS_COLLECTION); + + const stats: MigrationStats = { + totalSource: await source.countDocuments(), + totalMigrated: 0, + totalFailed: 0, + }; + + const cursor = source.find(); + // Derive the op type from the exact bulkWrite we call so it matches whichever `mongodb` copy + // Mongoose resolves (see 0002 for the nested-node_modules background) + type BulkOp = Parameters[0][number]; + const operations: BulkOp[] = []; + const invalidDocs: Document[] = []; + + while (await cursor.hasNext()) { + const doc = await cursor.next(); + if (!doc) { + continue; + } + + // Time-series inserts reject documents without the timeField + if (!doc.createdAt) { + invalidDocs.push(doc); + continue; + } + + operations.push({ insertOne: { document: transform(doc) } }); + + if (operations.length >= BATCH_SIZE) { + const result = await target.bulkWrite(operations, { ordered: false }); + stats.totalMigrated += result.insertedCount; + operations.length = 0; + } + } + + if (operations.length) { + const result = await target.bulkWrite(operations, { ordered: false }); + stats.totalMigrated += result.insertedCount; + } + + if (collectFailures) { + for (const doc of invalidDocs) { + await storeFailedDocument(doc, "Missing createdAt (required timeField)"); + stats.totalFailed++; + } + } else { + stats.totalFailed = invalidDocs.length; + } + + await cursor.close(); + + return stats; +}; + +// Restore cannot be a simple rename like 0002's: $out produced a standard collection, and +// renaming it to `checks` would silently lose the time-series type. Rebuild and re-insert instead. +const restoreFromBackup = async () => { + const db = getDb(); + const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray(); + if (backupExists.length === 0) { + throw new Error("Cannot restore: backup collection does not exist"); + } + + const checksExists = await db.listCollections({ name: CHECKS_COLLECTION }).toArray(); + if (checksExists.length > 0) { + await db.collection(CHECKS_COLLECTION).drop(); + } + + await createTimeSeriesCollection(); + await streamBackupIntoChecks(stripId, false); +}; + +export async function recomputeResponseTimeFromTimings(logger: ILogger): Promise { + const db = getDb(); + + const checksExists = await db.listCollections({ name: CHECKS_COLLECTION }).toArray(); + if (checksExists.length === 0) { + logger.info({ service: SERVICE_NAME, message: "No checks collection, nothing to migrate" }); + return; + } + + // Fresh installs and non-HTTP-only deployments have no stored timings — skip the rewrite entirely + const needsMigration = await db.collection(CHECKS_COLLECTION).countDocuments({ "timings.phases.total": { $exists: true } }, { limit: 1 }); + if (needsMigration === 0) { + logger.info({ service: SERVICE_NAME, message: "No checks with timings.phases.total, nothing to migrate" }); + return; + } + + const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray(); + if (backupExists.length > 0) { + throw new Error(`Backup collection "${BACKUP_COLLECTION}" already exists. Please remove it manually before running migration.`); + } + + await db + .collection(CHECKS_COLLECTION) + .aggregate([{ $match: {} }, { $out: BACKUP_COLLECTION }]) + .toArray(); + await db.collection(CHECKS_COLLECTION).drop(); + + try { + await createTimeSeriesCollection(); + const stats = await streamBackupIntoChecks(recomputeResponseTime, true); + logger.info({ + service: SERVICE_NAME, + message: + `Migrated ${stats.totalMigrated}/${stats.totalSource} checks, ${stats.totalFailed} failed ` + + `(failures stored in ${FAILED_DOCS_COLLECTION}); backup retained in ${BACKUP_COLLECTION}`, + }); + } catch (error) { + try { + await restoreFromBackup(); + logger.error({ service: SERVICE_NAME, message: "Migration failed, checks restored from backup" }); + } catch (restoreError) { + const restoreMessage = restoreError instanceof Error ? restoreError.message : String(restoreError); + logger.error({ + service: SERVICE_NAME, + message: `Migration failed AND rollback failed (${restoreMessage}) — original data is preserved in ${BACKUP_COLLECTION}`, + }); + } + throw error; + } +} diff --git a/server/src/db/migration/index.ts b/server/src/db/migration/index.ts index 0b0745d19..f59af32b7 100644 --- a/server/src/db/migration/index.ts +++ b/server/src/db/migration/index.ts @@ -7,6 +7,7 @@ import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArra import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js"; import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js"; import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js"; +import { recomputeResponseTimeFromTimings } from "./0009_recomputeResponseTimeFromTimings.js"; import type { ILogger } from "@/utils/logger.js"; type MigrationEntry = { @@ -23,6 +24,7 @@ const migrations: MigrationEntry[] = [ { name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex }, { name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray }, { name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt }, + { name: "0009_recomputeResponseTimeFromTimings", execute: recomputeResponseTimeFromTimings }, ]; const runMigrations = async (logger: ILogger) => { diff --git a/server/src/service/network/HttpProvider.ts b/server/src/service/network/HttpProvider.ts index b1debd969..fb5f9fea5 100644 --- a/server/src/service/network/HttpProvider.ts +++ b/server/src/service/network/HttpProvider.ts @@ -117,7 +117,7 @@ export class HttpProvider implements IStatusProvider { if (error instanceof HTTPError || error instanceof RequestError) { const statusCode = error.response?.statusCode; const statusUp = isStatusUp(statusCode, monitor.customUpCodes); - const responseTime = error.timings?.phases?.firstByte ?? error.timings?.phases?.total ?? 0; + const responseTime = error.timings?.phases?.total ?? 0; if (!statusUp) { return { @@ -184,7 +184,7 @@ export class HttpProvider implements IStatusProvider { statusCode: response.statusCode, statusUp, message: response.statusMessage ?? "OK", - responseTime: response.timings.phases.firstByte ?? 0, + responseTime: response.timings.phases.total ?? 0, timings: response.timings, }); } catch (error: unknown) {