Files
Checkmate/server/src/db/migration/0002_convertChecksToTimeSeries.ts
T

183 lines
4.9 KiB
TypeScript
Raw Normal View History

2026-01-12 19:32:23 +00:00
import mongoose from "mongoose";
const CHECKS_COLLECTION = "checks";
const BACKUP_COLLECTION = "checks_backup";
2026-01-26 23:02:30 +00:00
const FAILED_DOCS_COLLECTION = "checks_migration_failed";
2026-01-12 19:32:23 +00:00
const BATCH_SIZE = 1000;
2026-01-26 23:02:30 +00:00
interface MigrationStats {
totalSource: number;
totalMigrated: number;
totalFailed: number;
}
2026-01-12 19:32:23 +00:00
const getDb = () => {
const db = mongoose.connection.db;
if (!db) {
throw new Error("Database connection is not initialized");
}
return db;
};
2026-01-26 23:02:30 +00:00
const restoreFromBackup = async () => {
const db = getDb();
const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray();
if (backupExists.length === 0) {
throw new Error("Cannot restore: backup collection does not exist");
}
const checksExists = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
if (checksExists.length > 0) {
await db.collection(CHECKS_COLLECTION).drop();
}
await db.collection(BACKUP_COLLECTION).rename(CHECKS_COLLECTION);
};
2026-01-12 19:32:23 +00:00
const backupAndDropExistingCollection = async () => {
const db = getDb();
const collections = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
if (collections.length === 0) {
return false;
}
2026-01-26 23:02:30 +00:00
const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray();
if (backupExists.length > 0) {
2026-01-26 23:21:49 +00:00
throw new Error(`Backup collection "${BACKUP_COLLECTION}" already exists. ` + `Please remove it manually before running migration.`);
2026-01-26 23:02:30 +00:00
}
2026-01-12 19:32:23 +00:00
2026-01-13 20:20:29 +00:00
await db
.collection(CHECKS_COLLECTION)
.aggregate([{ $match: {} }, { $out: BACKUP_COLLECTION }])
.toArray();
2026-01-12 19:32:23 +00:00
await db.collection(CHECKS_COLLECTION).drop();
return true;
};
2026-01-26 23:02:30 +00:00
const createTimeSeriesCollection = async (backedUp: boolean) => {
2026-01-12 19:32:23 +00:00
const db = getDb();
2026-01-26 23:02:30 +00:00
try {
const existing = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
if (existing.length === 0) {
await db.createCollection(CHECKS_COLLECTION, {
timeseries: {
timeField: "createdAt",
metaField: "metadata",
granularity: "seconds",
},
});
}
await db
.collection(CHECKS_COLLECTION)
.createIndexes([
{ key: { createdAt: 1 } },
{ key: { "metadata.monitorId": 1, createdAt: 1 } },
{ key: { "metadata.monitorId": 1, createdAt: -1 } },
{ key: { "metadata.teamId": 1, createdAt: -1 } },
2026-01-26 23:20:18 +00:00
{ key: { "metadata.monitorId": 1, "metadata.type": 1, createdAt: -1 } },
{ key: { "metadata.teamId": 1, status: 1, createdAt: -1 } },
2026-01-26 23:02:30 +00:00
]);
} catch (error) {
if (backedUp) {
await restoreFromBackup();
}
throw error;
2026-01-12 19:32:23 +00:00
}
2026-01-26 23:02:30 +00:00
};
2026-01-12 19:32:23 +00:00
2026-01-26 23:02:30 +00:00
const validateDocument = (doc: any): boolean => {
if (!doc.createdAt) {
return false;
}
if (!doc.monitorId || !doc.teamId || !doc.type) {
return false;
}
return true;
2026-01-12 19:32:23 +00:00
};
2026-01-26 23:02:30 +00:00
const storeFailedDocument = async (doc: any, reason: string) => {
const db = getDb();
await db.collection(FAILED_DOCS_COLLECTION).insertOne({
originalDoc: doc,
reason,
failedAt: new Date(),
});
};
const migrateBackupData = async (backedUp: boolean): Promise<MigrationStats> => {
2026-01-12 19:32:23 +00:00
const db = getDb();
const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray();
if (backupExists.length === 0) {
2026-01-26 23:02:30 +00:00
return { totalSource: 0, totalMigrated: 0, totalFailed: 0 };
2026-01-12 19:32:23 +00:00
}
2026-01-26 23:02:30 +00:00
const stats: MigrationStats = {
totalSource: 0,
totalMigrated: 0,
totalFailed: 0,
};
try {
const source = db.collection(BACKUP_COLLECTION);
const target = db.collection(CHECKS_COLLECTION);
stats.totalSource = await source.countDocuments();
2026-01-12 19:32:23 +00:00
2026-01-26 23:02:30 +00:00
const cursor = source.find().addCursorFlag("noCursorTimeout", true);
const operations: any[] = [];
const invalidDocs: any[] = [];
while (await cursor.hasNext()) {
const doc = await cursor.next();
if (!doc) {
continue;
}
if (!validateDocument(doc)) {
invalidDocs.push(doc);
continue;
}
const { _id, monitorId, teamId, type, ...rest } = doc;
const metadata = { monitorId, teamId, type };
operations.push({ insertOne: { document: { ...rest, metadata } } });
if (operations.length >= BATCH_SIZE) {
2026-01-27 21:17:57 +00:00
const result = await target.bulkWrite(operations, { ordered: false });
2026-01-26 23:02:30 +00:00
stats.totalMigrated += result.insertedCount;
operations.length = 0;
}
2026-01-12 19:32:23 +00:00
}
2026-01-26 23:02:30 +00:00
if (operations.length) {
2026-01-27 21:17:57 +00:00
const result = await target.bulkWrite(operations, { ordered: false });
2026-01-26 23:02:30 +00:00
stats.totalMigrated += result.insertedCount;
}
2026-01-12 19:32:23 +00:00
2026-01-26 23:02:30 +00:00
for (const doc of invalidDocs) {
await storeFailedDocument(doc, "Missing required fields (createdAt, monitorId, teamId, or type)");
stats.totalFailed++;
2026-01-12 19:32:23 +00:00
}
2026-01-26 23:02:30 +00:00
await cursor.close();
return stats;
} catch (error) {
if (backedUp) {
await restoreFromBackup();
}
throw error;
2026-01-12 19:32:23 +00:00
}
};
export const convertChecksToTimeSeries = async () => {
const backedUp = await backupAndDropExistingCollection();
2026-01-26 23:02:30 +00:00
await createTimeSeriesCollection(backedUp);
2026-01-12 19:32:23 +00:00
if (backedUp) {
2026-01-26 23:02:30 +00:00
const stats = await migrateBackupData(backedUp);
console.log(
`Migration completed: ${stats.totalMigrated} documents migrated, ` +
2026-01-26 23:21:49 +00:00
`${stats.totalFailed} documents failed (stored in ${FAILED_DOCS_COLLECTION})`
2026-01-26 23:02:30 +00:00
);
2026-01-12 19:32:23 +00:00
}
};