-> timeseries

This commit is contained in:
Alex Holliday committed 2026-01-12 19:32:23 +00:00
1 parent 5c14c68899
commit e090d3cfd3
91 files changed
+941 -987

No files matched your search

File renamed without changes.
@@ -0,0 +1,10 @@
import Monitor from "@/db/models/Monitor.js";
async function migrateStatusWindowThreshold() {
const monitors = await Monitor.find({ statusWindowThreshold: { $lt: 1 } });
for (const monitor of monitors) {
monitor.statusWindowThreshold = monitor.statusWindowThreshold * 100;
await monitor.save();
}
}
export { migrateStatusWindowThreshold };
@@ -0,0 +1,93 @@
import mongoose from "mongoose";
const CHECKS_COLLECTION = "checks";
const BACKUP_COLLECTION = "checks_backup";
const BATCH_SIZE = 1000;
const getDb = () => {
const db = mongoose.connection.db;
if (!db) {
throw new Error("Database connection is not initialized");
}
return db;
};
const backupAndDropExistingCollection = async () => {
const db = getDb();
const collections = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
if (collections.length === 0) {
return false;
}
await db.collection(CHECKS_COLLECTION).aggregate([
{ $match: {} },
{ $out: BACKUP_COLLECTION },
]).toArray();
await db.collection(CHECKS_COLLECTION).drop();
return true;
};
const createTimeSeriesCollection = async () => {
const db = getDb();
const existing = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
if (existing.length === 0) {
await db.createCollection(CHECKS_COLLECTION, {
timeseries: {
timeField: "createdAt",
metaField: "metadata",
granularity: "seconds",
},
});
}
await db.collection(CHECKS_COLLECTION).createIndexes([
{ key: { updatedAt: 1 } },
{ key: { "metadata.monitorId": 1, updatedAt: 1 } },
{ key: { "metadata.monitorId": 1, updatedAt: -1 } },
{ key: { "metadata.teamId": 1, updatedAt: -1 } },
]);
};
const migrateBackupData = async () => {
const db = getDb();
const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray();
if (backupExists.length === 0) {
return;
}
const source = db.collection(BACKUP_COLLECTION);
const target = db.collection(CHECKS_COLLECTION);
const cursor = source.find();
const operations: any[] = [];
while (await cursor.hasNext()) {
const doc = await cursor.next();
if (!doc) {
continue;
}
const metadata = {
monitorId: doc.monitorId,
teamId: doc.teamId,
type: doc.type,
};
operations.push({ insertOne: { document: { ...doc, metadata } } });
if (operations.length >= BATCH_SIZE) {
await target.bulkWrite(operations);
operations.length = 0;
}
}
if (operations.length) {
await target.bulkWrite(operations);
}
};
export const convertChecksToTimeSeries = async () => {
const backedUp = await backupAndDropExistingCollection();
await createTimeSeriesCollection();
if (backedUp) {
await migrateBackupData();
}
};
+51
View File
@@ -0,0 +1,51 @@
import { migrateStatusWindowThreshold } from "./0001_migrateStatusWindowThreshold.js";
import { convertChecksToTimeSeries } from "./0002_convertChecksToTimeSeries.js";
import MigrationModel from "../models/Migration.js";
type MigrationEntry = {
name: string;
execute: () => Promise<void>;
};
const migrations: MigrationEntry[] = [
{ name: "0001_migrateStatusWindowThreshold", execute: migrateStatusWindowThreshold },
{ name: "0002_convertChecksToTimeSeries", execute: convertChecksToTimeSeries },
];
const runMigrations = async (logger?: { info: Function; error: Function }) => {
try {
logger?.info({ message: "Running migrations", service: "Migrations" });
for (const migration of migrations) {
const exists = await MigrationModel.findOne({ name: migration.name, status: "completed" });
if (exists) {
logger?.info({ message: `Skipping ${migration.name}`, service: "Migrations" });
continue;
}
try {
await migration.execute();
await MigrationModel.findOneAndUpdate(
{ name: migration.name },
{ status: "completed", completedAt: new Date(), error: undefined },
{ upsert: true }
);
logger?.info({ message: `Completed ${migration.name}`, service: "Migrations" });
} catch (error) {
const err = error as Error;
await MigrationModel.findOneAndUpdate(
{ name: migration.name },
{ status: "failed", error: err?.message },
{ upsert: true }
);
throw error;
}
}
logger?.info({ message: "Migrations completed", service: "Migrations" });
} catch (error) {
const err = error as Error;
logger?.error({ message: "Migration failed", service: "Migrations", details: err?.message, stack: err?.stack });
throw error;
}
};
export { runMigrations };
File renamed without changes.
+214
View File
@@ -0,0 +1,214 @@
import mongoose from "mongoose";
import { MonitorTypes } from "@/types/monitor.js";
const cpuSchema = new mongoose.Schema(
{
physical_core: { type: Number, default: 0 },
logical_core: { type: Number, default: 0 },
frequency: { type: Number, default: 0 },
temperature: { type: [Number], default: [] },
free_percent: { type: Number, default: 0 },
usage_percent: { type: Number, default: 0 },
},
{ _id: false }
);
const memorySchema = new mongoose.Schema(
{
total_bytes: { type: Number, default: 0 },
available_bytes: { type: Number, default: 0 },
used_bytes: { type: Number, default: 0 },
usage_percent: { type: Number, default: 0 },
},
{ _id: false }
);
const diskSchema = new mongoose.Schema(
{
device: { type: String, default: "" },
mountpoint: { type: String, default: "" },
read_speed_bytes: { type: Number, default: 0 },
write_speed_bytes: { type: Number, default: 0 },
total_bytes: { type: Number, default: 0 },
free_bytes: { type: Number, default: 0 },
usage_percent: { type: Number, default: 0 },
},
{ _id: false }
);
const hostSchema = new mongoose.Schema(
{
os: { type: String, default: "" },
platform: { type: String, default: "" },
kernel_version: { type: String, default: "" },
},
{ _id: false }
);
const errorSchema = new mongoose.Schema(
{
metric: { type: [String], default: [] },
err: { type: String, default: "" },
},
{ _id: false }
);
const captureSchema = new mongoose.Schema(
{
version: { type: String, default: "" },
mode: { type: String, default: "" },
},
{ _id: false }
);
const networkInterfaceSchema = new mongoose.Schema(
{
name: { type: String },
bytes_sent: { type: Number, default: 0 },
bytes_recv: { type: Number, default: 0 },
packets_sent: { type: Number, default: 0 },
packets_recv: { type: Number, default: 0 },
err_in: { type: Number, default: 0 },
err_out: { type: Number, default: 0 },
drop_in: { type: Number, default: 0 },
drop_out: { type: Number, default: 0 },
fifo_in: { type: Number, default: 0 },
fifo_out: { type: Number, default: 0 },
},
{ _id: false }
);
const metadataSchema = new mongoose.Schema(
{
monitorId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Monitor",
immutable: true,
index: true,
},
teamId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Team",
immutable: true,
index: true,
},
type: {
type: String,
enum: MonitorTypes,
required: true,
index: true,
},
},
{ _id: false }
);
const CheckSchema = new mongoose.Schema(
{
metadata: {
type: metadataSchema,
required: true,
},
status: {
type: Boolean,
index: true,
},
responseTime: {
type: Number,
},
timings: {
type: Object,
default: {},
},
statusCode: {
type: Number,
index: true,
},
message: {
type: String,
},
expiry: {
type: Date,
default: Date.now,
},
ack: {
type: Boolean,
default: false,
},
ackAt: {
type: Date,
},
// Hardware fields
cpu: {
type: cpuSchema,
default: () => ({}),
},
memory: {
type: memorySchema,
default: () => ({}),
},
disk: {
type: [diskSchema],
default: () => [],
},
host: {
type: hostSchema,
default: () => ({}),
},
errors: {
type: [errorSchema],
default: () => [],
},
capture: {
type: captureSchema,
default: () => ({}),
},
net: {
type: [networkInterfaceSchema],
default: () => [],
},
// PageSpeed fields
accessibility: {
type: Number,
},
bestPractices: {
type: Number,
},
seo: {
type: Number,
},
performance: {
type: Number,
},
audits: {
type: Object,
},
},
{
timestamps: true,
strict: false,
timeseries: {
timeField: "createdAt",
metaField: "metadata",
granularity: "seconds",
},
}
);
CheckSchema.index({ updatedAt: 1 });
CheckSchema.index({ "metadata.monitorId": 1, updatedAt: 1 });
CheckSchema.index({ "metadata.monitorId": 1, updatedAt: -1 });
CheckSchema.index({ "metadata.teamId": 1, updatedAt: -1 });
export default mongoose.model("Check", CheckSchema);
File renamed without changes.
File renamed without changes.
+13
View File
@@ -0,0 +1,13 @@
import mongoose from "mongoose";
const MigrationSchema = new mongoose.Schema(
{
name: { type: String, required: true, unique: true },
status: { type: String, enum: ["completed", "failed"], default: "completed" },
completedAt: { type: Date },
error: { type: String },
},
{ timestamps: true }
);
export default mongoose.model("Migration", MigrationSchema);
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,18 +0,0 @@
// import Monitor from "@/db/v1/models/Monitor.js";
async function migrateStatusWindowThreshold() {
try {
// const monitors = await Monitor.find({ statusWindowThreshold: { $lt: 1 } });
// for (const monitor of monitors) {
// monitor.statusWindowThreshold = monitor.statusWindowThreshold * 100;
// await monitor.save();
// console.log(`Migrated monitor ${monitor._id}: statusWindowThreshold set to ${monitor.statusWindowThreshold}`);
// }
// console.log("StatusWindowThreshold migration complete.");
return true;
} catch (err) {
console.error("Migration error:", err);
return false;
}
}
export { migrateStatusWindowThreshold };
-7
View File
@@ -1,7 +0,0 @@
import { migrateStatusWindowThreshold } from "./0001_migrateStatusWindowThreshold.js";
const runMigrations = async () => {
await migrateStatusWindowThreshold();
};
export { runMigrations };
-178
View File
@@ -1,178 +0,0 @@
import mongoose from "mongoose";
const cpuSchema = mongoose.Schema({
physical_core: { type: Number, default: 0 },
logical_core: { type: Number, default: 0 },
frequency: { type: Number, default: 0 },
temperature: { type: [Number], default: [] },
free_percent: { type: Number, default: 0 },
usage_percent: { type: Number, default: 0 },
});
const memorySchema = mongoose.Schema({
total_bytes: { type: Number, default: 0 },
available_bytes: { type: Number, default: 0 },
used_bytes: { type: Number, default: 0 },
usage_percent: { type: Number, default: 0 },
});
const diskSchema = mongoose.Schema({
device: { type: String, default: "" },
mountpoint: { type: String, default: "" },
read_speed_bytes: { type: Number, default: 0 },
write_speed_bytes: { type: Number, default: 0 },
total_bytes: { type: Number, default: 0 },
free_bytes: { type: Number, default: 0 },
usage_percent: { type: Number, default: 0 },
});
const hostSchema = mongoose.Schema({
os: { type: String, default: "" },
platform: { type: String, default: "" },
kernel_version: { type: String, default: "" },
});
const errorSchema = mongoose.Schema({
metric: { type: [String], default: [] },
err: { type: String, default: "" },
});
const captureSchema = mongoose.Schema({
version: { type: String, default: "" },
mode: { type: String, default: "" },
});
const networkInterfaceSchema = mongoose.Schema({
name: { type: String },
bytes_sent: { type: Number, default: 0 },
bytes_recv: { type: Number, default: 0 },
packets_sent: { type: Number, default: 0 },
packets_recv: { type: Number, default: 0 },
err_in: { type: Number, default: 0 },
err_out: { type: Number, default: 0 },
drop_in: { type: Number, default: 0 },
drop_out: { type: Number, default: 0 },
fifo_in: { type: Number, default: 0 },
fifo_out: { type: Number, default: 0 },
});
const CheckSchema = new mongoose.Schema(
{
// Common fields
monitorId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Monitor",
immutable: true,
index: true,
},
teamId: {
type: mongoose.Schema.Types.ObjectId,
ref: "Team",
immutable: true,
index: true,
},
type: {
type: String,
enum: ["http", "ping", "pagespeed", "hardware", "docker", "port", "game"],
required: true,
index: true,
},
status: {
type: Boolean,
index: true,
},
responseTime: {
type: Number,
},
timings: {
type: Object,
default: {},
},
statusCode: {
type: Number,
index: true,
},
message: {
type: String,
},
expiry: {
type: Date,
default: Date.now,
expires: 60 * 60 * 24 * 30, // 30 days
},
ack: {
type: Boolean,
default: false,
},
ackAt: {
type: Date,
},
// Hardware fields
cpu: {
type: cpuSchema,
default: () => ({}),
},
memory: {
type: memorySchema,
default: () => ({}),
},
disk: {
type: [diskSchema],
default: () => [],
},
host: {
type: hostSchema,
default: () => ({}),
},
errors: {
type: [errorSchema],
default: () => [],
},
capture: {
type: captureSchema,
default: () => ({}),
},
net: {
type: [networkInterfaceSchema],
default: () => [],
},
// PageSpeed fields
accessibility: {
type: Number,
},
bestPractices: {
type: Number,
},
seo: {
type: Number,
},
performance: {
type: Number,
},
audits: {
type: Object,
},
},
{ timestamps: true }
);
CheckSchema.index({ updatedAt: 1 });
CheckSchema.index({ monitorId: 1, updatedAt: 1 });
CheckSchema.index({ monitorId: 1, updatedAt: -1 });
CheckSchema.index({ teamId: 1, updatedAt: -1 });
export default mongoose.model("Check", CheckSchema);