-> timeseries
This commit is contained in:
1 parent
5c14c68899
commit
e090d3cfd3
91 files changed
+941
-987
No files matched your search
File renamed without changes.
@@ -0,0 +1,10 @@
|
||||
import Monitor from "@/db/models/Monitor.js";
|
||||
async function migrateStatusWindowThreshold() {
|
||||
const monitors = await Monitor.find({ statusWindowThreshold: { $lt: 1 } });
|
||||
for (const monitor of monitors) {
|
||||
monitor.statusWindowThreshold = monitor.statusWindowThreshold * 100;
|
||||
await monitor.save();
|
||||
}
|
||||
}
|
||||
|
||||
export { migrateStatusWindowThreshold };
|
||||
@@ -0,0 +1,93 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const CHECKS_COLLECTION = "checks";
|
||||
const BACKUP_COLLECTION = "checks_backup";
|
||||
const BATCH_SIZE = 1000;
|
||||
|
||||
const getDb = () => {
|
||||
const db = mongoose.connection.db;
|
||||
if (!db) {
|
||||
throw new Error("Database connection is not initialized");
|
||||
}
|
||||
return db;
|
||||
};
|
||||
|
||||
const backupAndDropExistingCollection = async () => {
|
||||
const db = getDb();
|
||||
const collections = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
|
||||
if (collections.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await db.collection(CHECKS_COLLECTION).aggregate([
|
||||
{ $match: {} },
|
||||
{ $out: BACKUP_COLLECTION },
|
||||
]).toArray();
|
||||
await db.collection(CHECKS_COLLECTION).drop();
|
||||
return true;
|
||||
};
|
||||
|
||||
const createTimeSeriesCollection = async () => {
|
||||
const db = getDb();
|
||||
const existing = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
|
||||
if (existing.length === 0) {
|
||||
await db.createCollection(CHECKS_COLLECTION, {
|
||||
timeseries: {
|
||||
timeField: "createdAt",
|
||||
metaField: "metadata",
|
||||
granularity: "seconds",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await db.collection(CHECKS_COLLECTION).createIndexes([
|
||||
{ key: { updatedAt: 1 } },
|
||||
{ key: { "metadata.monitorId": 1, updatedAt: 1 } },
|
||||
{ key: { "metadata.monitorId": 1, updatedAt: -1 } },
|
||||
{ key: { "metadata.teamId": 1, updatedAt: -1 } },
|
||||
]);
|
||||
};
|
||||
|
||||
const migrateBackupData = async () => {
|
||||
const db = getDb();
|
||||
const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray();
|
||||
if (backupExists.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const source = db.collection(BACKUP_COLLECTION);
|
||||
const target = db.collection(CHECKS_COLLECTION);
|
||||
const cursor = source.find();
|
||||
const operations: any[] = [];
|
||||
|
||||
while (await cursor.hasNext()) {
|
||||
const doc = await cursor.next();
|
||||
if (!doc) {
|
||||
continue;
|
||||
}
|
||||
const metadata = {
|
||||
monitorId: doc.monitorId,
|
||||
teamId: doc.teamId,
|
||||
type: doc.type,
|
||||
};
|
||||
|
||||
operations.push({ insertOne: { document: { ...doc, metadata } } });
|
||||
|
||||
if (operations.length >= BATCH_SIZE) {
|
||||
await target.bulkWrite(operations);
|
||||
operations.length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (operations.length) {
|
||||
await target.bulkWrite(operations);
|
||||
}
|
||||
};
|
||||
|
||||
export const convertChecksToTimeSeries = async () => {
|
||||
const backedUp = await backupAndDropExistingCollection();
|
||||
await createTimeSeriesCollection();
|
||||
if (backedUp) {
|
||||
await migrateBackupData();
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,51 @@
|
||||
import { migrateStatusWindowThreshold } from "./0001_migrateStatusWindowThreshold.js";
|
||||
import { convertChecksToTimeSeries } from "./0002_convertChecksToTimeSeries.js";
|
||||
import MigrationModel from "../models/Migration.js";
|
||||
|
||||
type MigrationEntry = {
|
||||
name: string;
|
||||
execute: () => Promise<void>;
|
||||
};
|
||||
|
||||
const migrations: MigrationEntry[] = [
|
||||
{ name: "0001_migrateStatusWindowThreshold", execute: migrateStatusWindowThreshold },
|
||||
{ name: "0002_convertChecksToTimeSeries", execute: convertChecksToTimeSeries },
|
||||
];
|
||||
|
||||
const runMigrations = async (logger?: { info: Function; error: Function }) => {
|
||||
try {
|
||||
logger?.info({ message: "Running migrations", service: "Migrations" });
|
||||
for (const migration of migrations) {
|
||||
const exists = await MigrationModel.findOne({ name: migration.name, status: "completed" });
|
||||
if (exists) {
|
||||
logger?.info({ message: `Skipping ${migration.name}`, service: "Migrations" });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await migration.execute();
|
||||
await MigrationModel.findOneAndUpdate(
|
||||
{ name: migration.name },
|
||||
{ status: "completed", completedAt: new Date(), error: undefined },
|
||||
{ upsert: true }
|
||||
);
|
||||
logger?.info({ message: `Completed ${migration.name}`, service: "Migrations" });
|
||||
} catch (error) {
|
||||
const err = error as Error;
|
||||
await MigrationModel.findOneAndUpdate(
|
||||
{ name: migration.name },
|
||||
{ status: "failed", error: err?.message },
|
||||
{ upsert: true }
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
logger?.info({ message: "Migrations completed", service: "Migrations" });
|
||||
} catch (error) {
|
||||
const err = error as Error;
|
||||
logger?.error({ message: "Migration failed", service: "Migrations", details: err?.message, stack: err?.stack });
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export { runMigrations };
|
||||
File renamed without changes.
Executable
+214
@@ -0,0 +1,214 @@
|
||||
import mongoose from "mongoose";
|
||||
import { MonitorTypes } from "@/types/monitor.js";
|
||||
|
||||
const cpuSchema = new mongoose.Schema(
|
||||
{
|
||||
physical_core: { type: Number, default: 0 },
|
||||
logical_core: { type: Number, default: 0 },
|
||||
frequency: { type: Number, default: 0 },
|
||||
temperature: { type: [Number], default: [] },
|
||||
free_percent: { type: Number, default: 0 },
|
||||
usage_percent: { type: Number, default: 0 },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const memorySchema = new mongoose.Schema(
|
||||
{
|
||||
total_bytes: { type: Number, default: 0 },
|
||||
available_bytes: { type: Number, default: 0 },
|
||||
used_bytes: { type: Number, default: 0 },
|
||||
usage_percent: { type: Number, default: 0 },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const diskSchema = new mongoose.Schema(
|
||||
{
|
||||
device: { type: String, default: "" },
|
||||
mountpoint: { type: String, default: "" },
|
||||
read_speed_bytes: { type: Number, default: 0 },
|
||||
write_speed_bytes: { type: Number, default: 0 },
|
||||
total_bytes: { type: Number, default: 0 },
|
||||
free_bytes: { type: Number, default: 0 },
|
||||
usage_percent: { type: Number, default: 0 },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const hostSchema = new mongoose.Schema(
|
||||
{
|
||||
os: { type: String, default: "" },
|
||||
platform: { type: String, default: "" },
|
||||
kernel_version: { type: String, default: "" },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const errorSchema = new mongoose.Schema(
|
||||
{
|
||||
metric: { type: [String], default: [] },
|
||||
err: { type: String, default: "" },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const captureSchema = new mongoose.Schema(
|
||||
{
|
||||
version: { type: String, default: "" },
|
||||
mode: { type: String, default: "" },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const networkInterfaceSchema = new mongoose.Schema(
|
||||
{
|
||||
name: { type: String },
|
||||
bytes_sent: { type: Number, default: 0 },
|
||||
bytes_recv: { type: Number, default: 0 },
|
||||
packets_sent: { type: Number, default: 0 },
|
||||
packets_recv: { type: Number, default: 0 },
|
||||
err_in: { type: Number, default: 0 },
|
||||
err_out: { type: Number, default: 0 },
|
||||
drop_in: { type: Number, default: 0 },
|
||||
drop_out: { type: Number, default: 0 },
|
||||
fifo_in: { type: Number, default: 0 },
|
||||
fifo_out: { type: Number, default: 0 },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const metadataSchema = new mongoose.Schema(
|
||||
{
|
||||
monitorId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: "Monitor",
|
||||
immutable: true,
|
||||
index: true,
|
||||
},
|
||||
teamId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: "Team",
|
||||
immutable: true,
|
||||
index: true,
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
enum: MonitorTypes,
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const CheckSchema = new mongoose.Schema(
|
||||
{
|
||||
metadata: {
|
||||
type: metadataSchema,
|
||||
required: true,
|
||||
},
|
||||
status: {
|
||||
type: Boolean,
|
||||
index: true,
|
||||
},
|
||||
|
||||
responseTime: {
|
||||
type: Number,
|
||||
},
|
||||
|
||||
timings: {
|
||||
type: Object,
|
||||
default: {},
|
||||
},
|
||||
|
||||
statusCode: {
|
||||
type: Number,
|
||||
index: true,
|
||||
},
|
||||
|
||||
message: {
|
||||
type: String,
|
||||
},
|
||||
|
||||
expiry: {
|
||||
type: Date,
|
||||
default: Date.now,
|
||||
},
|
||||
|
||||
ack: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
|
||||
ackAt: {
|
||||
type: Date,
|
||||
},
|
||||
|
||||
// Hardware fields
|
||||
cpu: {
|
||||
type: cpuSchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
memory: {
|
||||
type: memorySchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
disk: {
|
||||
type: [diskSchema],
|
||||
default: () => [],
|
||||
},
|
||||
host: {
|
||||
type: hostSchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
|
||||
errors: {
|
||||
type: [errorSchema],
|
||||
default: () => [],
|
||||
},
|
||||
|
||||
capture: {
|
||||
type: captureSchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
|
||||
net: {
|
||||
type: [networkInterfaceSchema],
|
||||
default: () => [],
|
||||
},
|
||||
|
||||
// PageSpeed fields
|
||||
accessibility: {
|
||||
type: Number,
|
||||
},
|
||||
bestPractices: {
|
||||
type: Number,
|
||||
},
|
||||
seo: {
|
||||
type: Number,
|
||||
},
|
||||
performance: {
|
||||
type: Number,
|
||||
},
|
||||
audits: {
|
||||
type: Object,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
strict: false,
|
||||
timeseries: {
|
||||
timeField: "createdAt",
|
||||
metaField: "metadata",
|
||||
granularity: "seconds",
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
CheckSchema.index({ updatedAt: 1 });
|
||||
CheckSchema.index({ "metadata.monitorId": 1, updatedAt: 1 });
|
||||
CheckSchema.index({ "metadata.monitorId": 1, updatedAt: -1 });
|
||||
CheckSchema.index({ "metadata.teamId": 1, updatedAt: -1 });
|
||||
|
||||
export default mongoose.model("Check", CheckSchema);
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,13 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const MigrationSchema = new mongoose.Schema(
|
||||
{
|
||||
name: { type: String, required: true, unique: true },
|
||||
status: { type: String, enum: ["completed", "failed"], default: "completed" },
|
||||
completedAt: { type: Date },
|
||||
error: { type: String },
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
export default mongoose.model("Migration", MigrationSchema);
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,18 +0,0 @@
|
||||
// import Monitor from "@/db/v1/models/Monitor.js";
|
||||
async function migrateStatusWindowThreshold() {
|
||||
try {
|
||||
// const monitors = await Monitor.find({ statusWindowThreshold: { $lt: 1 } });
|
||||
// for (const monitor of monitors) {
|
||||
// monitor.statusWindowThreshold = monitor.statusWindowThreshold * 100;
|
||||
// await monitor.save();
|
||||
// console.log(`Migrated monitor ${monitor._id}: statusWindowThreshold set to ${monitor.statusWindowThreshold}`);
|
||||
// }
|
||||
// console.log("StatusWindowThreshold migration complete.");
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error("Migration error:", err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export { migrateStatusWindowThreshold };
|
||||
@@ -1,7 +0,0 @@
|
||||
import { migrateStatusWindowThreshold } from "./0001_migrateStatusWindowThreshold.js";
|
||||
|
||||
const runMigrations = async () => {
|
||||
await migrateStatusWindowThreshold();
|
||||
};
|
||||
|
||||
export { runMigrations };
|
||||
@@ -1,178 +0,0 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const cpuSchema = mongoose.Schema({
|
||||
physical_core: { type: Number, default: 0 },
|
||||
logical_core: { type: Number, default: 0 },
|
||||
frequency: { type: Number, default: 0 },
|
||||
temperature: { type: [Number], default: [] },
|
||||
free_percent: { type: Number, default: 0 },
|
||||
usage_percent: { type: Number, default: 0 },
|
||||
});
|
||||
|
||||
const memorySchema = mongoose.Schema({
|
||||
total_bytes: { type: Number, default: 0 },
|
||||
available_bytes: { type: Number, default: 0 },
|
||||
used_bytes: { type: Number, default: 0 },
|
||||
usage_percent: { type: Number, default: 0 },
|
||||
});
|
||||
|
||||
const diskSchema = mongoose.Schema({
|
||||
device: { type: String, default: "" },
|
||||
mountpoint: { type: String, default: "" },
|
||||
read_speed_bytes: { type: Number, default: 0 },
|
||||
write_speed_bytes: { type: Number, default: 0 },
|
||||
total_bytes: { type: Number, default: 0 },
|
||||
free_bytes: { type: Number, default: 0 },
|
||||
usage_percent: { type: Number, default: 0 },
|
||||
});
|
||||
|
||||
const hostSchema = mongoose.Schema({
|
||||
os: { type: String, default: "" },
|
||||
platform: { type: String, default: "" },
|
||||
kernel_version: { type: String, default: "" },
|
||||
});
|
||||
|
||||
const errorSchema = mongoose.Schema({
|
||||
metric: { type: [String], default: [] },
|
||||
err: { type: String, default: "" },
|
||||
});
|
||||
|
||||
const captureSchema = mongoose.Schema({
|
||||
version: { type: String, default: "" },
|
||||
mode: { type: String, default: "" },
|
||||
});
|
||||
|
||||
const networkInterfaceSchema = mongoose.Schema({
|
||||
name: { type: String },
|
||||
bytes_sent: { type: Number, default: 0 },
|
||||
bytes_recv: { type: Number, default: 0 },
|
||||
packets_sent: { type: Number, default: 0 },
|
||||
packets_recv: { type: Number, default: 0 },
|
||||
err_in: { type: Number, default: 0 },
|
||||
err_out: { type: Number, default: 0 },
|
||||
drop_in: { type: Number, default: 0 },
|
||||
drop_out: { type: Number, default: 0 },
|
||||
fifo_in: { type: Number, default: 0 },
|
||||
fifo_out: { type: Number, default: 0 },
|
||||
});
|
||||
|
||||
const CheckSchema = new mongoose.Schema(
|
||||
{
|
||||
// Common fields
|
||||
monitorId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: "Monitor",
|
||||
immutable: true,
|
||||
index: true,
|
||||
},
|
||||
|
||||
teamId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: "Team",
|
||||
immutable: true,
|
||||
index: true,
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
enum: ["http", "ping", "pagespeed", "hardware", "docker", "port", "game"],
|
||||
required: true,
|
||||
index: true,
|
||||
},
|
||||
|
||||
status: {
|
||||
type: Boolean,
|
||||
index: true,
|
||||
},
|
||||
|
||||
responseTime: {
|
||||
type: Number,
|
||||
},
|
||||
|
||||
timings: {
|
||||
type: Object,
|
||||
default: {},
|
||||
},
|
||||
|
||||
statusCode: {
|
||||
type: Number,
|
||||
index: true,
|
||||
},
|
||||
|
||||
message: {
|
||||
type: String,
|
||||
},
|
||||
|
||||
expiry: {
|
||||
type: Date,
|
||||
default: Date.now,
|
||||
expires: 60 * 60 * 24 * 30, // 30 days
|
||||
},
|
||||
|
||||
ack: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
|
||||
ackAt: {
|
||||
type: Date,
|
||||
},
|
||||
|
||||
// Hardware fields
|
||||
cpu: {
|
||||
type: cpuSchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
memory: {
|
||||
type: memorySchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
disk: {
|
||||
type: [diskSchema],
|
||||
default: () => [],
|
||||
},
|
||||
host: {
|
||||
type: hostSchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
|
||||
errors: {
|
||||
type: [errorSchema],
|
||||
default: () => [],
|
||||
},
|
||||
|
||||
capture: {
|
||||
type: captureSchema,
|
||||
default: () => ({}),
|
||||
},
|
||||
|
||||
net: {
|
||||
type: [networkInterfaceSchema],
|
||||
default: () => [],
|
||||
},
|
||||
|
||||
// PageSpeed fields
|
||||
accessibility: {
|
||||
type: Number,
|
||||
},
|
||||
bestPractices: {
|
||||
type: Number,
|
||||
},
|
||||
seo: {
|
||||
type: Number,
|
||||
},
|
||||
performance: {
|
||||
type: Number,
|
||||
},
|
||||
audits: {
|
||||
type: Object,
|
||||
},
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
CheckSchema.index({ updatedAt: 1 });
|
||||
CheckSchema.index({ monitorId: 1, updatedAt: 1 });
|
||||
CheckSchema.index({ monitorId: 1, updatedAt: -1 });
|
||||
CheckSchema.index({ teamId: 1, updatedAt: -1 });
|
||||
|
||||
export default mongoose.model("Check", CheckSchema);
|
||||
Reference in new issue
Block a user