From 19b873ca87de6221c0080a0cfca057ffe0aff28a Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Sun, 7 Jun 2026 10:02:25 -0700 Subject: [PATCH] convert findByIds to aggregate pipeline to include uptime percentage --- server/scripts/seed-monitors.js | 268 ++++++++++++++++++ .../monitors/MongoMonitorsRepository.ts | 31 +- 2 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 server/scripts/seed-monitors.js diff --git a/server/scripts/seed-monitors.js b/server/scripts/seed-monitors.js new file mode 100644 index 000000000..b316eec4f --- /dev/null +++ b/server/scripts/seed-monitors.js @@ -0,0 +1,268 @@ +import mongoose from "mongoose"; +import { MonitorModel } from "../dist/db/models/Monitor.js"; +import { UserModel } from "../dist/db/models/User.js"; + +// Seeds a large number of HTTP monitors against real, resolving domains so the +// scheduler can be exercised under load. Run after `npm run build` (imports the +// compiled models from dist/, like scripts/generate-checks.js). +// +// node scripts/seed-monitors.js # 10k monitors, owner auto-detected +// COUNT=2000 INTERVAL=30000 node scripts/seed-monitors.js +// CLEAR=true node scripts/seed-monitors.js # remove prior seed monitors first +// TEAM_ID= USER_ID= node scripts/seed-monitors.js +// +// NOTE: the running queue only schedules monitors it finds at init(), so restart +// the server after seeding for the new monitors to actually start checking. + +const DEFAULT_COUNT = 5_000; +const DEFAULT_BATCH = 1_000; +const DEFAULT_INTERVAL = 60_000; +const NAME_PREFIX = "Seed"; +// Stored in `description` so a re-run with CLEAR=true can find and remove them. +const SEED_MARKER = "checkmate-scaling-seed"; + +// A spread of well-known, reliably-resolving domains across regions, TLDs and +// CDNs. Monitors round-robin through this pool, so the load is split across many +// hosts rather than hammering one. Every entry returns a real HTTP response +// (some may 403/429 under load — still a valid response for scaling purposes). +const DOMAINS = [ + "google.com", + "youtube.com", + "facebook.com", + "wikipedia.org", + "amazon.com", + "reddit.com", + "x.com", + "instagram.com", + "linkedin.com", + "microsoft.com", + "apple.com", + "cloudflare.com", + "github.com", + "gitlab.com", + "stackoverflow.com", + "netflix.com", + "bing.com", + "yahoo.com", + "duckduckgo.com", + "mozilla.org", + "wordpress.org", + "wordpress.com", + "adobe.com", + "salesforce.com", + "oracle.com", + "ibm.com", + "intel.com", + "nvidia.com", + "amd.com", + "dell.com", + "hp.com", + "cisco.com", + "vmware.com", + "redhat.com", + "ubuntu.com", + "debian.org", + "kernel.org", + "python.org", + "nodejs.org", + "npmjs.com", + "docker.com", + "kubernetes.io", + "golang.org", + "rust-lang.org", + "php.net", + "ruby-lang.org", + "java.com", + "spring.io", + "djangoproject.com", + "flask.palletsprojects.com", + "vuejs.org", + "reactjs.org", + "angular.io", + "svelte.dev", + "jquery.com", + "npm.im", + "pypi.org", + "rubygems.org", + "packagist.org", + "maven.apache.org", + "apache.org", + "nginx.org", + "postgresql.org", + "mysql.com", + "mongodb.com", + "redis.io", + "elastic.co", + "grafana.com", + "prometheus.io", + "hashicorp.com", + "digitalocean.com", + "linode.com", + "vultr.com", + "heroku.com", + "netlify.com", + "vercel.com", + "render.com", + "fly.io", + "railway.app", + "cloudinary.com", + "twilio.com", + "stripe.com", + "paypal.com", + "shopify.com", + "squarespace.com", + "wix.com", + "godaddy.com", + "namecheap.com", + "name.com", + "gandi.net", + "bbc.co.uk", + "theguardian.com", + "nytimes.com", + "washingtonpost.com", + "cnn.com", + "reuters.com", + "bloomberg.com", + "forbes.com", + "wsj.com", + "economist.com", + "nature.com", + "sciencedirect.com", + "arxiv.org", + "mit.edu", + "stanford.edu", + "harvard.edu", + "berkeley.edu", + "ox.ac.uk", + "cam.ac.uk", + "ethz.ch", + "spotify.com", + "soundcloud.com", + "twitch.tv", + "vimeo.com", + "dailymotion.com", + "imdb.com", + "wikimedia.org", + "archive.org", + "ietf.org", + "w3.org", + "who.int", + "un.org", + "europa.eu", + "gov.uk", + "canada.ca", + "alibaba.com", + "aliexpress.com", + "baidu.com", + "tencent.com", + "naver.com", + "rakuten.co.jp", + "yandex.com", + "ozon.ru", + "mercadolibre.com", + "flipkart.com", + "booking.com", + "airbnb.com", + "expedia.com", + "tripadvisor.com", + "uber.com", + "dropbox.com", + "box.com", + "atlassian.com", + "trello.com", + "notion.so", + "slack.com", + "zoom.us", + "asana.com", + "figma.com", + "canva.com", +]; + +const parseObjectId = (value) => { + if (!value) return null; + try { + return new mongoose.Types.ObjectId(value); + } catch { + console.warn(`Ignoring invalid ObjectId '${value}'`); + return null; + } +}; + +// Monitors need a real teamId/userId to show up for an account and to be picked +// up by the queue. Prefer explicit env ids, otherwise derive from the oldest +// existing user (the typical single-tenant dev setup). +async function resolveOwner() { + const teamId = parseObjectId(process.env.TEAM_ID); + const userId = parseObjectId(process.env.USER_ID); + if (teamId && userId) return { teamId, userId }; + + const user = await UserModel.findOne(userId ? { _id: userId } : {}).sort({ createdAt: 1 }); + if (!user?.teamId) { + throw new Error( + "Could not determine an owner: no existing user with a teamId. " + "Create an account first, or pass TEAM_ID and USER_ID explicitly." + ); + } + return { teamId: teamId ?? user.teamId, userId: user._id }; +} + +async function run() { + const mongoUri = process.env.MONGO_URI ?? "mongodb://localhost:27017/uptime_db"; + const total = Number(process.env.COUNT ?? DEFAULT_COUNT); + const batchSize = Number(process.env.BATCH_SIZE ?? DEFAULT_BATCH); + const interval = Number(process.env.INTERVAL ?? DEFAULT_INTERVAL); + + console.log(`Connecting to MongoDB at ${mongoUri}`); + await mongoose.connect(mongoUri); + + const { teamId, userId } = await resolveOwner(); + + if (process.env.CLEAR === "true") { + const { deletedCount } = await MonitorModel.deleteMany({ description: SEED_MARKER }); + console.log(`Removed ${deletedCount} previously seeded monitors`); + } + + console.log( + `Seeding ${total} http monitors for team ${teamId} (user ${userId}) across ${DOMAINS.length} domains, ` + + `interval ${interval}ms, in batches of ${batchSize}` + ); + + const docs = []; + const startTime = Date.now(); + let inserted = 0; + + for (let i = 0; i < total; i += 1) { + const domain = DOMAINS[i % DOMAINS.length]; + docs.push({ + userId, + teamId, + name: `${NAME_PREFIX} ${String(i + 1).padStart(5, "0")} ${domain}`, + description: SEED_MARKER, + type: "http", + url: `https://${domain}`, + isActive: true, + interval, + }); + + if (docs.length === batchSize) { + await MonitorModel.insertMany(docs, { ordered: false }); + inserted += docs.length; + console.log(`Inserted ${inserted} / ${total}`); + docs.length = 0; + } + } + + if (docs.length > 0) { + await MonitorModel.insertMany(docs, { ordered: false }); + inserted += docs.length; + } + + await mongoose.disconnect(); + const duration = ((Date.now() - startTime) / 1000).toFixed(2); + console.log(`Finished inserting ${inserted} monitors in ${duration}s`); + console.log("Restart the server so the queue schedules the new monitors."); +} + +run().catch((error) => { + console.error("Failed to seed monitors", error); + process.exit(1); +}); diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index 07382921a..0b9de8ace 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -151,9 +151,36 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; findByIds = async (monitorIds: string[]): Promise => { + if (!monitorIds.length) { + return []; + } + const objectIds = monitorIds.map((id) => new mongoose.Types.ObjectId(id)); - const monitors = await MonitorModel.find({ _id: { $in: objectIds } }); - return this.mapDocuments(monitors); + + const pipeline: PipelineStage[] = [ + { $match: { _id: { $in: objectIds } } }, + { + $lookup: { + from: "monitorstats", + localField: "_id", + foreignField: "monitorId", + as: "stats", + }, + }, + { + $addFields: { + uptimePercentage: { $arrayElemAt: ["$stats.uptimePercentage", 0] }, + }, + }, + { + $project: { + stats: 0, + }, + }, + ]; + + const documents = await MonitorModel.aggregate(pipeline); + return documents.map((doc) => this.toEntity(doc)); }; findByIdsWithChecks = async (monitorIds: string[], checksCount: number = 25): Promise => {