From c882daf000a6657fe967abe022bf1942924b413c Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 14:18:21 -0700 Subject: [PATCH 01/75] add job domain --- server/src/domain/jobs/job.model.ts | 79 +++++++ .../domain/jobs/job.repository.interface.ts | 55 +++++ .../src/domain/jobs/job.repository.mongo.ts | 194 ++++++++++++++++++ server/src/domain/jobs/job.type.ts | 25 +++ 4 files changed, 353 insertions(+) create mode 100644 server/src/domain/jobs/job.model.ts create mode 100644 server/src/domain/jobs/job.repository.interface.ts create mode 100644 server/src/domain/jobs/job.repository.mongo.ts create mode 100644 server/src/domain/jobs/job.type.ts diff --git a/server/src/domain/jobs/job.model.ts b/server/src/domain/jobs/job.model.ts new file mode 100644 index 000000000..f64ef53b8 --- /dev/null +++ b/server/src/domain/jobs/job.model.ts @@ -0,0 +1,79 @@ +import { Schema, model } from "mongoose"; +import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js"; + +export interface JobDocument extends Omit { + _id: string; +} + +const JobSchema = new Schema( + { + _id: { + type: String, + required: true, + }, + type: { + type: String, + enum: JOB_TYPES, + required: true, + }, + // monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate). + refId: { + type: String, + default: null, + }, + // paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob) + isActive: { + type: Boolean, + default: true, + }, + + // Scheduling — epoch ms, so the claim/$min queries operate on plain numbers + nextScheduledAt: { + type: Number, + required: true, + }, + intervalMs: { + type: Number, + default: null, + }, + + // Lock / lease + lockedBy: { + type: String, + default: null, + }, + lockedUntil: { + type: Number, + default: null, + }, + + // Observability + runCount: { + type: Number, + default: 0, + }, + failCount: { + type: Number, + default: 0, + }, + lastFinishedAt: { + type: Number, + default: null, + }, + lastFailReason: { + type: String, + default: null, + }, + }, + { timestamps: true } +); + +// Used for scanning for claims +JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 }); +// for findById, upsertEvaluate, deleteById +JobSchema.index({ refId: 1, type: 1 }); + +const JobModel = model("Job", JobSchema); + +export { JobModel }; +export default JobModel; diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts new file mode 100644 index 000000000..044ea709e --- /dev/null +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -0,0 +1,55 @@ +import { Job, JobType } from "@/domain/jobs/job.type.js"; + +export type JobPageQuery = { + page?: number; + rowsPerPage?: number; +}; +export type JobPage = { + jobs: Job[]; + count: number; +}; + +export interface IJobsRepository { + // ******************** + // Claims + // ******************** + + // Atomically claim a job + claimDue(type: JobType, now: number): Promise; + + // Completes a repeating job + recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise; + + // Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry + recordFailure(id: string, error: unknown, now: number): Promise; + + // ******************** + // Hand off to evaluator // This is accomplished by creating an "evaluation" type job + // ******************** + upsertEvaluate(monitorId: string, now: number): Promise; + + // ******************** + // Job CRUD + // ******************** + + // Create jobs + upsertJob(job: Job): Promise; + + // Pause/Resume + setActiveById(refId: string, isActive: boolean): Promise; + + // Edit job schedule + updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise; + + // Delete, drop all rows + deleteById(refId: string): Promise; + + // Get job + findById(refId: string): Promise; + + // ******************** + // Observability + // ******************** + findPage(pagination: JobPageQuery): Promise; + findAll(): Promise; +} diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts new file mode 100644 index 000000000..1e0fb9d35 --- /dev/null +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -0,0 +1,194 @@ +import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js"; +import JobModel, { JobDocument } from "@/domain/jobs/job.model.js"; +import { JobType, Job, BACKOFF_MS, LOCK_MS } from "@/domain/jobs/job.type.js"; +import { hostname } from "node:os"; +import { randomUUID } from "node:crypto"; + +class MongoJobsRepository implements IJobsRepository { + private readonly workerId = `${hostname()}:${process.pid}:${randomUUID()}`; + + private toEntity(doc: JobDocument): Job { + return { + id: doc._id, + type: doc.type, + refId: doc.refId, + isActive: doc.isActive, + nextScheduledAt: doc.nextScheduledAt, + intervalMs: doc.intervalMs, + lockedBy: doc.lockedBy, + lockedUntil: doc.lockedUntil, + runCount: doc.runCount, + failCount: doc.failCount, + lastFinishedAt: doc.lastFinishedAt, + lastFailReason: doc.lastFailReason, + }; + } + + claimDue = async (type: JobType, now: number) => { + const doc = await JobModel.findOneAndUpdate( + { + type, + isActive: true, + nextScheduledAt: { $lte: now }, // Find due jobs + $or: [ + { lockedUntil: null }, // Not locked + { lockedUntil: { $lt: now } }, // Free or expired lock + ], + }, + { + $set: { + lockedBy: this.workerId, + lockedUntil: now + LOCK_MS, + }, + }, + { + new: true, + sort: { nextScheduledAt: 1 }, // Get the oldest due job + } + ).lean(); + return doc ? this.toEntity(doc) : null; + }; + + recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => { + // Fixed rate job scheduling + + let nextScheduledAt = scheduledRunAt + intervalMs; + + // If the job fell behind, skipped missed runs + if (nextScheduledAt <= now) { + nextScheduledAt = now + intervalMs; + } + + const res = await JobModel.updateOne( + { + _id: id, + }, + { + $set: { + nextScheduledAt, + lockedBy: null, // Remove lock + lockedUntil: null, + lastFinishedAt: now, + }, + $inc: { + runCount: 1, // Increment run count + }, + } + ); + return res.modifiedCount === 1; + }; + + recordFailure = async (id: string, error: unknown, now: number) => { + const res = await JobModel.updateOne( + { + _id: id, + }, + { + $set: { + nextScheduledAt: now + BACKOFF_MS, + lockedBy: null, // Remove lock + lockedUntil: null, + lastFailReason: error instanceof Error ? error.message : String(error), + }, + $inc: { + failCount: 1, // Increment fail count + }, + } + ); + return res.modifiedCount === 1; + }; + + upsertEvaluate = async (monitorId: string, now: number) => { + const filter = { type: "evaluate", refId: monitorId }; + const update = { + $setOnInsert: { + _id: `evaluate:${monitorId}`, + // refId and Type inferred from filter + isActive: true, + intervalMs: null, + lockedBy: null, + lockedUntil: null, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + }, + $min: { + nextScheduledAt: now, // no op if there is already a pending evaluation that is older + }, + }; + try { + const res = await JobModel.updateOne(filter, update, { upsert: true }); + return res.acknowledged; + } catch (error) { + // Two callers can race the first insert for the same monitor, so the + // loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min. + if (error && typeof error === "object" && (error as { code?: number }).code === 11000) { + const res = await JobModel.updateOne(filter, update, { upsert: true }); + return res.acknowledged; + } + throw error; + } + }; + upsertJob = async (job: Job) => { + const res = await JobModel.updateOne( + { _id: job.id }, + { + $set: { + // These fields are owned by the monitor, always safe to set + type: job.type, + refId: job.refId, + isActive: job.isActive, + intervalMs: job.intervalMs, + }, + $setOnInsert: { + // These fields are scheduling related, only set on insert so they aren't clobbered + nextScheduledAt: job.nextScheduledAt, + lockedBy: null, + lockedUntil: null, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + }, + }, + { upsert: true } + ); + return res.acknowledged; + }; + + setActiveById = async (refId: string, isActive: boolean) => { + const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together + return res.modifiedCount > 0; + }; + + updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => { + const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } }); + return res.modifiedCount === 1; + }; + + deleteById = async (refId: string) => { + const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor + return res.deletedCount > 0; + }; + + findById = async (refId: string) => { + const docs = await JobModel.find({ refId }).lean(); + return docs.map(this.toEntity); + }; + + findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => { + const count = await JobModel.countDocuments(); + const query = JobModel.find().sort({ nextScheduledAt: 1 }); + if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage); + const docs = await query.lean(); + return { jobs: docs.map(this.toEntity), count }; + }; + + findAll = async () => { + const docs = await JobModel.find().lean(); + return docs.map(this.toEntity); + }; +} + +export default MongoJobsRepository; diff --git a/server/src/domain/jobs/job.type.ts b/server/src/domain/jobs/job.type.ts new file mode 100644 index 000000000..1a9098160 --- /dev/null +++ b/server/src/domain/jobs/job.type.ts @@ -0,0 +1,25 @@ +export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const; +export type JobType = (typeof JOB_TYPES)[number]; +export const LOCK_MS = 30_000; +export const BACKOFF_MS = 5_000; + +export type Job = { + id: string; + type: JobType; + refId: string | null; + isActive: boolean; + + // Scheduling fields + nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now + intervalMs: number | null; // repeat period in ms; null = one time job + + // Lock fields + lockedBy: string | null; // workerId holding the lease + lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash + + // observability (replaces what getMetrics/getJobs) + runCount: number; + failCount: number; + lastFinishedAt: number | null; + lastFailReason: string | null; +}; From 8dedf2a7420961db5dd9dc7ab481b7dba7b0235d Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 14:18:32 -0700 Subject: [PATCH 02/75] add last evaluated at field --- server/src/domain/monitors/monitor.model.ts | 5 +++++ server/src/domain/monitors/monitor.repository.mongo.ts | 2 ++ server/src/domain/monitors/monitor.service.ts | 1 + server/src/domain/monitors/monitor.types.ts | 1 + 4 files changed, 9 insertions(+) diff --git a/server/src/domain/monitors/monitor.model.ts b/server/src/domain/monitors/monitor.model.ts index e45e975e8..417599d2c 100644 --- a/server/src/domain/monitors/monitor.model.ts +++ b/server/src/domain/monitors/monitor.model.ts @@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase { teamId: Types.ObjectId; createdAt: Date; updatedAt: Date; + lastEvaluatedAt: number; // epoch ms } const snapshotTimingPhasesSchema = new Schema( @@ -377,6 +378,10 @@ const MonitorSchema = new Schema( type: [checkSnapshotSchema], default: [], }, + lastEvaluatedAt: { + type: Number, + default: 0, + }, }, { timestamps: true, diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index 165db24e3..e41c8a6f2 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -571,6 +571,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { dnsRecordType: doc.dnsRecordType ?? undefined, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), + lastEvaluatedAt: doc.lastEvaluatedAt, }; }; @@ -636,6 +637,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { dnsRecordType: doc.dnsRecordType ?? undefined, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), + lastEvaluatedAt: doc.lastEvaluatedAt, }; }; diff --git a/server/src/domain/monitors/monitor.service.ts b/server/src/domain/monitors/monitor.service.ts index a72d5a50d..8fce77723 100644 --- a/server/src/domain/monitors/monitor.service.ts +++ b/server/src/domain/monitors/monitor.service.ts @@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService { recentChecks: [], createdAt: "", updatedAt: "", + lastEvaluatedAt: 0, })); const createdMonitors = await this.createMonitors(cleanedMonitors); diff --git a/server/src/domain/monitors/monitor.types.ts b/server/src/domain/monitors/monitor.types.ts index 273d8de66..37d477704 100644 --- a/server/src/domain/monitors/monitor.types.ts +++ b/server/src/domain/monitors/monitor.types.ts @@ -88,6 +88,7 @@ export interface Monitor { recentChecks: CheckSnapshot[]; createdAt: string; updatedAt: string; + lastEvaluatedAt: number; // epoch ms } export interface MonitorsSummary { From 771fd5b50f8a631757b4ff8e907b5f3950d46a3b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 14:18:44 -0700 Subject: [PATCH 03/75] add migration for lastEvaluatedAt backfill --- .../0008_backfillMonitorLastEvaluatedAt.ts | 24 +++++++++++++++++++ server/src/db/migration/index.ts | 2 ++ 2 files changed, 26 insertions(+) create mode 100644 server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts diff --git a/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts new file mode 100644 index 000000000..8f8de7308 --- /dev/null +++ b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts @@ -0,0 +1,24 @@ +import { MonitorModel } from "../../domain/monitors/monitor.model.js"; +import { logger } from "@/utils/logger.js"; + +/** + * Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed. + */ +export async function backfillMonitorLastEvaluatedAt(): Promise { + const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt"; + + try { + logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" }); + + const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: 0 } }); + + logger.info({ + service: SERVICE_NAME, + message: `Backfilled lastEvaluatedAt on ${result.modifiedCount} monitors`, + }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error({ service: SERVICE_NAME, message: `Error backfilling lastEvaluatedAt: ${errorMessage}` }); + throw error; + } +} diff --git a/server/src/db/migration/index.ts b/server/src/db/migration/index.ts index ac78cc813..d68a2468e 100644 --- a/server/src/db/migration/index.ts +++ b/server/src/db/migration/index.ts @@ -6,6 +6,7 @@ import MigrationModel from "../../domain/migrations/migration.model.js"; import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js"; import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js"; import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js"; +import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js"; import type { ILogger } from "@/utils/logger.js"; type MigrationEntry = { @@ -21,6 +22,7 @@ const migrations: MigrationEntry[] = [ { name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray }, { name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex }, { name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray }, + { name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt }, ]; const runMigrations = async (logger?: ILogger) => { From 8d637dbcacec094b5e08d2ffc763097c3d7b0276 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 14:18:56 -0700 Subject: [PATCH 04/75] update validation --- server/src/api/validation/monitorValidation.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index f44cc88ae..2dfee1239 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -272,5 +272,6 @@ export const monitorResponseSchema = z userId: z.string(), createdAt: z.string(), updatedAt: z.string(), + lastEvaluatedAt: z.number(), }) .passthrough(); From 54596fe77626cd9222a41e8a5d2a2662c8a50973 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 14:19:14 -0700 Subject: [PATCH 05/75] add mongodb-memory-server for testing --- server/package-lock.json | 421 +++++++++++++++++++++++++++++++++++++++ server/package.json | 1 + 2 files changed, 422 insertions(+) diff --git a/server/package-lock.json b/server/package-lock.json index 0f16e6b5d..e6bd58c1b 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -74,6 +74,7 @@ "globals": "^15.14.0", "jest": "^30.2.0", "lint-staged": "^16.2.7", + "mongodb-memory-server": "11.2.0", "nodemon": "^3.1.11", "prettier": "^3.3.3", "prettier-plugin-ini": "1.3.0", @@ -5463,6 +5464,16 @@ "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", "license": "MIT" }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -5506,6 +5517,21 @@ "node": ">= 6" } }, + "node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/babel-jest": { "version": "30.2.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.2.0.tgz", @@ -5648,6 +5674,104 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.1.tgz", + "integrity": "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.7.2", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.2.tgz", + "integrity": "sha512-aTvMFUWkBmjzKtEQMDGGDNF8bkfpD5N1b/FCwt7A3wrU4t1o/e/85Wzkluh6JlODCjqVESYCkQCdTXqZ9G7VFg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.1.tgz", + "integrity": "sha512-6M5XjcnsygQNPMCMPXSK379xrJFiZ/AEMNBmFEmQW8d/789VQATvriyi5r0HYTL9TkQ26rn3kgdTG3aisbrXkQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.1.tgz", + "integrity": "sha512-ghj2DSK/2e99a1anTVPCV4m4YIYtrbXhfM7V3D7XZLOTsybnYyaJloymGqssQc8l/or0UoDyRtNQkmkEF/ysgQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.2.tgz", + "integrity": "sha512-2V5j0dOfxv3iIngIWMnW1O6UPXpeoU70HJzUJGVth/+RURhDyq7SEWTD70Y2SkUB0MQonYYWpIX3f85P/I22+Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.5.tgz", + "integrity": "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, "node_modules/barse": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/barse/-/barse-0.4.3.tgz", @@ -6450,6 +6574,13 @@ "node": ">=18" } }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true, + "license": "MIT" + }, "node_modules/compressible": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", @@ -7794,6 +7925,16 @@ "dev": true, "license": "MIT" }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -7972,6 +8113,13 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -8162,6 +8310,50 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, + "node_modules/find-cache-dir": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", + "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "dev": true, + "license": "MIT", + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/avajs/find-cache-dir?sponsor=1" + } + }, + "node_modules/find-cache-dir/node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/find-cache-dir/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -11660,6 +11852,170 @@ "whatwg-url": "^14.1.0 || ^13.0.0" } }, + "node_modules/mongodb-memory-server": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/mongodb-memory-server/-/mongodb-memory-server-11.2.0.tgz", + "integrity": "sha512-506AD8qvClVx8Raw/WhAUUWBgIXPyi856iC01aa5vAzHmn6WOXC6ulvudkTF7oTMzJxkyA0A84VpD4BpyfqJ9w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "mongodb-memory-server-core": "11.2.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/mongodb-memory-server-core": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/mongodb-memory-server-core/-/mongodb-memory-server-core-11.2.0.tgz", + "integrity": "sha512-vOoDtn0JiLrHvZY81Rp/UtKXXK0rtJHZGZFVnccvJwYitPLNspO0Ty0grqFQOe7iAET8+GI4zAQcphg+R3vxQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "async-mutex": "^0.5.0", + "camelcase": "^6.3.0", + "debug": "^4.4.3", + "find-cache-dir": "^3.3.2", + "follow-redirects": "^1.16.0", + "https-proxy-agent": "^7.0.6", + "mongodb": "^7.2.0", + "new-find-package-json": "^2.0.0", + "semver": "^7.7.3", + "tar-stream": "^3.1.8", + "tslib": "^2.8.1", + "yauzl": "^3.3.1" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/@types/whatwg-url": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-13.0.0.tgz", + "integrity": "sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/bson": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.2.0.tgz", + "integrity": "sha512-YCEo7KjMlbNlyHhz7zAZNDpIpQbd+wOEHJYezv0nMYTn4x31eIUM2yomNNubclAt63dObUzKHWsBLJ9QcZNSnQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/mongodb-memory-server-core/node_modules/mongodb": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.3.0.tgz", + "integrity": "sha512-WpCqSx7JAU9vcyjm/SU7ydnHls2YrfU3Y3sx4Ml9D7sPe4mXPlaapndiurDXrQ7/VvJkB4/i7b7WovHb8bd8sg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^7.2.0", + "mongodb-connection-string-url": "^7.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.806.0", + "@mongodb-js/zstd": "^7.0.0", + "gcp-metadata": "^7.0.1", + "kerberos": "^7.0.0", + "mongodb-client-encryption": ">=7.0.0 <7.1.0", + "snappy": "^7.3.2", + "socks": "^2.8.6" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-memory-server-core/node_modules/mongodb-connection-string-url": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-7.0.1.tgz", + "integrity": "sha512-h0AZ9A7IDVwwHyMxmdMXKy+9oNlF0zFoahHiX3vQ8e3KFcSP3VmsmfvtRSuLPxmyv2vjIDxqty8smTgie/SNRQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^13.0.0", + "whatwg-url": "^14.1.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/tar-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz", + "integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/mongoose": { "version": "8.23.1", "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.23.1.tgz", @@ -11818,6 +12174,19 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "license": "MIT" }, + "node_modules/new-find-package-json": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/new-find-package-json/-/new-find-package-json-2.0.0.tgz", + "integrity": "sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">=12.22.0" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -12338,6 +12707,13 @@ "node": ">=8" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true, + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -14106,6 +14482,18 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.27.0.tgz", + "integrity": "sha512-WZ189TKnHoAokYHvwzaAQMpd55cgUmFIcJFzBSgGcb886jau5DL+XdDhTWV4ps3FLvk+OORp0dLRTPsLZ21CSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -14475,6 +14863,26 @@ "node": ">=6" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -15678,6 +16086,19 @@ "node": ">=8" } }, + "node_modules/yauzl": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz", + "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/server/package.json b/server/package.json index 2ac0d47f9..66ca955eb 100755 --- a/server/package.json +++ b/server/package.json @@ -96,6 +96,7 @@ "globals": "^15.14.0", "jest": "^30.2.0", "lint-staged": "^16.2.7", + "mongodb-memory-server": "11.2.0", "nodemon": "^3.1.11", "prettier": "^3.3.3", "prettier-plugin-ini": "1.3.0", From be12991997adff7644ab0e3ef103a4887f8a72f2 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 14:19:21 -0700 Subject: [PATCH 06/75] add tests --- .../test/integration/jobsRepository.test.ts | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 server/test/integration/jobsRepository.test.ts diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts new file mode 100644 index 000000000..f8aedc06f --- /dev/null +++ b/server/test/integration/jobsRepository.test.ts @@ -0,0 +1,343 @@ +import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals"; +import mongoose from "mongoose"; +import { MongoMemoryServer } from "mongodb-memory-server"; +import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts"; +import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts"; +import { BACKOFF_MS, LOCK_MS, type Job, type JobType } from "../../src/domain/jobs/job.type.ts"; + +// ── Real-Mongo harness ───────────────────────────────────────────────────────── +// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's +// single-document atomicity on findOneAndUpdate, so it can only be exercised against a +// live engine. We boot a throwaway in-process mongod for this suite. + +let mongod: MongoMemoryServer; + +beforeAll(async () => { + mongod = await MongoMemoryServer.create(); + await mongoose.connect(mongod.getUri()); + await JobModel.init(); // build the indexes the claim scan relies on +}, 120_000); + +afterAll(async () => { + await mongoose.disconnect(); + await mongod.stop(); +}); + +beforeEach(async () => { + await JobModel.deleteMany({}); +}); + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const NOW = 1_000_000; + +// Insert a job row directly, full control over every field. Defaults describe a due, +// unlocked, active check job. +const seedJob = (overrides: Partial = {}): Promise => + JobModel.create({ + _id: overrides._id ?? `check:${overrides.refId ?? "mon-1"}`, + type: "check" as JobType, + refId: "mon-1", + isActive: true, + nextScheduledAt: NOW, + intervalMs: 60_000, + lockedBy: null, + lockedUntil: null, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + ...overrides, + }); + +const readRow = (id: string) => JobModel.findById(id).lean(); + +describe("MongoJobsRepository", () => { + let repo: MongoJobsRepository; + + beforeEach(() => { + repo = new MongoJobsRepository(); + }); + + // ── claimDue ─────────────────────────────────────────────────────────────── + describe("claimDue", () => { + it("claims a due, unlocked job and leases it", async () => { + await seedJob(); + + const claimed = await repo.claimDue("check", NOW); + + expect(claimed?.id).toBe("check:mon-1"); + const row = await readRow("check:mon-1"); + expect(row?.lockedBy).not.toBeNull(); + expect(row?.lockedUntil).toBe(NOW + LOCK_MS); + }); + + it("returns null when nothing is due yet", async () => { + await seedJob({ nextScheduledAt: NOW + 1 }); + expect(await repo.claimDue("check", NOW)).toBeNull(); + }); + + it("skips inactive (paused) jobs", async () => { + await seedJob({ isActive: false }); + expect(await repo.claimDue("check", NOW)).toBeNull(); + }); + + it("only claims the requested type", async () => { + await seedJob({ _id: "geo:mon-1", type: "geo-check" }); + expect(await repo.claimDue("check", NOW)).toBeNull(); + }); + + it("does not claim a job held by a live lease", async () => { + await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS }); + expect(await repo.claimDue("check", NOW)).toBeNull(); + }); + + it("reclaims a job whose lease has expired (crash recovery)", async () => { + await seedJob({ lockedBy: "dead-worker", lockedUntil: NOW - 1 }); + + const claimed = await repo.claimDue("check", NOW); + + expect(claimed?.id).toBe("check:mon-1"); + const row = await readRow("check:mon-1"); + expect(row?.lockedBy).not.toBe("dead-worker"); + expect(row?.lockedUntil).toBe(NOW + LOCK_MS); + }); + + it("returns the oldest-due job first", async () => { + await seedJob({ _id: "check:newer", refId: "newer", nextScheduledAt: NOW }); + await seedJob({ _id: "check:older", refId: "older", nextScheduledAt: NOW - 5_000 }); + + const claimed = await repo.claimDue("check", NOW); + expect(claimed?.id).toBe("check:older"); + }); + + it("lets exactly one of many concurrent claimers win a single due job", async () => { + await seedJob(); + + // Distinct repo instances = distinct workerIds = realistic competing workers. + const workers = Array.from({ length: 20 }, () => new MongoJobsRepository()); + const results = await Promise.all(workers.map((w) => w.claimDue("check", NOW))); + + const winners = results.filter((r): r is Job => r !== null); + expect(winners).toHaveLength(1); + expect(winners[0].id).toBe("check:mon-1"); + }); + + it("hands two concurrent claimers two distinct due jobs (no double-claim)", async () => { + await seedJob({ _id: "check:a", refId: "a" }); + await seedJob({ _id: "check:b", refId: "b" }); + + const [w1, w2] = [new MongoJobsRepository(), new MongoJobsRepository()]; + const [r1, r2] = await Promise.all([w1.claimDue("check", NOW), w2.claimDue("check", NOW)]); + + expect(r1).not.toBeNull(); + expect(r2).not.toBeNull(); + expect(r1?.id).not.toBe(r2?.id); + expect([r1?.id, r2?.id].sort()).toEqual(["check:a", "check:b"]); + }); + }); + + // ── recordSuccess (fixed-rate reschedule) ──────────────────────────────────── + describe("recordSuccess", () => { + it("reschedules fixed-rate from the scheduled tick and releases the lease", async () => { + await seedJob({ lockedBy: "me", lockedUntil: NOW + LOCK_MS, runCount: 4 }); + + const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_500); + + expect(ok).toBe(true); + const row = await readRow("check:mon-1"); + // next tick measured from the scheduled time (NOW), not completion time + expect(row?.nextScheduledAt).toBe(NOW + 60_000); + expect(row?.lockedBy).toBeNull(); + expect(row?.lockedUntil).toBeNull(); + expect(row?.lastFinishedAt).toBe(NOW + 1_500); + expect(row?.runCount).toBe(5); + }); + + it("skips missed ticks when the worker has fallen behind", async () => { + await seedJob({ lockedBy: "me", lockedUntil: NOW + LOCK_MS }); + + // scheduled NOW, interval 60s, but completion is 200s late → NOW+60s is already past + const now = NOW + 200_000; + await repo.recordSuccess("check:mon-1", NOW, 60_000, now); + + const row = await readRow("check:mon-1"); + // no catch-up burst: jump to the next aligned tick from now, not NOW+60_000 + expect(row?.nextScheduledAt).toBe(now + 60_000); + }); + + it("returns false for an unknown id", async () => { + expect(await repo.recordSuccess("missing", NOW, 60_000, NOW)).toBe(false); + }); + }); + + // ── recordFailure (backoff) ────────────────────────────────────────────────── + describe("recordFailure", () => { + it("backs off, records the reason, bumps failCount, releases the lease", async () => { + await seedJob({ lockedBy: "me", lockedUntil: NOW + LOCK_MS, failCount: 2 }); + + const ok = await repo.recordFailure("check:mon-1", new Error("connection refused"), NOW); + + expect(ok).toBe(true); + const row = await readRow("check:mon-1"); + expect(row?.nextScheduledAt).toBe(NOW + BACKOFF_MS); + expect(row?.lastFailReason).toBe("connection refused"); + expect(row?.failCount).toBe(3); + expect(row?.lockedBy).toBeNull(); + expect(row?.lockedUntil).toBeNull(); + }); + + it("stringifies a non-Error reason", async () => { + await seedJob(); + await repo.recordFailure("check:mon-1", "weird non-error", NOW); + const row = await readRow("check:mon-1"); + expect(row?.lastFailReason).toBe("weird non-error"); + }); + + it("returns false for an unknown id", async () => { + expect(await repo.recordFailure("missing", new Error("x"), NOW)).toBe(false); + }); + }); + + // ── upsertEvaluate (handoff + $min coalescing) ─────────────────────────────── + describe("upsertEvaluate", () => { + it("creates a single evaluate row keyed by monitor", async () => { + await repo.upsertEvaluate("mon-1", NOW); + + const row = await readRow("evaluate:mon-1"); + expect(row?.type).toBe("evaluate"); + expect(row?.refId).toBe("mon-1"); + expect(row?.nextScheduledAt).toBe(NOW); + expect(row?.intervalMs).toBeNull(); + expect(row?.isActive).toBe(true); + }); + + it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => { + await repo.upsertEvaluate("mon-1", NOW); + await repo.upsertEvaluate("mon-1", NOW + 9_000); + + const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean(); + expect(rows).toHaveLength(1); + expect(rows[0].nextScheduledAt).toBe(NOW); // earliest pending wins + }); + + it("lowers runAt when an earlier evaluation arrives ($min)", async () => { + await repo.upsertEvaluate("mon-1", NOW + 9_000); + await repo.upsertEvaluate("mon-1", NOW); + + const row = await readRow("evaluate:mon-1"); + expect(row?.nextScheduledAt).toBe(NOW); + }); + + it("concurrent upserts for one monitor produce exactly one row", async () => { + const workers = Array.from({ length: 20 }, () => new MongoJobsRepository()); + await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW))); + + const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" }); + expect(count).toBe(1); + }); + }); + + // ── monitor lifecycle CRUD ─────────────────────────────────────────────────── + describe("CRUD", () => { + it("upsertJob inserts, then refreshes config without clobbering scheduling state", async () => { + const job: Job = { + id: "check:mon-1", + type: "check", + refId: "mon-1", + isActive: true, + nextScheduledAt: NOW, + intervalMs: 60_000, + lockedBy: null, + lockedUntil: null, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + }; + await repo.upsertJob(job); + + // simulate a running job that has advanced + holds a lease + await JobModel.updateOne({ _id: "check:mon-1" }, { $set: { nextScheduledAt: NOW + 5_000, lockedBy: "live", runCount: 7 } }); + + // a reconcile sweep re-upserts the same id with new config + await repo.upsertJob({ ...job, intervalMs: 30_000, isActive: false }); + + const row = await readRow("check:mon-1"); + expect(row?.intervalMs).toBe(30_000); // config refreshed ($set) + expect(row?.isActive).toBe(false); + expect(row?.nextScheduledAt).toBe(NOW + 5_000); // scheduling untouched ($setOnInsert) + expect(row?.lockedBy).toBe("live"); + expect(row?.runCount).toBe(7); + }); + + it("setActiveById flips check and geo rows together", async () => { + await seedJob({ _id: "check:mon-1", type: "check" }); + await seedJob({ _id: "geo:mon-1", type: "geo-check" }); + + await repo.setActiveById("mon-1", false); + + const rows = await JobModel.find({ refId: "mon-1" }).lean(); + expect(rows).toHaveLength(2); + expect(rows.every((r) => r.isActive === false)).toBe(true); + }); + + it("updateScheduleById changes the interval on a single typed row", async () => { + await seedJob({ _id: "check:mon-1", type: "check", intervalMs: 60_000 }); + await seedJob({ _id: "geo:mon-1", type: "geo-check", intervalMs: 60_000 }); + + await repo.updateScheduleById("mon-1", "check", 15_000); + + expect((await readRow("check:mon-1"))?.intervalMs).toBe(15_000); + expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched + }); + + it("deleteById drops every row for the monitor", async () => { + await seedJob({ _id: "check:mon-1", type: "check" }); + await seedJob({ _id: "geo:mon-1", type: "geo-check" }); + await repo.upsertEvaluate("mon-1", NOW); + + await repo.deleteById("mon-1"); + + expect(await JobModel.countDocuments({ refId: "mon-1" })).toBe(0); + }); + + it("findById returns all of a monitor's rows", async () => { + await seedJob({ _id: "check:mon-1", type: "check" }); + await seedJob({ _id: "geo:mon-1", type: "geo-check" }); + + const jobs = await repo.findById("mon-1"); + expect(jobs.map((j) => j.id).sort()).toEqual(["check:mon-1", "geo:mon-1"]); + }); + }); + + // ── observability ──────────────────────────────────────────────────────────── + describe("observability", () => { + it("findPage returns a page plus the total count", async () => { + await seedJob({ _id: "check:a", refId: "a", nextScheduledAt: NOW + 1 }); + await seedJob({ _id: "check:b", refId: "b", nextScheduledAt: NOW + 2 }); + await seedJob({ _id: "check:c", refId: "c", nextScheduledAt: NOW + 3 }); + + const { jobs, count } = await repo.findPage({ page: 0, rowsPerPage: 2 }); + expect(count).toBe(3); + expect(jobs).toHaveLength(2); + expect(jobs.map((j) => j.id)).toEqual(["check:a", "check:b"]); // sorted by nextScheduledAt + }); + + it("findPage returns everything when rowsPerPage is 0", async () => { + await seedJob({ _id: "check:a", refId: "a" }); + await seedJob({ _id: "check:b", refId: "b" }); + + const { jobs, count } = await repo.findPage({}); + expect(count).toBe(2); + expect(jobs).toHaveLength(2); + }); + + it("findAll returns every row", async () => { + await seedJob({ _id: "check:a", refId: "a" }); + await seedJob({ _id: "geo:b", refId: "b", type: "geo-check" }); + + expect(await repo.findAll()).toHaveLength(2); + }); + }); +}); From 1b3f2f0365848d7780a0268f857376db1ba7a0e4 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 15:19:45 -0700 Subject: [PATCH 07/75] add toStatusResponse to service, add findUnevaluated to repo --- .../checks/check.repository.interface.ts | 1 + .../domain/checks/check.repository.mongo.ts | 10 ++++++++ server/src/domain/checks/check.service.ts | 25 ++++++++++++++++--- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/server/src/domain/checks/check.repository.interface.ts b/server/src/domain/checks/check.repository.interface.ts index 46c2a12c6..95c0ade80 100644 --- a/server/src/domain/checks/check.repository.interface.ts +++ b/server/src/domain/checks/check.repository.interface.ts @@ -42,6 +42,7 @@ export interface IChecksRepository { options?: { type?: MonitorType } ): Promise; findSummaryByTeamId(teamId: string, dateRange: string): Promise; + findUnevaluatedByMonitorId(monitorId: string, since: number): Promise; // update //delete deleteByMonitorId(monitorId: string): Promise; diff --git a/server/src/domain/checks/check.repository.mongo.ts b/server/src/domain/checks/check.repository.mongo.ts index 35d3440c8..4f9ad6f4c 100644 --- a/server/src/domain/checks/check.repository.mongo.ts +++ b/server/src/domain/checks/check.repository.mongo.ts @@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository { }; }; + findUnevaluatedByMonitorId = async (monitorId: string, since: number) => { + const docs = await CheckModel.find({ + "metadata.monitorId": new mongoose.Types.ObjectId(monitorId), + createdAt: { $gt: new Date(since) }, + }) + .sort({ createdAt: 1 }) + .lean(); + return docs.map(this.toEntity); + }; + deleteByMonitorId = async (monitorId: string): Promise => { const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) }); return result.deletedCount; diff --git a/server/src/domain/checks/check.service.ts b/server/src/domain/checks/check.service.ts index 58990eaaf..e7fabe5be 100644 --- a/server/src/domain/checks/check.service.ts +++ b/server/src/domain/checks/check.service.ts @@ -11,7 +11,8 @@ const SERVICE_NAME = "checkService"; export interface ICheckService { createChecks(checks: Check[]): Promise; - buildCheck(statusResponse: MonitorStatusResponse): Check | undefined; + toCheck(statusResponse: MonitorStatusResponse): Check | undefined; + toStatusResponse(check: Check): MonitorStatusResponse; getChecksByMonitor(params: { monitorId: string; teamId: string; @@ -56,7 +57,7 @@ export class CheckService implements ICheckService { return this.checksRepository.createChecks(checks); }; - buildCheck = (statusResponse: MonitorStatusResponse): Check | undefined => { + toCheck = (statusResponse: MonitorStatusResponse): Check | undefined => { const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse; const now = new Date().toISOString(); @@ -82,7 +83,7 @@ export class CheckService implements ICheckService { this.logger.warn({ message: "Failed to build check", service: SERVICE_NAME, - method: "buildCheck", + method: "toCheck", }); return undefined; } @@ -131,6 +132,24 @@ export class CheckService implements ICheckService { return check; }; + toStatusResponse = (check: Check) => { + const statusResponse: MonitorStatusResponse = { + monitorId: check.metadata.monitorId, + teamId: check.metadata.teamId, + type: check.metadata.type, + status: check.status, + code: check.statusCode, + message: check.message, + responseTime: check.responseTime, + // re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data + payload: + check.metadata.type === "hardware" + ? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload) + : undefined, + }; + return statusResponse; + }; + getChecksByMonitor = async ({ monitorId, teamId, From bca902471dd17f0fd42ab6802c2fe5ffef54fe7b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 15:19:53 -0700 Subject: [PATCH 08/75] update tests --- server/test/helpers/heartbeatTestHarness.ts | 2 +- .../test/unit/services/checkPipeline.test.ts | 6 +- .../test/unit/services/checkService.test.ts | 112 +++++++++++++++--- 3 files changed, 100 insertions(+), 20 deletions(-) diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index ea78d5a5d..26dcbbe0f 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -131,7 +131,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { }), }; const checkService = { - buildCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => { + toCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => { return makeCheck(response.status, response.code); }), }; diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/checkPipeline.test.ts index 20ab65950..4a71d9592 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/checkPipeline.test.ts @@ -30,7 +30,7 @@ const createCheckPipeline = (overrides?: Record) => { statusService: { updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }), }, - checkService: { buildCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, + checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, buffer: { addToBuffer: jest.fn() }, monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, @@ -85,9 +85,9 @@ describe("CheckPipeline", () => { await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response"); }); - it("returns null and warns when buildCheck returns null", async () => { + it("returns null and warns when toCheck returns null", async () => { const { pipeline, defaults } = createCheckPipeline({ - checkService: { buildCheck: jest.fn().mockReturnValue(null) }, + checkService: { toCheck: jest.fn().mockReturnValue(null) }, }); const result = await pipeline.run(makeMonitor()); diff --git a/server/test/unit/services/checkService.test.ts b/server/test/unit/services/checkService.test.ts index b658c78da..104ccbaa2 100644 --- a/server/test/unit/services/checkService.test.ts +++ b/server/test/unit/services/checkService.test.ts @@ -4,6 +4,8 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts"; +import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -53,12 +55,12 @@ describe("CheckService", () => { }); }); - // ── buildCheck ─────────────────────────────────────────────────────────── + // ── toCheck ────────────────────────────────────────────────────────────── - describe("buildCheck", () => { + describe("toCheck", () => { it("builds a basic HTTP check with correct fields", () => { const { service } = createService(); - const check = service.buildCheck(makeStatusResponse()); + const check = service.toCheck(makeStatusResponse()); expect(check).toBeDefined(); expect(check!.metadata).toEqual({ monitorId: "mon-1", teamId: "team-1", type: "http" }); @@ -72,14 +74,14 @@ describe("CheckService", () => { it("defaults responseTime to 0 when falsy", () => { const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ responseTime: undefined })); + const check = service.toCheck(makeStatusResponse({ responseTime: undefined })); expect(check!.responseTime).toBe(0); }); it("includes timings when provided", () => { const timings = { start: 0, socket: 10, lookup: 20 }; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ timings } as any)); + const check = service.toCheck(makeStatusResponse({ timings } as any)); expect(check!.timings).toBe(timings); }); @@ -106,7 +108,7 @@ describe("CheckService", () => { } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); expect(check!.accessibility).toBe(90); expect(check!.bestPractices).toBe(80); @@ -119,7 +121,7 @@ describe("CheckService", () => { it("returns undefined when pagespeed payload is missing", () => { const { service, logger } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any)); + const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any)); expect(check).toBeUndefined(); expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build check" })); @@ -127,7 +129,7 @@ describe("CheckService", () => { it("defaults categories and audits to empty when lighthouseResult is missing", () => { const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any)); + const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any)); expect(check!.accessibility).toBe(0); expect(check!.bestPractices).toBe(0); @@ -146,7 +148,7 @@ describe("CheckService", () => { }, } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); expect(check!.audits!.cls!.score).toBe("informative"); }); @@ -161,7 +163,7 @@ describe("CheckService", () => { }, } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); expect(check!.audits!.cls!.score).toBeNull(); }); @@ -176,7 +178,7 @@ describe("CheckService", () => { }, } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); expect(check!.audits!.si!.numericValue).toBeUndefined(); }); @@ -192,7 +194,7 @@ describe("CheckService", () => { }, } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any)); expect(check!.audits!.cls).toBeUndefined(); expect(check!.audits!.si).toBeUndefined(); @@ -214,7 +216,7 @@ describe("CheckService", () => { capture: { screenshot: "base64data" }, } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any)); expect(check!.cpu).toEqual({ usage_percent: 0.5 }); expect(check!.memory).toEqual({ usage_percent: 0.6 }); @@ -230,7 +232,7 @@ describe("CheckService", () => { errors: [{ message: "timeout" }], } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any)); expect(check!.errors).toEqual([{ message: "timeout" }]); }); @@ -241,14 +243,14 @@ describe("CheckService", () => { errors: { errors: [{ message: "nested error" }] }, } as any; const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any)); + const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any)); expect(check!.errors).toEqual([{ message: "nested error" }]); }); it("handles undefined payload gracefully", () => { const { service } = createService(); - const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any)); + const check = service.toCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any)); expect(check).toBeDefined(); expect(check!.cpu).toBeUndefined(); @@ -259,6 +261,84 @@ describe("CheckService", () => { }); }); + // ── toStatusResponse round-trip (Phase 2 §5 guard) ───────────────────────── + // The DB-queue split persists a Check, then the evaluator rebuilds the status response from it. + // toStatusResponse is only correct because every field a downstream consumer reads survives + // toCheck → toStatusResponse. These tests pin that: if a future change drops a consumed field + // from the round-trip, they fail instead of silently regressing the evaluator/reactors. + + describe("toStatusResponse round-trip", () => { + const makeHardwareStatus = (overrides?: Partial): MonitorStatusResponse => + makeStatusResponse({ + type: "hardware", + status: true, + code: 200, + message: "OK", + responseTime: 42, + payload: { + data: { + cpu: { usage_percent: 0.92 }, + memory: { usage_percent: 0.81 }, + disk: [{ device: "/dev/sda", usage_percent: 0.75 }], + host: { os: "linux" }, + net: [{ name: "eth0" }], + }, + } as HardwareStatusPayload, + ...overrides, + } as any); + + it("uptime: preserves every consumed scalar field", () => { + const { service } = createService(); + const status = makeStatusResponse({ type: "http", status: false, code: 503, message: "Service Unavailable", responseTime: 87 }); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + expect(rebuilt.monitorId).toBe(status.monitorId); + expect(rebuilt.teamId).toBe(status.teamId); + expect(rebuilt.type).toBe(status.type); + expect(rebuilt.status).toBe(status.status); + expect(rebuilt.code).toBe(status.code); + expect(rebuilt.message).toBe(status.message); + expect(rebuilt.responseTime).toBe(status.responseTime); + }); + + it("hardware: preserves payload.data metrics verbatim, same units (usage_percent stays a 0–1 decimal)", () => { + const { service } = createService(); + const status = makeHardwareStatus(); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + const original = status.payload as HardwareStatusPayload; + const roundtripped = rebuilt.payload as HardwareStatusPayload; + expect(roundtripped.data).toEqual(original.data); + expect(roundtripped.data.cpu!.usage_percent).toBe(0.92); // not rescaled to 92 + }); + + it("hardware: a real consumer (extractThresholdBreaches) behaves identically on raw vs round-tripped status", () => { + const { service } = createService(); + const builder = new NotificationMessageBuilder(); + const monitor = { + id: "mon-1", + type: "hardware", + cpuAlertThreshold: 90, // 0.92 → 92% breaches + memoryAlertThreshold: 90, // 0.81 → 81% does not + diskAlertThreshold: 90, // 0.75 → 75% does not + tempAlertThreshold: 100, + } as Monitor; + const status = makeHardwareStatus(); + + const rebuilt = service.toStatusResponse(service.toCheck(status)!); + + const fromRaw = builder.extractThresholdBreaches(monitor, status as MonitorStatusResponse); + const fromRebuilt = builder.extractThresholdBreaches(monitor, rebuilt as MonitorStatusResponse); + + // sanity: the fixture actually triggers a breach, so the equality below is not vacuous + expect(fromRaw).toHaveLength(1); + expect(fromRaw[0].metric).toBe("cpu"); + expect(fromRebuilt).toEqual(fromRaw); + }); + }); + // ── createChecks ───────────────────────────────────────────────────────── describe("createChecks", () => { From eff4f8ad69d15228534169d63d05ec5d60eee696 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 11 Jun 2026 15:20:13 -0700 Subject: [PATCH 09/75] fix method name --- server/src/worker/worker.check-pipeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/worker/worker.check-pipeline.ts b/server/src/worker/worker.check-pipeline.ts index f0958fa20..c218b8fe8 100644 --- a/server/src/worker/worker.check-pipeline.ts +++ b/server/src/worker/worker.check-pipeline.ts @@ -71,7 +71,7 @@ export class CheckPipeline implements ICheckPipeline { // **************************** // Step 2a: Create & record a check, return null if fail - const check = this.checkService.buildCheck(status); + const check = this.checkService.toCheck(status); if (!check) { this.logger.warn({ message: `No check could be built for monitor ${monitor.id}`, From 51b6ec1751695a5b323d42ea9a2ba39d8ffa6a3e Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 12 Jun 2026 12:36:50 -0700 Subject: [PATCH 10/75] add check producer and evaluator --- server/src/config/services.ts | 17 ++-- server/src/worker/worker.check-evaluator.ts | 33 +++++++ server/src/worker/worker.check-pipeline.ts | 86 ++----------------- server/src/worker/worker.check-producer.ts | 82 ++++++++++++++++++ server/test/helpers/heartbeatTestHarness.ts | 8 +- .../test/unit/services/checkPipeline.test.ts | 8 +- 6 files changed, 140 insertions(+), 94 deletions(-) create mode 100644 server/src/worker/worker.check-evaluator.ts create mode 100644 server/src/worker/worker.check-producer.ts diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 8d872201e..db6104b83 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -107,6 +107,8 @@ import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; +import { CheckProducer } from "@/worker/worker.check-producer.js"; +import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; export type InitializedServices = { settingsService: ISettingsService; @@ -320,17 +322,12 @@ export const initializeServices = async ({ const incidentReactor = new IncidentReactor(incidentService); const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); + // Check producer/evaluator + const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger); + const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy); + // pipelines - const checkPipeline = new CheckPipeline( - monitorsRepository, - maintenanceWindowsRepository, - checkService, - networkService, - bufferService, - monitorStatusPolicy, - statusService, - logger - ); + const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator); const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger); diff --git a/server/src/worker/worker.check-evaluator.ts b/server/src/worker/worker.check-evaluator.ts new file mode 100644 index 000000000..bbae96d69 --- /dev/null +++ b/server/src/worker/worker.check-evaluator.ts @@ -0,0 +1,33 @@ +import { MonitorStatusResponse } from "@/types/network.js"; +import { MonitorEvaluation } from "@/worker/worker.interface.js"; +import { Check } from "@/domain/checks/check.type.js"; +import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; +import { IStatusService } from "@/service/statusService.js"; + +export interface ICheckEvaluator { + evaluate(status: MonitorStatusResponse, check: Check): Promise; +} + +export class CheckEvaluator implements ICheckEvaluator { + constructor( + private statusService: IStatusService, + private monitorStatusPolicy: IMonitorStatusPolicy + ) {} + evaluate = async (status: MonitorStatusResponse, check: Check) => { + // **************************** + // Step 3: Evaluate and return result to reactors + // **************************** + const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); + + // Step 5. Get decisions and create an evaluation obj + const decision = this.monitorStatusPolicy.evaluate(statusChangeResult); + const evaluation: MonitorEvaluation = { + monitor: statusChangeResult.monitor, + status, + check, + statusChange: statusChangeResult, + decision, + }; + return evaluation; + }; +} diff --git a/server/src/worker/worker.check-pipeline.ts b/server/src/worker/worker.check-pipeline.ts index c218b8fe8..c472c4bfa 100644 --- a/server/src/worker/worker.check-pipeline.ts +++ b/server/src/worker/worker.check-pipeline.ts @@ -1,17 +1,14 @@ import { Monitor } from "@/domain/monitors/monitor.types.js"; import { MonitorEvaluation } from "@/worker/worker.interface.js"; import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; -import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import { isWindowActive } from "@/utils/maintenanceWindow.js"; import { ILogger } from "@/utils/logger.js"; -import { INetworkService } from "@/service/networkService.js"; -import { ICheckService } from "@/domain/checks/check.service.js"; import { IBufferService } from "@/service/bufferService.js"; -import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; -import { IStatusService } from "@/service/statusService.js"; import { AppError } from "@/utils/AppError.js"; import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; +import { ICheckProducer } from "@/worker/worker.check-producer.js"; +import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; const SERVICE_NAME = "CheckPipeline"; @@ -21,83 +18,16 @@ export interface ICheckPipeline { export class CheckPipeline implements ICheckPipeline { constructor( - private monitorsRepository: IMonitorsRepository, - private maintenanceWindowsRepository: IMaintenanceWindowsRepository, - private checkService: ICheckService, - private networkService: INetworkService, - private bufferService: IBufferService, - private monitorStatusPolicy: IMonitorStatusPolicy, - private statusService: IStatusService, - private logger: ILogger + private checkProducer: ICheckProducer, + private checkEvaluator: ICheckEvaluator ) {} - private async isInMaintenanceWindow(monitorId: string, teamId: string) { - const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId); - const now = new Date(); - return maintenanceWindows.some((window) => isWindowActive(window, now)); - } - run = async (monitor: Monitor): Promise => { - const monitorId = monitor.id; - if (!monitorId) { - throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getMonitorJob" }); + const checkResult = await this.checkProducer.produce(monitor); + if (!checkResult) { + return null; // skipped due to maintenance / geo guards / other guards } - // **************************** - // Step 1: Acquire - // **************************** - - // Step 1a: Maintenance window gate - skip if in maintenance - const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId); - if (maintenanceWindowActive) { - this.logger.debug({ - message: `Monitor ${monitor.id} is in maintenance window`, - service: SERVICE_NAME, - method: "runCheckPipeline", - }); - if (monitor.status !== "maintenance") { - await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" }); - } - return null; - } - - // Step 1b: Acquire status - const status = await this.networkService.requestStatus(monitor); - if (!status) { - throw new Error("No network response"); - } - - // **************************** - // Step 2: Record - // **************************** - - // Step 2a: Create & record a check, return null if fail - const check = this.checkService.toCheck(status); - if (!check) { - this.logger.warn({ - message: `No check could be built for monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "runCHeckPipeline", - details: { code: status.code, message: status.message }, - }); - return null; - } - // Step 2b: Add to buffer - this.bufferService.addToBuffer(check); - - // **************************** - // Step 3: Evaluate and return result to reactors - // **************************** - const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); - - // Step 5. Get decisions and create an evaluation obj - const decision = this.monitorStatusPolicy.evaluate(statusChangeResult); - const evaluation: MonitorEvaluation = { - monitor: statusChangeResult.monitor, - status, - check, - statusChange: statusChangeResult, - decision, - }; + const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check); return evaluation; }; } diff --git a/server/src/worker/worker.check-producer.ts b/server/src/worker/worker.check-producer.ts new file mode 100644 index 000000000..ad3975909 --- /dev/null +++ b/server/src/worker/worker.check-producer.ts @@ -0,0 +1,82 @@ +import { Monitor } from "@/domain/monitors/monitor.types.js"; +import { Check } from "@/domain/checks/check.type.js"; +import { MonitorStatusResponse } from "@/types/network.js"; +import { AppError } from "@/utils/AppError.js"; +import { ILogger } from "@/utils/logger.js"; +import { IBufferService } from "@/service/bufferService.js"; +import { INetworkService } from "@/service/networkService.js"; +import { ICheckService } from "@/domain/checks/check.service.js"; +import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; +import { isWindowActive } from "@/utils/maintenanceWindow.js"; + +export interface ICheckProducer { + produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>; +} + +const SERVICE_NAME = "CheckProducer"; + +export class CheckProducer implements ICheckProducer { + constructor( + private monitorsRepository: IMonitorsRepository, + private maintenanceWindowsRepository: IMaintenanceWindowsRepository, + private checkService: ICheckService, + private networkService: INetworkService, + private bufferService: IBufferService, + private logger: ILogger + ) {} + + private async isInMaintenanceWindow(monitorId: string, teamId: string) { + const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId); + const now = new Date(); + return windows.some((w) => isWindowActive(w, now)); + } + + produce = async (monitor: Monitor) => { + if (!monitor.id) { + throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" }); + } + // **************************** + // Step 1: Acquire + // **************************** + + // Step 1a: Maintenance window gate - skip if in maintenance + const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId); + if (maintenanceWindowActive) { + this.logger.debug({ + message: `Monitor ${monitor.id} is in maintenance window`, + service: SERVICE_NAME, + method: "produce", + }); + if (monitor.status !== "maintenance") { + await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" }); + } + return null; + } + + // Step 1b: Acquire status + const status = await this.networkService.requestStatus(monitor); + if (!status) { + throw new Error("No network response"); + } + + // **************************** + // Step 2: Record + // **************************** + + // Step 2a: Create & record a check, return null if fail + const check = this.checkService.toCheck(status); + if (!check) { + this.logger.warn({ + message: `No check could be built for monitor ${monitor.id}`, + service: SERVICE_NAME, + method: "produce", + details: { code: status.code, message: status.message }, + }); + return null; + } + // Step 2b: Add to buffer + this.bufferService.addToBuffer(check); + return { status, check }; + }; +} diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index 26dcbbe0f..88187ac15 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -2,6 +2,8 @@ import { jest } from "@jest/globals"; import { WorkerHelper } from "../../src/worker/worker.helper.ts"; import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts"; import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts"; +import { CheckProducer } from "../../src/worker/worker.check-producer.ts"; +import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts"; import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts"; import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts"; import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts"; @@ -149,16 +151,16 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) }; const geoChecksService = { buildGeoCheck: jest.fn() }; - const checkPipeline = new CheckPipeline( + const checkProducer = new CheckProducer( monitorsRepo as any, maintenanceWindowsRepo as any, checkService as any, networkService as any, bufferStub as any, - new MonitorStatusPolicy(), - statusService as any, logger ); + const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy()); + const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator); const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger); const notificationReactor = new NotificationReactor(notificationsService as any); diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/checkPipeline.test.ts index 4a71d9592..9b1fffd8b 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/checkPipeline.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it, jest } from "@jest/globals"; import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts"; +import { CheckProducer } from "../../../src/worker/worker.check-producer.ts"; +import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts"; import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; @@ -37,16 +39,16 @@ const createCheckPipeline = (overrides?: Record) => { monitorStatusPolicy: new MonitorStatusPolicy(), ...overrides, }; - const pipeline = new CheckPipeline( + const checkProducer = new CheckProducer( defaults.monitorsRepository as any, defaults.maintenanceWindowsRepository as any, defaults.checkService as any, defaults.networkService as any, defaults.buffer as any, - defaults.monitorStatusPolicy as any, - defaults.statusService as any, defaults.logger as any ); + const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); + const pipeline = new CheckPipeline(checkProducer, checkEvaluator); return { pipeline, defaults }; }; From 8b9c41464a0996a9eea201dff3230eda90b3778c Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 12 Jun 2026 15:43:36 -0700 Subject: [PATCH 11/75] dbQueueWorker --- server/scripts/seed-monitors.js | 5 +- server/src/config/services.ts | 18 + .../domain/app-settings/app-settings.type.ts | 2 +- server/src/domain/checks/check.service.ts | 5 + .../domain/jobs/job.repository.interface.ts | 11 +- .../src/domain/jobs/job.repository.mongo.ts | 31 +- server/src/domain/jobs/job.type.ts | 7 + server/src/worker/worker.db-queue.ts | 324 ++++++++++++++++++ 8 files changed, 394 insertions(+), 9 deletions(-) create mode 100644 server/src/worker/worker.db-queue.ts diff --git a/server/scripts/seed-monitors.js b/server/scripts/seed-monitors.js index b316eec4f..e8419f73f 100644 --- a/server/scripts/seed-monitors.js +++ b/server/scripts/seed-monitors.js @@ -1,6 +1,6 @@ import mongoose from "mongoose"; -import { MonitorModel } from "../dist/db/models/Monitor.js"; -import { UserModel } from "../dist/db/models/User.js"; +import { MonitorModel } from "../dist/domain/monitors/monitor.model.js"; +import { UserModel } from "../dist/domain/users/user.model.js"; // Seeds a large number of HTTP monitors against real, resolving domains so the // scheduler can be exercised under load. Run after `npm run build` (imports the @@ -241,6 +241,7 @@ async function run() { url: `https://${domain}`, isActive: true, interval, + lastEvaluatedAt: 0, // evaluator watermark; without it findUnevaluatedByMonitorId gets new Date(undefined) }); if (docs.length === batchSize) { diff --git a/server/src/config/services.ts b/server/src/config/services.ts index db6104b83..e7741f378 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -109,6 +109,8 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; import { CheckProducer } from "@/worker/worker.check-producer.js"; import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; +import { DBQueueWorker } from "@/worker/worker.db-queue.js"; +import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js"; export type InitializedServices = { settingsService: ISettingsService; @@ -233,6 +235,7 @@ export const initializeServices = async ({ const teamsRepository = new MongoTeamsRepository(); const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); const queueWorkersRepository = new MongoQueueWorkersRepository(); + const jobsRepository = new MongoJobsRepository(); // Inject settings repository into settings service (now that DB is connected) (settingsService as SettingsService).setRepository(settingsRepository); @@ -360,6 +363,21 @@ export const initializeServices = async ({ case "lessSimpleQueue": worker = await LessSimpleWorker.create(logger, workerHelper, monitorsRepository, queueWorkersRepository, envSettings, queueMode); break; + case "dbQueue": + worker = await DBQueueWorker.create( + logger, + jobsRepository, + monitorsRepository, + checksRepository, + checkService, + checkProducer, + checkEvaluator, + geoCheckPipeline, + reactorDispatcher, + workerHelper + ); + break; + default: worker = await SuperSimpleQueue.create(logger, workerHelper, monitorsRepository); } diff --git a/server/src/domain/app-settings/app-settings.type.ts b/server/src/domain/app-settings/app-settings.type.ts index d1b4c254b..7b0216548 100644 --- a/server/src/domain/app-settings/app-settings.type.ts +++ b/server/src/domain/app-settings/app-settings.type.ts @@ -1,7 +1,7 @@ export const DbTypes = ["mongodb"] as const; export type DbType = (typeof DbTypes)[number]; -export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const; +export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue", "dbQueue"] as const; export type QueueType = (typeof QueueTypes)[number]; export const QueueModes = ["primary", "worker"] as const; diff --git a/server/src/domain/checks/check.service.ts b/server/src/domain/checks/check.service.ts index e7fabe5be..ca7fca457 100644 --- a/server/src/domain/checks/check.service.ts +++ b/server/src/domain/checks/check.service.ts @@ -11,6 +11,7 @@ const SERVICE_NAME = "checkService"; export interface ICheckService { createChecks(checks: Check[]): Promise; + toLastEvaluatedAt(check: Check): number; toCheck(statusResponse: MonitorStatusResponse): Check | undefined; toStatusResponse(check: Check): MonitorStatusResponse; getChecksByMonitor(params: { @@ -57,6 +58,10 @@ export class CheckService implements ICheckService { return this.checksRepository.createChecks(checks); }; + toLastEvaluatedAt = (check: Check) => { + return new Date(check.createdAt).getTime(); + }; + toCheck = (statusResponse: MonitorStatusResponse): Check | undefined => { const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse; diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index 044ea709e..e431675a0 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -1,4 +1,4 @@ -import { Job, JobType } from "@/domain/jobs/job.type.js"; +import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js"; export type JobPageQuery = { page?: number; @@ -23,6 +23,8 @@ export interface IJobsRepository { // Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry recordFailure(id: string, error: unknown, now: number): Promise; + recordOneShot(id: string, now: number): Promise; + // ******************** // Hand off to evaluator // This is accomplished by creating an "evaluation" type job // ******************** @@ -32,8 +34,8 @@ export interface IJobsRepository { // Job CRUD // ******************** - // Create jobs - upsertJob(job: Job): Promise; + // Create jobs. Takes a JobSeed — the repo owns volatile lease/counter defaults on insert. + upsertJob(job: JobSeed): Promise; // Pause/Resume setActiveById(refId: string, isActive: boolean): Promise; @@ -44,6 +46,9 @@ export interface IJobsRepository { // Delete, drop all rows deleteById(refId: string): Promise; + // Delete a single typed row for a monitor (e.g. drop just the geo row, keep check/evaluate) + deleteByIdAndType(refId: string, type: JobType): Promise; + // Get job findById(refId: string): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index 1e0fb9d35..a3a80f221 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -1,6 +1,6 @@ import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js"; import JobModel, { JobDocument } from "@/domain/jobs/job.model.js"; -import { JobType, Job, BACKOFF_MS, LOCK_MS } from "@/domain/jobs/job.type.js"; +import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js"; import { hostname } from "node:os"; import { randomUUID } from "node:crypto"; @@ -98,11 +98,31 @@ class MongoJobsRepository implements IJobsRepository { return res.modifiedCount === 1; }; + recordOneShot = async (id: string, now: number) => { + const res = await JobModel.updateOne( + { + _id: id, + }, + { + $set: { + nextScheduledAt: PARKED, + lockedBy: null, // Remove lock + lockedUntil: null, + lastFinishedAt: now, + }, + $inc: { + runCount: 1, // Increment run count + }, + } + ); + return res.modifiedCount === 1; + }; + upsertEvaluate = async (monitorId: string, now: number) => { const filter = { type: "evaluate", refId: monitorId }; const update = { $setOnInsert: { - _id: `evaluate:${monitorId}`, + _id: jobId("evaluate", monitorId), // refId and Type inferred from filter isActive: true, intervalMs: null, @@ -130,7 +150,7 @@ class MongoJobsRepository implements IJobsRepository { throw error; } }; - upsertJob = async (job: Job) => { + upsertJob = async (job: JobSeed) => { const res = await JobModel.updateOne( { _id: job.id }, { @@ -172,6 +192,11 @@ class MongoJobsRepository implements IJobsRepository { return res.deletedCount > 0; }; + deleteByIdAndType = async (refId: string, type: JobType) => { + const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row + return res.deletedCount > 0; + }; + findById = async (refId: string) => { const docs = await JobModel.find({ refId }).lean(); return docs.map(this.toEntity); diff --git a/server/src/domain/jobs/job.type.ts b/server/src/domain/jobs/job.type.ts index 1a9098160..7ffd0f2f8 100644 --- a/server/src/domain/jobs/job.type.ts +++ b/server/src/domain/jobs/job.type.ts @@ -2,6 +2,10 @@ export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", export type JobType = (typeof JOB_TYPES)[number]; export const LOCK_MS = 30_000; export const BACKOFF_MS = 5_000; +export const PARKED = Number.MAX_SAFE_INTEGER; + +// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type. +export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`); export type Job = { id: string; @@ -23,3 +27,6 @@ export type Job = { lastFinishedAt: number | null; lastFailReason: string | null; }; + +// These are the only values to set when creating a job. Repo owns the rest +export type JobSeed = Pick; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts new file mode 100644 index 000000000..cb56ac477 --- /dev/null +++ b/server/src/worker/worker.db-queue.ts @@ -0,0 +1,324 @@ +import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; +import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js"; +import { type Job, type JobSeed, type JobType, jobId } from "@/domain/jobs/job.type.js"; +import { randomUUID } from "node:crypto"; +import { hostname } from "node:os"; +import { ILogger } from "@/utils/logger.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; +import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; +import { ICheckService } from "@/domain/checks/check.service.js"; +import { ICheckProducer } from "@/worker/worker.check-producer.js"; +import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; +import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; +import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; +import { IWorkerHelper } from "@/worker/worker.helper.js"; +const SERVICE_NAME = "JobQueue"; +const POLL_MS = 250; +const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; +const CONCURRENCY: Record = { + check: 50, + "geo-check": 20, + evaluate: 20, + "cleanup-orphaned": 1, + "cleanup-retention": 1, +}; + +export class DBQueueWorker implements IWorker { + static SERVICE_NAME = SERVICE_NAME; + private readonly workerId = `${hostname()}:${process.pid}:${randomUUID().slice(0, 8)}`; + private stopped = false; + private timers = new Map(); + private inFlight: Record = { + check: 0, + "geo-check": 0, + evaluate: 0, + "cleanup-orphaned": 0, + "cleanup-retention": 0, + }; + + constructor( + private logger: ILogger, + private jobsRepository: IJobsRepository, + private monitorsRepository: IMonitorsRepository, + private checksRepository: IChecksRepository, + private checkService: ICheckService, // toStatusResponse + toLastEvaluatedAt + private checkProducer: ICheckProducer, + private checkEvaluator: ICheckEvaluator, + private geoCheckPipeline: ICheckPipeline, + private dispatcher: IReactorDispatcher, + private helper: IWorkerHelper + ) {} + + get serviceName() { + return DBQueueWorker.SERVICE_NAME; + } + + static async create( + logger: ILogger, + jobsRepository: IJobsRepository, + monitorsRepository: IMonitorsRepository, + checksRepository: IChecksRepository, + checkService: ICheckService, + checkProducer: ICheckProducer, + checkEvaluator: ICheckEvaluator, + geoCheckPipeline: ICheckPipeline, + dispatcher: IReactorDispatcher, + helper: IWorkerHelper + ): Promise { + const instance = new DBQueueWorker( + logger, + jobsRepository, + monitorsRepository, + checksRepository, + checkService, + checkProducer, + checkEvaluator, + geoCheckPipeline, + dispatcher, + helper + ); + await instance.init(); + return instance; + } + + // Helpers, builds job seeds + private toCheckJob = (monitor: Monitor, now: number): JobSeed => ({ + id: jobId("check", monitor.id), + type: "check", + refId: monitor.id, + isActive: monitor.isActive, + nextScheduledAt: now + Math.floor(Math.random() * monitor.interval), // Avoid herd + intervalMs: monitor.interval, + }); + + private toGeoCheckJob = (monitor: Monitor, now: number): JobSeed => ({ + ...this.toCheckJob(monitor, now), + id: jobId("geo-check", monitor.id), + type: "geo-check", + intervalMs: monitor.geoCheckInterval!, + }); + + private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({ + id: jobId(type, null), + type, + refId: null, + isActive: true, + nextScheduledAt: now, + intervalMs: CLEANUP_INTERVAL_MS, + }); + + private toSummary = (job: Job): WorkerJobSummary => ({ + monitorId: job.refId ?? job.id, + monitorUrl: null, // Jobs don't have URL + monitorType: job.type, + monitorInterval: job.intervalMs, + monitorGeoInterval: null, + monitorActive: job.isActive, + active: job.isActive, + lockedBy: job.lockedBy, + lockedUntil: job.lockedUntil, + lockedAt: null, // no lockedAt + runCount: job.runCount, + failCount: job.failCount, + failReason: job.lastFailReason, + lastRunAt: null, // not tracked + lastFinishedAt: job.lastFinishedAt, + lastRunTook: null, // not tracked + lastFailedAt: null, // not tracked + repeat: job.intervalMs, + }); + + // ******************** + // Stage 1: This can eventually be offloaded to a separate service + // ******************** + + private runCheck = async (job: Job) => { + if (!job.refId) return; + const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId + if (!monitor) return; + const produced = await this.checkProducer.produce(monitor); + if (produced) { + await this.jobsRepository.upsertEvaluate(monitor.id, Date.now()); + } + }; + + // ******************** + // Stage 2: Evaluator loop stays here + // ******************** + + private runEvaluate = async (job: Job) => { + if (!job.refId) return; + const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId + if (!monitor) return; + const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt); + for (const check of checks) { + const status = this.checkService.toStatusResponse(check); + const evaluation = await this.checkEvaluator.evaluate(status, check); + await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications + await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) }); + } + }; + + private runGeoCheck = async (job: Job) => { + const [monitor] = await this.monitorsRepository.findByIds([job.refId!]); + if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff + }; + + private runJob = async (job: Job) => { + try { + switch (job.type) { + case "check": + await this.runCheck(job); + break; + case "evaluate": + await this.runEvaluate(job); + break; + case "geo-check": + await this.runGeoCheck(job); + break; + case "cleanup-orphaned": + await this.helper.getCleanupOrphanedJob()(); // Get job and execute + break; + case "cleanup-retention": + await this.helper.getCleanupRetentionJob()(); // Get job and execute + break; + } + + if (job.intervalMs === null) { + // One-shot job + await this.jobsRepository.recordOneShot(job.id, Date.now()); + } else { + await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now()); + } + } catch (error: unknown) { + await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock + this.logger.warn({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: `runJob:${job.type}`, + }); + } + }; + + private startLoop = (type: JobType) => { + const tick = async () => { + if (this.stopped) return; + try { + // Drain rows up to concurrency limit + while (this.inFlight[type] < CONCURRENCY[type]) { + const job = await this.jobsRepository.claimDue(type, Date.now()); // Claim a job + if (!job) break; // we're done, no jobs due + this.inFlight[type]++; // increment in flight jobs to stay within concurrency limits + this.runJob(job).finally(() => { + this.inFlight[type]--; // job is done, decrement in flight count + }); + } + } catch (error: unknown) { + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: `loop:${type}`, + }); + } + if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down + }; + tick(); + }; + + // Seed queue + private reconcile = async () => { + const now = Date.now(); + const monitors = (await this.monitorsRepository.findAll()) ?? []; + for (const monitor of monitors) { + await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now)); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { + await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now)); + } + } + await this.jobsRepository.upsertJob(this.toCleanupJob("cleanup-orphaned", now)); + await this.jobsRepository.upsertJob(this.toCleanupJob("cleanup-retention", now)); + }; + + init = async () => { + this.stopped = false; + await this.reconcile(); + for (const type of Object.keys(this.inFlight) as JobType[]) { + this.startLoop(type); + } + return true; + }; + addJob = async (_monitorId: string, monitor: Monitor) => { + const now = Date.now(); + await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now)); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now)); + }; + deleteJob = async (monitor: Monitor) => { + await this.jobsRepository.deleteById(monitor.id); + }; + pauseJob = async (monitor: Monitor) => { + await this.jobsRepository.setActiveById(monitor.id, false); + }; + resumeJob = async (monitor: Monitor) => { + await this.jobsRepository.setActiveById(monitor.id, true); + }; + updateJob = async (monitor: Monitor) => { + await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { + await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now())); + } else { + await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check"); + } + }; + shutdown = async () => { + this.stopped = true; + // Stop all timers + for (const timer of this.timers.values()) { + clearTimeout(timer); + } + this.timers.clear(); // Clear the map out + }; + getMetrics = async (): Promise => { + const rows = await this.jobsRepository.findAll(); + const now = Date.now(); + return rows.reduce( + (acc, job) => { + acc.jobs++; + acc.totalRuns += job.runCount; + acc.totalFailures += job.failCount; + if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++; + if (job.failCount > 0) { + acc.failingJobs++; + acc.jobsWithFailures.push({ + monitorId: job.refId ?? job.id, + monitorUrl: null, + monitorType: job.type, + failedAt: job.lastFinishedAt, + failCount: job.failCount, + failReason: job.lastFailReason, + }); + } + return acc; + }, + { + jobs: 0, + activeJobs: 0, + failingJobs: 0, + jobsWithFailures: [], + totalRuns: 0, + totalFailures: 0, + workers: [{ workerId: this.workerId, mode: "primary", lastSeenAt: now }], + } + ); + }; + + getJobs = async (pagination: WorkerJobsPagination) => { + const { jobs, count } = await this.jobsRepository.findPage(pagination); + return { jobs: jobs.map(this.toSummary), count }; + }; + flushQueues = async () => { + await this.shutdown(); + const ok = await this.init(); + return { success: ok }; + }; +} From 47162f2064eda6f3f0bf893e14f965a8dda7018d Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 12 Jun 2026 16:04:31 -0700 Subject: [PATCH 12/75] add a batch claim method --- .../domain/jobs/job.repository.interface.ts | 7 +++-- .../src/domain/jobs/job.repository.mongo.ts | 27 +++++++++++++++++++ server/src/domain/jobs/job.type.ts | 2 +- server/src/worker/worker.db-queue.ts | 14 +++++----- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index e431675a0..5a9bf901e 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -17,6 +17,9 @@ export interface IJobsRepository { // Atomically claim a job claimDue(type: JobType, now: number): Promise; + // locks up to `limit` due rows in one batch + claimDueBatch(type: JobType, limit: number, now: number): Promise; + // Completes a repeating job recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise; @@ -34,7 +37,7 @@ export interface IJobsRepository { // Job CRUD // ******************** - // Create jobs. Takes a JobSeed — the repo owns volatile lease/counter defaults on insert. + // Create jobs. Takes a JobSeed upsertJob(job: JobSeed): Promise; // Pause/Resume @@ -46,7 +49,7 @@ export interface IJobsRepository { // Delete, drop all rows deleteById(refId: string): Promise; - // Delete a single typed row for a monitor (e.g. drop just the geo row, keep check/evaluate) + // Delete a single job of type deleteByIdAndType(refId: string, type: JobType): Promise; // Get job diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index a3a80f221..a8c0a5d12 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -49,6 +49,33 @@ class MongoJobsRepository implements IJobsRepository { return doc ? this.toEntity(doc) : null; }; + claimDueBatch = async (type: JobType, limit: number, now: number) => { + if (limit <= 0) return []; + const dueAndFree = { + type, + isActive: true, + nextScheduledAt: { $lte: now }, // due + $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock + }; + + // 1. Pick the oldest-due candidates + const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>(); + if (candidates.length === 0) return []; + const ids = candidates.map((d) => d._id); + + // 2. lock them. Re-check free/expired so we never steal another worker's live lock + // only the rows this updateMany actually flips end up with claimed + const lockedUntil = now + LOCK_MS; + await JobModel.updateMany( + { _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] }, + { $set: { lockedBy: this.workerId, lockedUntil } } + ); + + // 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time + const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean(); + return docs.map(this.toEntity); + }; + recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => { // Fixed rate job scheduling diff --git a/server/src/domain/jobs/job.type.ts b/server/src/domain/jobs/job.type.ts index 7ffd0f2f8..70d635f81 100644 --- a/server/src/domain/jobs/job.type.ts +++ b/server/src/domain/jobs/job.type.ts @@ -1,6 +1,6 @@ export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const; export type JobType = (typeof JOB_TYPES)[number]; -export const LOCK_MS = 30_000; +export const LOCK_MS = 60_000; export const BACKOFF_MS = 5_000; export const PARKED = Number.MAX_SAFE_INTEGER; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index cb56ac477..c333491df 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -17,7 +17,7 @@ const SERVICE_NAME = "JobQueue"; const POLL_MS = 250; const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; const CONCURRENCY: Record = { - check: 50, + check: 500, "geo-check": 20, evaluate: 20, "cleanup-orphaned": 1, @@ -205,13 +205,13 @@ export class DBQueueWorker implements IWorker { const tick = async () => { if (this.stopped) return; try { - // Drain rows up to concurrency limit - while (this.inFlight[type] < CONCURRENCY[type]) { - const job = await this.jobsRepository.claimDue(type, Date.now()); // Claim a job - if (!job) break; // we're done, no jobs due - this.inFlight[type]++; // increment in flight jobs to stay within concurrency limits + // Claim a batch sized to the free capacity + const capacity = CONCURRENCY[type] - this.inFlight[type]; + const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now()); + for (const job of jobs) { + this.inFlight[type]++; // stay within concurrency limits this.runJob(job).finally(() => { - this.inFlight[type]--; // job is done, decrement in flight count + this.inFlight[type]--; // job done, free the slot }); } } catch (error: unknown) { From d2461ef118a7766ee210050c93ef06554e85b307 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 10:39:48 -0700 Subject: [PATCH 13/75] remove old queues --- server/package-lock.json | 113 ----- server/package.json | 2 - server/src/config/services.ts | 104 +---- .../queue-workers/queue-worker.model.ts | 18 - .../queue-worker.repository.interface.ts | 8 - .../queue-worker.repository.mongo.ts | 33 -- .../domain/queue-workers/queue-worker.type.ts | 18 - server/src/index.ts | 2 +- server/src/worker/worker.db-queue.ts | 22 +- server/src/worker/worker.less-simple.ts | 427 ------------------ server/src/worker/worker.super-simple.ts | 384 ---------------- 11 files changed, 29 insertions(+), 1102 deletions(-) delete mode 100644 server/src/domain/queue-workers/queue-worker.model.ts delete mode 100644 server/src/domain/queue-workers/queue-worker.repository.interface.ts delete mode 100644 server/src/domain/queue-workers/queue-worker.repository.mongo.ts delete mode 100644 server/src/domain/queue-workers/queue-worker.type.ts delete mode 100644 server/src/worker/worker.less-simple.ts delete mode 100644 server/src/worker/worker.super-simple.ts diff --git a/server/package-lock.json b/server/package-lock.json index e6bd58c1b..7c69b6c1e 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -31,7 +31,6 @@ "jmespath": "^0.16.0", "jsdom": "^26.1.0", "jsonwebtoken": "9.0.2", - "less-simple-scheduler": "2.6.0", "mailersend": "^2.2.0", "mjml": "^5.0.0-alpha.4", "mongoose": "^8.3.3", @@ -40,7 +39,6 @@ "ping": "0.4.4", "sharp": "0.33.5", "ssl-checker": "2.0.10", - "super-simple-scheduler": "1.9.2", "swagger-ui-express": "5.0.1", "winston": "^3.13.0", "ws": "^8.19.0", @@ -10608,95 +10606,6 @@ "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", "license": "MIT" }, - "node_modules/less-simple-scheduler": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/less-simple-scheduler/-/less-simple-scheduler-2.6.0.tgz", - "integrity": "sha512-Vi8ouiodLqe2fyh+DqCmxE5cCJZDI3xOyUAH4jcj9sf7GunLUzH71PBcYJy2P8lxTK3LxEkHtr1reymiogs6JQ==", - "license": "ISC", - "dependencies": { - "mongodb": "7.2.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/less-simple-scheduler/node_modules/@types/whatwg-url": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-13.0.0.tgz", - "integrity": "sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==", - "license": "MIT", - "dependencies": { - "@types/webidl-conversions": "*" - } - }, - "node_modules/less-simple-scheduler/node_modules/bson": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/bson/-/bson-7.2.0.tgz", - "integrity": "sha512-YCEo7KjMlbNlyHhz7zAZNDpIpQbd+wOEHJYezv0nMYTn4x31eIUM2yomNNubclAt63dObUzKHWsBLJ9QcZNSnQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=20.19.0" - } - }, - "node_modules/less-simple-scheduler/node_modules/mongodb": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.2.0.tgz", - "integrity": "sha512-F/2+BMZtLVhY30ioZp0dAmZ+IRZMBqI+nrv6t5+9/1AIwCa8sMRC3jBf81lpxMhnZgqq8CoUD503Z1oZWq1/sw==", - "license": "Apache-2.0", - "dependencies": { - "@mongodb-js/saslprep": "^1.3.0", - "bson": "^7.2.0", - "mongodb-connection-string-url": "^7.0.0" - }, - "engines": { - "node": ">=20.19.0" - }, - "peerDependencies": { - "@aws-sdk/credential-providers": "^3.806.0", - "@mongodb-js/zstd": "^7.0.0", - "gcp-metadata": "^7.0.1", - "kerberos": "^7.0.0", - "mongodb-client-encryption": ">=7.0.0 <7.1.0", - "snappy": "^7.3.2", - "socks": "^2.8.6" - }, - "peerDependenciesMeta": { - "@aws-sdk/credential-providers": { - "optional": true - }, - "@mongodb-js/zstd": { - "optional": true - }, - "gcp-metadata": { - "optional": true - }, - "kerberos": { - "optional": true - }, - "mongodb-client-encryption": { - "optional": true - }, - "snappy": { - "optional": true - }, - "socks": { - "optional": true - } - } - }, - "node_modules/less-simple-scheduler/node_modules/mongodb-connection-string-url": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-7.0.1.tgz", - "integrity": "sha512-h0AZ9A7IDVwwHyMxmdMXKy+9oNlF0zFoahHiX3vQ8e3KFcSP3VmsmfvtRSuLPxmyv2vjIDxqty8smTgie/SNRQ==", - "license": "Apache-2.0", - "dependencies": { - "@types/whatwg-url": "^13.0.0", - "whatwg-url": "^14.1.0" - }, - "engines": { - "node": ">=20.19.0" - } - }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -14754,28 +14663,6 @@ "postcss": "^8.4.32" } }, - "node_modules/super-simple-scheduler": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/super-simple-scheduler/-/super-simple-scheduler-1.9.2.tgz", - "integrity": "sha512-dfeI1R8jFLeHXIrNsDRxqvdJN2r4BjwKrvv/pPThu0X4RQa008/ubkFno96esHad04J6KyWYTK+P/qgOcoaReg==", - "license": "MIT", - "dependencies": { - "uuid": "11.1.1" - } - }, - "node_modules/super-simple-scheduler/node_modules/uuid": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", - "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/esm/bin/uuid" - } - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/server/package.json b/server/package.json index 66ca955eb..8341bfb2c 100755 --- a/server/package.json +++ b/server/package.json @@ -53,7 +53,6 @@ "jmespath": "^0.16.0", "jsdom": "^26.1.0", "jsonwebtoken": "9.0.2", - "less-simple-scheduler": "2.6.0", "mailersend": "^2.2.0", "mjml": "^5.0.0-alpha.4", "mongoose": "^8.3.3", @@ -62,7 +61,6 @@ "ping": "0.4.4", "sharp": "0.33.5", "ssl-checker": "2.0.10", - "super-simple-scheduler": "1.9.2", "swagger-ui-express": "5.0.1", "winston": "^3.13.0", "ws": "^8.19.0", diff --git a/server/src/config/services.ts b/server/src/config/services.ts index e7741f378..74a73ad7e 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -30,8 +30,6 @@ import { EmailService, IEmailService } from "@/service/emailService.js"; import { GlobalPingService } from "@/service/globalPingService.js"; import { WorkerHelper } from "@/worker/worker.helper.js"; import { IWorker } from "@/worker/worker.interface.js"; -import { LessSimpleWorker } from "@/worker/worker.less-simple.js"; -import { SuperSimpleQueue } from "@/worker/worker.super-simple.js"; import { INetworkService, NetworkService } from "@/service/networkService.js"; import { IStatusService, StatusService } from "@/service/statusService.js"; import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; @@ -89,7 +87,6 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js"; import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js"; import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js"; -import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js"; import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js"; import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js"; import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js"; @@ -101,7 +98,6 @@ import MongoTeamsRepository from "@/domain/teams/team.repository.model.js"; import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; import MongoUsersRepository from "@/domain/users/user.repository.mongo.js"; import { ILogger } from "@/utils/logger.js"; -import { AppError } from "@/utils/AppError.js"; import { type QueueMode } from "@/domain/app-settings/app-settings.type.js"; import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; @@ -155,12 +151,10 @@ export const initializeServices = async ({ logger, envSettings, settingsService, - queueMode = "primary", }: { logger: ILogger; envSettings: EnvConfig; settingsService: ISettingsService; - queueMode?: QueueMode; }): Promise => { // Create DB @@ -168,58 +162,6 @@ export const initializeServices = async ({ db = new MongoDB(logger, envSettings); await db.connect(); - // ** NOTE ** - // const dbType = envSettings.dbType; - // DB type has been fixed to MongoDB for now - - // if (dbType === "mongodb") { - // db = new MongoDB(logger, envSettings); - // } - - // if (!db) { - // throw new AppError({ message: "Unsupported database type", status: 500 }); - // } - - // await db.connect(); - - // Repositories - - // ** NOTE ** - // DB Type fixed to MongoDB for now - // let monitorsRepository: IMonitorsRepository; - // let checksRepository: IChecksRepository; - // let geoChecksRepository: IGeoChecksRepository; - // let monitorStatsRepository: IMonitorStatsRepository; - // let statusPagesRepository: IStatusPagesRepository; - // let usersRepository: IUsersRepository; - // let invitesRepository: IInvitesRepository; - // let recoveryTokensRepository: IRecoveryTokensRepository; - // let settingsRepository: ISettingsRepository; - // let notificationsRepository: INotificationsRepository; - // let tagsRepository: ITagsRepository; - // let incidentsRepository: IIncidentsRepository; - // let teamsRepository: ITeamsRepository; - // let maintenanceWindowsRepository: IMaintenanceWindowsRepository; - - // if (dbType === "mongodb") { - // monitorsRepository = new MongoMonitorsRepository(); - // checksRepository = new MongoChecksRepository(logger); - // geoChecksRepository = new MongoGeoChecksRepository(logger); - // monitorStatsRepository = new MongoMonitorStatsRepository(); - // statusPagesRepository = new MongoStatusPagesRepository(); - // usersRepository = new MongoUsersRepository(); - // invitesRepository = new MongoInvitesRepository(); - // recoveryTokensRepository = new MongoRecoveryTokensRepository(); - // settingsRepository = new MongoSettingsRepository(); - // notificationsRepository = new MongoNotificationsRepository(); - // tagsRepository = new MongoTagsRepository(); - // incidentsRepository = new MongoIncidentsRepository(); - // teamsRepository = new MongoTeamsRepository(); - // maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); - // } else { - // throw new AppError({ message: "Unsupported database type", status: 500 }); - // } - const monitorsRepository = new MongoMonitorsRepository(); const checksRepository = new MongoChecksRepository(logger); const geoChecksRepository = new MongoGeoChecksRepository(logger); @@ -234,7 +176,6 @@ export const initializeServices = async ({ const incidentsRepository = new MongoIncidentsRepository(); const teamsRepository = new MongoTeamsRepository(); const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); - const queueWorkersRepository = new MongoQueueWorkersRepository(); const jobsRepository = new MongoJobsRepository(); // Inject settings repository into settings service (now that DB is connected) @@ -349,38 +290,19 @@ export const initializeServices = async ({ geoCheckPipeline ); - if (queueMode === "worker" && envSettings.queueType !== "lessSimpleQueue") { - throw new AppError({ - message: `Worker mode requires QUEUE_TYPE="lessSimpleQueue" got ${envSettings.queueType}`, - status: 500, - service: "config services", - method: "initializeServices", - }); - } - - let worker: IWorker; - switch (envSettings.queueType) { - case "lessSimpleQueue": - worker = await LessSimpleWorker.create(logger, workerHelper, monitorsRepository, queueWorkersRepository, envSettings, queueMode); - break; - case "dbQueue": - worker = await DBQueueWorker.create( - logger, - jobsRepository, - monitorsRepository, - checksRepository, - checkService, - checkProducer, - checkEvaluator, - geoCheckPipeline, - reactorDispatcher, - workerHelper - ); - break; - - default: - worker = await SuperSimpleQueue.create(logger, workerHelper, monitorsRepository); - } + const worker = await DBQueueWorker.create( + logger, + jobsRepository, + monitorsRepository, + checksRepository, + checkService, + checkProducer, + checkEvaluator, + geoCheckPipeline, + reactorDispatcher, + workerHelper, + envSettings.queueMode + ); // Business services const userService = new UserService({ diff --git a/server/src/domain/queue-workers/queue-worker.model.ts b/server/src/domain/queue-workers/queue-worker.model.ts deleted file mode 100644 index 06299d339..000000000 --- a/server/src/domain/queue-workers/queue-worker.model.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Schema, model } from "mongoose"; -import { QueueModes } from "@/domain/app-settings/app-settings.type.js"; -import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js"; - -const QueueWorkerSchema = new Schema( - { - _id: { type: String, required: true }, - mode: { type: String, enum: QueueModes, required: true }, - lastSeenAt: { type: Date, default: Date.now, expires: 30 }, - }, - { timestamps: true } -); - -const QueueWorkerModel = model("QueueWorker", QueueWorkerSchema); - -export type { QueueWorkerDocument }; -export { QueueWorkerModel }; -export default QueueWorkerModel; diff --git a/server/src/domain/queue-workers/queue-worker.repository.interface.ts b/server/src/domain/queue-workers/queue-worker.repository.interface.ts deleted file mode 100644 index e4522c916..000000000 --- a/server/src/domain/queue-workers/queue-worker.repository.interface.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; -import type { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; - -export interface IQueueWorkersRepository { - upsert(workerId: string, mode: QueueMode): Promise; - findRecent(maxAgeMs: number): Promise; - deleteById(workerId: string): Promise; -} diff --git a/server/src/domain/queue-workers/queue-worker.repository.mongo.ts b/server/src/domain/queue-workers/queue-worker.repository.mongo.ts deleted file mode 100644 index 35565bd03..000000000 --- a/server/src/domain/queue-workers/queue-worker.repository.mongo.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; -import type { IQueueWorkersRepository } from "./queue-worker.repository.interface.js"; -import type { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; -import { QueueWorkerModel, type QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.model.js"; -const SERVICE_NAME = "MongoQueueWorkersRepository"; - -class MongoQueueWorkersRepository implements IQueueWorkersRepository { - static SERVICE_NAME = SERVICE_NAME; - - protected toEntity = (doc: QueueWorkerDocument): QueueWorker => { - return { - workerId: doc._id, - mode: doc.mode, - lastSeenAt: doc.lastSeenAt.getTime(), - }; - }; - - upsert = async (workerId: string, mode: QueueMode): Promise => { - await QueueWorkerModel.updateOne({ _id: workerId }, { $set: { mode, lastSeenAt: new Date() } }, { upsert: true }); - }; - - findRecent = async (maxAgeMs: number): Promise => { - const cutoff = new Date(Date.now() - maxAgeMs); - const docs = await QueueWorkerModel.find({ lastSeenAt: { $gte: cutoff } }); - return docs.map(this.toEntity); - }; - - deleteById = async (workerId: string): Promise => { - await QueueWorkerModel.deleteOne({ _id: workerId }); - }; -} - -export default MongoQueueWorkersRepository; diff --git a/server/src/domain/queue-workers/queue-worker.type.ts b/server/src/domain/queue-workers/queue-worker.type.ts deleted file mode 100644 index a83c831ad..000000000 --- a/server/src/domain/queue-workers/queue-worker.type.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { type QueueMode } from "@/domain/app-settings/app-settings.type.js"; - -// Heartbeat presence record for a scheduler instance. _id is the scheduler's -// workerId (hostname:pid:uuid). The TTL index on lastSeenAt garbage-collects -// records for instances that have stopped heartbeating. -export interface QueueWorkerDocument { - _id: string; - mode: QueueMode; - lastSeenAt: Date; - createdAt: Date; - updatedAt: Date; -} - -export type QueueWorker = { - workerId: string; // hostname:pid:uuid - mode: QueueMode; - lastSeenAt: number; // epoch ms of the last heartbeat -}; diff --git a/server/src/index.ts b/server/src/index.ts index cd3a39cdd..37d4e59d8 100755 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -24,7 +24,7 @@ const startApp = async () => { logger = new Logger({ envSettings }); // Initialize services (connects DB, creates repositories, injects settingsRepository) - const services = await initializeServices({ logger, envSettings, settingsService, queueMode: envSettings.queueMode }); + const services = await initializeServices({ logger, envSettings, settingsService }); // If this is a worker instance, we're done. No need for express if (envSettings.queueMode === "worker") { diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index c333491df..62d2ca7e5 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -13,6 +13,7 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IWorkerHelper } from "@/worker/worker.helper.js"; +import { QueueMode, QueueModes } from "@/domain/app-settings/app-settings.type.js"; const SERVICE_NAME = "JobQueue"; const POLL_MS = 250; const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; @@ -47,7 +48,8 @@ export class DBQueueWorker implements IWorker { private checkEvaluator: ICheckEvaluator, private geoCheckPipeline: ICheckPipeline, private dispatcher: IReactorDispatcher, - private helper: IWorkerHelper + private helper: IWorkerHelper, + private mode: QueueMode ) {} get serviceName() { @@ -64,7 +66,8 @@ export class DBQueueWorker implements IWorker { checkEvaluator: ICheckEvaluator, geoCheckPipeline: ICheckPipeline, dispatcher: IReactorDispatcher, - helper: IWorkerHelper + helper: IWorkerHelper, + mode: QueueMode ): Promise { const instance = new DBQueueWorker( logger, @@ -76,7 +79,8 @@ export class DBQueueWorker implements IWorker { checkEvaluator, geoCheckPipeline, dispatcher, - helper + helper, + mode ); await instance.init(); return instance; @@ -242,9 +246,13 @@ export class DBQueueWorker implements IWorker { init = async () => { this.stopped = false; - await this.reconcile(); - for (const type of Object.keys(this.inFlight) as JobType[]) { - this.startLoop(type); + if (this.mode === "primary") { + await this.reconcile(); + return true; + } else { + for (const type of Object.keys(this.inFlight) as JobType[]) { + this.startLoop(type); + } } return true; }; @@ -307,7 +315,7 @@ export class DBQueueWorker implements IWorker { jobsWithFailures: [], totalRuns: 0, totalFailures: 0, - workers: [{ workerId: this.workerId, mode: "primary", lastSeenAt: now }], + workers: [{ workerId: this.workerId, mode: this.mode, lastSeenAt: now }], } ); }; diff --git a/server/src/worker/worker.less-simple.ts b/server/src/worker/worker.less-simple.ts deleted file mode 100644 index bd4752c6e..000000000 --- a/server/src/worker/worker.less-simple.ts +++ /dev/null @@ -1,427 +0,0 @@ -import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; -import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; -import { ILogger } from "@/utils/logger.js"; -import { MongoStore, Scheduler, IJob, AddJobInput } from "less-simple-scheduler"; -import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; -import { IWorkerHelper } from "@/worker/worker.helper.js"; -import { type WorkerMetrics, type WorkerJobsPagination, type WorkerJobsPage, IWorker } from "@/worker/worker.interface.js"; -import { type QueueMode } from "@/domain/app-settings/app-settings.type.js"; -import { EnvConfig } from "@/domain/app-settings/app-settings.service.js"; -const SERVICE_NAME = "JobQueue"; - -// How long after its last heartbeat a worker is still considered alive. Must be longer than the scheduler's heartbeat -const WORKER_TTL_MS = 30_000; - -// For discriminating job types -type MonitorJob = Omit & { data: Monitor }; -const MONITOR_TEMPLATES = new Set(["monitor-job", "geo-check-job"]); -const isMonitorJob = (job: IJob): job is MonitorJob => MONITOR_TEMPLATES.has(job.template); - -export class LessSimpleWorker implements IWorker { - static SERVICE_NAME = SERVICE_NAME; - - private logger: ILogger; - private helper: IWorkerHelper; - private monitorsRepository: IMonitorsRepository; - private workersRepository: IQueueWorkersRepository; - private readonly scheduler: Scheduler; - private queueMode: QueueMode; - private heartbeatListener: ((workerId: string) => void) | null = null; - - constructor( - logger: ILogger, - helper: IWorkerHelper, - monitorsRepository: IMonitorsRepository, - workersRepository: IQueueWorkersRepository, - scheduler: Scheduler, - queueMode: QueueMode - ) { - this.logger = logger; - this.helper = helper; - this.monitorsRepository = monitorsRepository; - this.workersRepository = workersRepository; - this.scheduler = scheduler; - this.queueMode = queueMode; - this.registerListeners(); - } - - get serviceName() { - return LessSimpleWorker.SERVICE_NAME; - } - - private registerListeners = () => { - this.scheduler.on("scheduler:start", () => { - this.logger.info({ - message: "Scheduler started", - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("scheduler:stop", () => { - this.logger.info({ - message: "Scheduler stopped", - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("scheduler:error", (workerId, error) => { - this.logger.error({ - message: `${workerId} error: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - stack: error instanceof Error ? error.stack : undefined, - }); - }); - - this.scheduler.on("job:locked", (workerId, job) => { - this.logger.debug({ - message: `${workerId} ${job.id} locked`, - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("job:abort", (workerId, job, reason) => { - this.logger.warn({ - message: `${workerId} ${job.id} aborted: ${reason}`, - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("job:attempt", (workerId, job, attempt) => { - this.logger.debug({ - message: `${workerId} ${job.id} attempt ${attempt}`, - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("job:complete", (workerId, job) => { - this.logger.debug({ - message: `${workerId} ${job.id} completed successfully`, - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("job:exhausted", (workerId, job, error) => { - this.logger.error({ - message: `${workerId} ${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - stack: error instanceof Error ? error.stack : undefined, - }); - }); - - this.scheduler.on("job:fail", (workerId, job, error, attempt) => { - this.logger.warn({ - message: `${workerId} ${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - stack: error instanceof Error ? error.stack : undefined, - }); - }); - - this.scheduler.on("job:start", (workerId, job) => { - this.logger.debug({ - message: `${workerId} ${job.id} started`, - service: SERVICE_NAME, - }); - }); - }; - - static async create( - logger: ILogger, - helper: IWorkerHelper, - monitorsRepository: IMonitorsRepository, - workersRepository: IQueueWorkersRepository, - envSettings: EnvConfig, - queueMode: QueueMode - ) { - const store = new MongoStore({ - url: envSettings.dbConnectionString ?? "mongodb://localhost:27017/uptime_db", - }); - const scheduler = new Scheduler(store, { - concurrency: envSettings.queueMode === "primary" ? 0 : 50, - processEvery: 100, - }); - const instance = new LessSimpleWorker(logger, helper, monitorsRepository, workersRepository, scheduler, queueMode); - await instance.init(queueMode); - - return instance; - } - - private startHeartbeat = (queueMode: QueueMode) => { - const beat = ( - workerId: string // This is the heartbeat function that registers workers - ) => - this.workersRepository.upsert(workerId, queueMode).catch((error: unknown) => { - this.logger.warn({ - message: `Worker heartbeat failed: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "startHeartbeat", - }); - }); - beat(this.scheduler.workerId); // Beat immediately to register - if (this.heartbeatListener) { - this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); // remove old listeners - } - this.heartbeatListener = beat; /// Keep a reference to the listener - this.scheduler.on("scheduler:heartbeat", beat); // beat on every heartbeat - }; - - init = async (queueMode: QueueMode = "primary") => { - try { - await this.scheduler.start(); - this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob()); - this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob()); - this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob()); - this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob()); - - // Register in the worker registry - this.startHeartbeat(queueMode); - - // If this is a worker, return early - primary will populate queue - if (queueMode === "worker") { - this.logger.info({ - message: `Worker queue: ${this.scheduler.workerId} initialized`, - service: SERVICE_NAME, - method: "init", - }); - return true; - } - - const monitors = await this.monitorsRepository.findAll(); - if (!monitors) { - return true; - } - const jobInputs = monitors.flatMap((monitor) => this.buildJobInputs(monitor)); - if (jobInputs.length > 0) { - await this.scheduler.addJobs(jobInputs); - } - - this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, upsert: true }); - this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000, upsert: true }); - - return true; - } catch (error: unknown) { - this.logger.error({ - message: `Failed to initialize LessSimpleQueue: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "init", - }); - return false; - } - }; - - // Builds the scheduler inputs for a monitor: always a monitor-job, plus a - // geo-check job when the monitor supports and enables geo checks. - private buildJobInputs = (monitor: Monitor): AddJobInput[] => { - const inputs: AddJobInput[] = [ - { - id: monitor.id, - template: "monitor-job", - repeat: monitor.interval, - active: monitor.isActive, - data: monitor, - upsert: true, - jitter: true, - }, - ]; - - if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { - inputs.push({ - id: `${monitor.id}-geo`, - template: "geo-check-job", - repeat: monitor.geoCheckInterval, - active: monitor.isActive, - data: monitor, - upsert: true, - jitter: true, - }); - } - - return inputs; - }; - - addJob = async (_monitorId: string, monitor: Monitor) => { - await this.scheduler.addJobs(this.buildJobInputs(monitor)); - }; - - deleteJob = async (monitor: Monitor) => { - this.scheduler.removeJob(monitor.id); - const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`); - if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`); - }; - - pauseJob = async (monitor: Monitor) => { - const result = await this.scheduler.pauseJob(monitor.id); - if (result === false) { - throw new Error("Failed to pause monitor"); - } - this.scheduler.updateJob(monitor.id, { data: monitor }); - - const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`); - if (geoJob) { - const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`); - if (geoRes === false) { - this.logger.error({ - message: `Failed to pause geo check job for monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "pauseJob", - }); - } - this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor }); - } - - this.logger.debug({ - message: `Paused monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "pauseJob", - }); - }; - - resumeJob = async (monitor: Monitor) => { - const result = await this.scheduler.resumeJob(monitor.id); - if (result === false) { - throw new Error("Failed to resume monitor"); - } - this.scheduler.updateJob(monitor.id, { data: monitor }); - - const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`); - if (geoJob) { - const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`); - if (geoRes === false) { - this.logger.error({ - message: `Failed to resume geo check job for monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "resumeJob", - }); - } - this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor }); - } - this.logger.debug({ - message: `Resumed monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "resumeJob", - }); - }; - - private syncGeoJob = async (monitor: Monitor) => { - const geoJobId = `${monitor.id}-geo`; - const existingGeoJob = await this.scheduler.getJob(geoJobId); - - // If geoChecks have been disabled, or the monitor type doesn't support them, remove - if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) { - if (existingGeoJob) this.scheduler.removeJob(geoJobId); - return; - } - - // If the job exists, update it - if (existingGeoJob) { - this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor }); - return; - } - - // Otherwise, create it - this.scheduler.addJob({ - id: geoJobId, - template: "geo-check-job", - repeat: monitor.geoCheckInterval, - active: monitor.isActive, - data: monitor, - upsert: true, - }); - }; - - updateJob = async (monitor: Monitor) => { - this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor }); - await this.syncGeoJob(monitor); - }; - - shutdown = async () => { - // Stop beating before we remove this worker, so a heartbeat firing between - // the remove and the scheduler stopping can't re-insert it. - if (this.heartbeatListener) { - this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); - this.heartbeatListener = null; - } - await this.workersRepository.deleteById(this.scheduler.workerId).catch((error: unknown) => { - this.logger.warn({ - message: `Failed to deregister worker on shutdown: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "shutdown", - }); - }); - this.scheduler.stop(); - }; - - getMetrics = async (): Promise => { - const stats = await this.scheduler.getStats(); - - const jobsWithFailures = stats.jobsWithFailures.map((failure) => { - // `data` is untyped at the store level: for monitor jobs it's the Monitor, - // for system jobs (e.g. cleanup) it has no url/type => null - const monitor = failure.data as Monitor | undefined; - return { - monitorId: failure.id, - monitorUrl: monitor?.url ?? null, - monitorType: monitor?.type ?? null, - failedAt: failure.failedAt, - failCount: failure.failCount, - failReason: failure.failReason, - }; - }); - - const workers = await this.workersRepository.findRecent(WORKER_TTL_MS); - return { - jobs: stats.jobs, - activeJobs: stats.activeJobs, - failingJobs: stats.failingJobs, - totalRuns: stats.totalRuns, - totalFailures: stats.totalFailures, - jobsWithFailures, - workers, - }; - }; - - getJobs = async ({ page = 0, rowsPerPage = 0 }: WorkerJobsPagination): Promise => { - const listOptions = rowsPerPage > 0 ? { skip: Math.max(page, 0) * rowsPerPage, limit: rowsPerPage } : undefined; - const [jobs, count] = await Promise.all([this.scheduler.getJobs(listOptions), this.scheduler.countJobs()]); - const mapped = jobs.map((job) => { - const monitor = isMonitorJob(job) ? job.data : undefined; - - return { - monitorId: job.id, - monitorUrl: monitor?.url || null, - monitorType: monitor?.type || null, - monitorInterval: monitor?.interval || null, - monitorGeoInterval: monitor?.geoCheckInterval || null, - monitorActive: monitor?.isActive ?? null, - active: job.active, - repeat: job.repeat ?? null, - lockedBy: job.lockedBy ?? null, - lockedUntil: job.lockedUntil ?? null, - lockedAt: job.lockedAt ?? null, - runCount: job.runCount ?? 0, - failCount: job.failCount ?? 0, - failReason: job.lastError ?? null, - lastRunAt: job.lastStartedAt ?? null, - lastFinishedAt: job.lastFinishedAt ?? null, - lastRunTook: job.lockedAt || !job.lastFinishedAt || !job.lastStartedAt ? null : job.lastFinishedAt - job.lastStartedAt, - lastFailedAt: job.lastFailedAt ?? null, - }; - }); - - return { jobs: mapped, count }; - }; - - flushQueues = async () => { - if (this.queueMode !== "primary") { - // Don't allow workers to flush, though this isn't currently reachable since workers have no API endpoints - this.logger.warn({ message: "Ignoring flush on non-primary worker", service: SERVICE_NAME, method: "flushQueues" }); - return { success: false }; - } - - const flushRes = await this.scheduler.flushJobs(); // Less Simple Queue must flush before stopping as it needs DB - const stopRes = await this.scheduler.stop(); // Destructive, other workers will abort their jobs - const initRes = await this.init(); - return { - success: Boolean(stopRes && flushRes && initRes), - }; - }; -} diff --git a/server/src/worker/worker.super-simple.ts b/server/src/worker/worker.super-simple.ts deleted file mode 100644 index 86b9f152a..000000000 --- a/server/src/worker/worker.super-simple.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; -import { ILogger } from "@/utils/logger.js"; -import Scheduler from "super-simple-scheduler"; -import { Monitor, MonitorType, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; -import { hostname } from "node:os"; -import { randomUUID } from "node:crypto"; -import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; -import { IWorker, WorkerJobSummary } from "@/worker/worker.interface.js"; -import { IWorkerHelper } from "@/worker/worker.helper.js"; - -const SERVICE_NAME = "JobQueue"; - -export class SuperSimpleQueue implements IWorker { - static SERVICE_NAME = SERVICE_NAME; - - private logger: ILogger; - private helper: IWorkerHelper; - private workerId: string; - private monitorsRepository: IMonitorsRepository; - private readonly scheduler: Scheduler; - - constructor(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) { - this.logger = logger; - this.helper = helper; - this.monitorsRepository = monitorsRepository; - this.scheduler = scheduler; - this.registerListeners(); - this.workerId = `${hostname()}:${process.pid}:${randomUUID()}`; - } - - get serviceName() { - return SuperSimpleQueue.SERVICE_NAME; - } - - private registerListeners = () => { - this.scheduler.on("scheduler:start", () => { - this.logger.info({ - message: "Scheduler started", - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("scheduler:stop", () => { - this.logger.info({ - message: "Scheduler stopped", - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("scheduler:error", (error) => { - this.logger.error({ - message: `Scheduler error: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - stack: error instanceof Error ? error.stack : undefined, - }); - }); - - this.scheduler.on("job:abort", (job, reason) => { - this.logger.warn({ - message: `${job.id} aborted: ${reason}`, - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("job:attempt", (job, attempt) => { - this.logger.debug({ - message: `${job.id} attempt ${attempt}`, - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("job:complete", (job) => { - this.logger.debug({ - message: `${job.id} completed successfully`, - service: SERVICE_NAME, - }); - }); - - this.scheduler.on("job:exhausted", (job, error) => { - this.logger.error({ - message: `${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - stack: error instanceof Error ? error.stack : undefined, - }); - }); - - this.scheduler.on("job:fail", (job, error, attempt) => { - this.logger.warn({ - message: `${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - stack: error instanceof Error ? error.stack : undefined, - }); - }); - - this.scheduler.on("job:start", (job) => { - this.logger.debug({ - message: `${job.id} started`, - service: SERVICE_NAME, - }); - }); - }; - - static async create(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository) { - const scheduler = new Scheduler({ - concurrency: 50, - // storeType: "mongo", - // storeType: "redis", - // dbUri: envSettings.dbConnectionString, - }); - const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler); - await instance.init(); - - return instance; - } - - init = async () => { - try { - this.scheduler.start(); - this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob()); - this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob()); - this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob()); - this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob()); - const monitors = await this.monitorsRepository.findAll(); - if (!monitors) { - return true; - } - for (const monitor of monitors) { - const randomOffset = Math.floor(Math.random() * 100); - setTimeout(() => { - this.addJob(monitor.id, monitor); - }, randomOffset); - } - - this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true }); - this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 }); - - return true; - } catch (error: unknown) { - this.logger.error({ - message: `Failed to initialize SuperSimpleQueue: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "init", - }); - return false; - } - }; - - addJob = async (monitorId: string, monitor: Monitor) => { - this.scheduler.addJob({ - id: monitorId, - template: "monitor-job", - repeat: monitor.interval, - active: monitor.isActive, - data: monitor, - }); - - // Return early if we don't need geo checks - if (!supportsGeoCheck(monitor.type)) { - return; - } - - // Add geo check job if enabled for HTTP monitors - if (monitor.geoCheckEnabled) { - this.scheduler.addJob({ - id: `${monitorId}-geo`, - template: "geo-check-job", - repeat: monitor.geoCheckInterval, - active: monitor.isActive, - data: monitor, - }); - } - }; - - deleteJob = async (monitor: Monitor) => { - this.scheduler.removeJob(monitor.id); - const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`); - if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`); - }; - - pauseJob = async (monitor: Monitor) => { - const result = await this.scheduler.pauseJob(monitor.id); - if (result === false) { - throw new Error("Failed to pause monitor"); - } - this.scheduler.updateJob(monitor.id, { data: monitor }); - - const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`); - if (geoJob) { - const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`); - if (geoRes === false) { - this.logger.error({ - message: `Failed to pause geo check job for monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "pauseJob", - }); - } - this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor }); - } - - this.logger.debug({ - message: `Paused monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "pauseJob", - }); - }; - - resumeJob = async (monitor: Monitor) => { - const result = await this.scheduler.resumeJob(monitor.id); - if (result === false) { - throw new Error("Failed to resume monitor"); - } - this.scheduler.updateJob(monitor.id, { data: monitor }); - - const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`); - if (geoJob) { - const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`); - if (geoRes === false) { - this.logger.error({ - message: `Failed to resume geo check job for monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "resumeJob", - }); - } - this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor }); - } - this.logger.debug({ - message: `Resumed monitor ${monitor.id}`, - service: SERVICE_NAME, - method: "resumeJob", - }); - }; - - private syncGeoJob = async (monitor: Monitor) => { - const geoJobId = `${monitor.id}-geo`; - const existingGeoJob = await this.scheduler.getJob(geoJobId); - - // If geoChecks have been disabled, or the monitor type doesn't support them, remove - if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) { - if (existingGeoJob) this.scheduler.removeJob(geoJobId); - return; - } - - // If the job exists, update it - if (existingGeoJob) { - this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor }); - return; - } - - // Otherwise, create it - this.scheduler.addJob({ - id: geoJobId, - template: "geo-check-job", - repeat: monitor.geoCheckInterval, - active: monitor.isActive, - data: monitor, - }); - }; - - updateJob = async (monitor: Monitor) => { - this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor }); - await this.syncGeoJob(monitor); - }; - - shutdown = async () => { - this.scheduler.stop(); - }; - - getMetrics = async () => { - const jobs = await this.scheduler.getJobs(); - const metrics: { - jobs: number; - activeJobs: number; - failingJobs: number; - jobsWithFailures: Array<{ - monitorId: string | number; - monitorUrl: string; - monitorType: MonitorType; - failedAt: number | null; - failCount: number; - failReason: string | null; - }>; - totalRuns: number; - totalFailures: number; - workers: Array<{ - workerId: string; - mode: QueueMode; - lastSeenAt: number; - }>; - } = jobs.reduce( - (acc, job) => { - const runCount = job.runCount ?? 0; - const failCount = job.failCount ?? 0; - const lastFailedAt = job.lastFailedAt ?? 0; - const lastRunAt = job.lastRunAt ?? 0; - acc.totalRuns += runCount; - acc.totalFailures += failCount; - acc.jobs++; - if (failCount > 0 && lastFailedAt >= lastRunAt) { - acc.failingJobs++; - } - - if (job.lockedAt) { - acc.activeJobs++; - } - - if (failCount > 0) { - acc.jobsWithFailures.push({ - monitorId: job.id, - monitorUrl: job.data?.url || null, - monitorType: job.data?.type || null, - failedAt: job.lastFailedAt ?? null, - failCount, - failReason: job.lastFailReason ?? null, - }); - } - return acc; - }, - { - jobs: 0, - activeJobs: 0, - failingJobs: 0, - jobsWithFailures: [] as Array<{ - monitorId: string | number; - monitorUrl: string; - monitorType: MonitorType; - failedAt: number | null; - failCount: number; - failReason: string | null; - }>, - totalRuns: 0, - totalFailures: 0, - workers: [ - { - workerId: this.workerId, - mode: "primary", - lastSeenAt: Date.now(), - }, - ], - } - ); - return metrics; - }; - - getJobs = async ({ page = 0, rowsPerPage = 0 }: { page?: number; rowsPerPage?: number }): Promise<{ jobs: WorkerJobSummary[]; count: number }> => { - const jobs = await this.scheduler.getJobs(); - const mapped = jobs.map((job) => { - return { - monitorId: job.id, - monitorUrl: job.data?.url || null, - monitorType: job.data?.type || null, - monitorInterval: job.data?.interval || null, - monitorGeoInterval: job.data?.geoCheckInterval || null, - monitorActive: job.data?.isActive ?? null, - active: job.active, - repeat: job.repeat ?? null, - lockedBy: job.lockedAt ? this.workerId : null, - lockedUntil: null, - lockedAt: job.lockedAt ?? null, - runCount: job.runCount ?? 0, - failCount: job.failCount ?? 0, - failReason: job.lastFailReason ?? null, - lastRunAt: job.lastRunAt ?? null, - lastFinishedAt: job.lastFinishedAt ?? null, - lastRunTook: job.lockedAt ? null : (job.lastFinishedAt ?? 0) - (job.lastRunAt ?? 0), - lastFailedAt: job.lastFailedAt ?? null, - }; - }); - - const start = Math.max(page, 0) * rowsPerPage; - return { - jobs: rowsPerPage > 0 ? mapped.slice(start, start + rowsPerPage) : mapped, - count: mapped.length, - }; - }; - - flushQueues = async () => { - const stopRes = await this.scheduler.stop(); - const flushRes = await this.scheduler.flushJobs(); - const initRes = await this.init(); - return { - success: Boolean(stopRes && flushRes && initRes), - }; - }; -} From ae4c43324610594c0724102661a6f15825ae25fd Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 11:02:46 -0700 Subject: [PATCH 14/75] add a feature flag to toggle primary worker processing jobs or not --- server/src/config/envValidation.ts | 6 ++--- server/src/config/services.ts | 3 ++- .../app-settings/app-settings.service.ts | 6 ++--- .../domain/app-settings/app-settings.type.ts | 3 --- server/src/worker/worker.db-queue.ts | 22 ++++++++++++------- server/src/worker/worker.interface.ts | 1 - 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/server/src/config/envValidation.ts b/server/src/config/envValidation.ts index 9d83336dc..4497119f8 100644 --- a/server/src/config/envValidation.ts +++ b/server/src/config/envValidation.ts @@ -1,5 +1,5 @@ -import { z } from "zod"; -import { DbTypes, QueueTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js"; +import { boolean, z } from "zod"; +import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js"; import { booleanCoercion } from "@/api/validation/shared.js"; const envSchema = z.object({ @@ -12,8 +12,8 @@ const envSchema = z.object({ DB_CONNECTION_STRING: z.string().min(1, "Database connection string is required"), DB_TYPE: z.enum(DbTypes).default("mongodb"), - QUEUE_TYPE: z.enum(QueueTypes).default("superSimpleQueue"), QUEUE_MODE: z.enum(QueueModes).default("primary"), + QUEUE_PRIMARY_PROCESSES: booleanCoercion.default(true), // JWT Authentication JWT_SECRET: z.string().min(1, "JWT_SECRET is required"), diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 74a73ad7e..341e7da2f 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -301,7 +301,8 @@ export const initializeServices = async ({ geoCheckPipeline, reactorDispatcher, workerHelper, - envSettings.queueMode + envSettings.queueMode, + envSettings.queuePrimaryProcesses ); // Business services diff --git a/server/src/domain/app-settings/app-settings.service.ts b/server/src/domain/app-settings/app-settings.service.ts index ef14a932b..8beb65d84 100755 --- a/server/src/domain/app-settings/app-settings.service.ts +++ b/server/src/domain/app-settings/app-settings.service.ts @@ -1,5 +1,5 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js"; -import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js"; +import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js"; import { AppError } from "@/utils/AppError.js"; import { ValidatedEnv } from "@/config/envValidation.js"; import type { StringValue } from "ms"; @@ -13,8 +13,8 @@ export type EnvConfig = { clientHost: string; dbConnectionString: string; dbType: DbType; - queueType: QueueType; queueMode: QueueMode; + queuePrimaryProcesses: boolean; statusPageThemesEnabled: boolean; }; @@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService { clientHost: env.CLIENT_HOST, dbConnectionString: env.DB_CONNECTION_STRING, dbType: env.DB_TYPE, - queueType: env.QUEUE_TYPE, queueMode: env.QUEUE_MODE, + queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES, statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED, }; } diff --git a/server/src/domain/app-settings/app-settings.type.ts b/server/src/domain/app-settings/app-settings.type.ts index 7b0216548..8cb52e93d 100644 --- a/server/src/domain/app-settings/app-settings.type.ts +++ b/server/src/domain/app-settings/app-settings.type.ts @@ -1,9 +1,6 @@ export const DbTypes = ["mongodb"] as const; export type DbType = (typeof DbTypes)[number]; -export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue", "dbQueue"] as const; -export type QueueType = (typeof QueueTypes)[number]; - export const QueueModes = ["primary", "worker"] as const; export type QueueMode = (typeof QueueModes)[number]; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 62d2ca7e5..9331a6927 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -13,7 +13,7 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IWorkerHelper } from "@/worker/worker.helper.js"; -import { QueueMode, QueueModes } from "@/domain/app-settings/app-settings.type.js"; +import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; const SERVICE_NAME = "JobQueue"; const POLL_MS = 250; const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; @@ -49,7 +49,8 @@ export class DBQueueWorker implements IWorker { private geoCheckPipeline: ICheckPipeline, private dispatcher: IReactorDispatcher, private helper: IWorkerHelper, - private mode: QueueMode + private queueMode: QueueMode, + private queuePrimaryProcesses: boolean ) {} get serviceName() { @@ -67,7 +68,8 @@ export class DBQueueWorker implements IWorker { geoCheckPipeline: ICheckPipeline, dispatcher: IReactorDispatcher, helper: IWorkerHelper, - mode: QueueMode + queueMode: QueueMode, + queuePrimaryProcesses: boolean ): Promise { const instance = new DBQueueWorker( logger, @@ -80,7 +82,8 @@ export class DBQueueWorker implements IWorker { geoCheckPipeline, dispatcher, helper, - mode + queueMode, + queuePrimaryProcesses ); await instance.init(); return instance; @@ -246,16 +249,19 @@ export class DBQueueWorker implements IWorker { init = async () => { this.stopped = false; - if (this.mode === "primary") { + + if (this.queueMode === "primary") { await this.reconcile(); - return true; - } else { + } + + if (this.queuePrimaryProcesses === true || this.queueMode === "worker") { for (const type of Object.keys(this.inFlight) as JobType[]) { this.startLoop(type); } } return true; }; + addJob = async (_monitorId: string, monitor: Monitor) => { const now = Date.now(); await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now)); @@ -315,7 +321,7 @@ export class DBQueueWorker implements IWorker { jobsWithFailures: [], totalRuns: 0, totalFailures: 0, - workers: [{ workerId: this.workerId, mode: this.mode, lastSeenAt: now }], + workers: [{ workerId: this.workerId, mode: this.queueMode, lastSeenAt: now }], } ); }; diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 5d1ccf97f..3b5eb0bd7 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,5 +1,4 @@ import { Monitor } from "@/domain/monitors/monitor.types.js"; -import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; import { Check } from "@/domain/checks/check.type.js"; import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js"; import { MonitorActionDecision } from "@/worker/worker.helper.js"; From 90bbc764fc9efb1e8aed57aa40e71bfd9a3f6b05 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 11:26:49 -0700 Subject: [PATCH 15/75] readd queue-worker domain, add heartbewat to db-queue --- server/src/config/services.ts | 3 ++ .../queue-workers/queue-worker.model.ts | 22 +++++++++++++ .../queue-worker.repository.interface.ts | 8 +++++ .../queue-worker.repository.mongo.ts | 33 +++++++++++++++++++ .../domain/queue-workers/queue-worker.type.ts | 18 ++++++++++ server/src/worker/worker.db-queue.ts | 30 +++++++++++++++-- server/src/worker/worker.interface.ts | 1 + 7 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 server/src/domain/queue-workers/queue-worker.model.ts create mode 100644 server/src/domain/queue-workers/queue-worker.repository.interface.ts create mode 100644 server/src/domain/queue-workers/queue-worker.repository.mongo.ts create mode 100644 server/src/domain/queue-workers/queue-worker.type.ts diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 341e7da2f..2afd263c9 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -107,6 +107,7 @@ import { CheckProducer } from "@/worker/worker.check-producer.js"; import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { DBQueueWorker } from "@/worker/worker.db-queue.js"; import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js"; +import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js"; export type InitializedServices = { settingsService: ISettingsService; @@ -177,6 +178,7 @@ export const initializeServices = async ({ const teamsRepository = new MongoTeamsRepository(); const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); const jobsRepository = new MongoJobsRepository(); + const queueWorkersRepository = new MongoQueueWorkersRepository(); // Inject settings repository into settings service (now that DB is connected) (settingsService as SettingsService).setRepository(settingsRepository); @@ -301,6 +303,7 @@ export const initializeServices = async ({ geoCheckPipeline, reactorDispatcher, workerHelper, + queueWorkersRepository, envSettings.queueMode, envSettings.queuePrimaryProcesses ); diff --git a/server/src/domain/queue-workers/queue-worker.model.ts b/server/src/domain/queue-workers/queue-worker.model.ts new file mode 100644 index 000000000..d1a7aa86b --- /dev/null +++ b/server/src/domain/queue-workers/queue-worker.model.ts @@ -0,0 +1,22 @@ +import { Schema, model } from "mongoose"; +import { QueueModes } from "@/domain/app-settings/app-settings.type.js"; +import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js"; + +// Single source of truth for worker liveness: the TTL that GCs stale records and +// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it. +export const WORKER_TTL_SECONDS = 180; + +const QueueWorkerSchema = new Schema( + { + _id: { type: String, required: true }, + mode: { type: String, enum: QueueModes, required: true }, + lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS }, + }, + { timestamps: true } +); + +const QueueWorkerModel = model("QueueWorker", QueueWorkerSchema); + +export type { QueueWorkerDocument }; +export { QueueWorkerModel }; +export default QueueWorkerModel; diff --git a/server/src/domain/queue-workers/queue-worker.repository.interface.ts b/server/src/domain/queue-workers/queue-worker.repository.interface.ts new file mode 100644 index 000000000..e4522c916 --- /dev/null +++ b/server/src/domain/queue-workers/queue-worker.repository.interface.ts @@ -0,0 +1,8 @@ +import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; +import type { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; + +export interface IQueueWorkersRepository { + upsert(workerId: string, mode: QueueMode): Promise; + findRecent(maxAgeMs: number): Promise; + deleteById(workerId: string): Promise; +} diff --git a/server/src/domain/queue-workers/queue-worker.repository.mongo.ts b/server/src/domain/queue-workers/queue-worker.repository.mongo.ts new file mode 100644 index 000000000..35565bd03 --- /dev/null +++ b/server/src/domain/queue-workers/queue-worker.repository.mongo.ts @@ -0,0 +1,33 @@ +import type { QueueMode } from "@/domain/app-settings/app-settings.type.js"; +import type { IQueueWorkersRepository } from "./queue-worker.repository.interface.js"; +import type { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; +import { QueueWorkerModel, type QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.model.js"; +const SERVICE_NAME = "MongoQueueWorkersRepository"; + +class MongoQueueWorkersRepository implements IQueueWorkersRepository { + static SERVICE_NAME = SERVICE_NAME; + + protected toEntity = (doc: QueueWorkerDocument): QueueWorker => { + return { + workerId: doc._id, + mode: doc.mode, + lastSeenAt: doc.lastSeenAt.getTime(), + }; + }; + + upsert = async (workerId: string, mode: QueueMode): Promise => { + await QueueWorkerModel.updateOne({ _id: workerId }, { $set: { mode, lastSeenAt: new Date() } }, { upsert: true }); + }; + + findRecent = async (maxAgeMs: number): Promise => { + const cutoff = new Date(Date.now() - maxAgeMs); + const docs = await QueueWorkerModel.find({ lastSeenAt: { $gte: cutoff } }); + return docs.map(this.toEntity); + }; + + deleteById = async (workerId: string): Promise => { + await QueueWorkerModel.deleteOne({ _id: workerId }); + }; +} + +export default MongoQueueWorkersRepository; diff --git a/server/src/domain/queue-workers/queue-worker.type.ts b/server/src/domain/queue-workers/queue-worker.type.ts new file mode 100644 index 000000000..a83c831ad --- /dev/null +++ b/server/src/domain/queue-workers/queue-worker.type.ts @@ -0,0 +1,18 @@ +import { type QueueMode } from "@/domain/app-settings/app-settings.type.js"; + +// Heartbeat presence record for a scheduler instance. _id is the scheduler's +// workerId (hostname:pid:uuid). The TTL index on lastSeenAt garbage-collects +// records for instances that have stopped heartbeating. +export interface QueueWorkerDocument { + _id: string; + mode: QueueMode; + lastSeenAt: Date; + createdAt: Date; + updatedAt: Date; +} + +export type QueueWorker = { + workerId: string; // hostname:pid:uuid + mode: QueueMode; + lastSeenAt: number; // epoch ms of the last heartbeat +}; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 9331a6927..b26bb4e93 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -13,9 +13,13 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IWorkerHelper } from "@/worker/worker.helper.js"; +import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; +import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; const SERVICE_NAME = "JobQueue"; const POLL_MS = 250; +const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window +const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; const CONCURRENCY: Record = { check: 500, @@ -29,7 +33,7 @@ export class DBQueueWorker implements IWorker { static SERVICE_NAME = SERVICE_NAME; private readonly workerId = `${hostname()}:${process.pid}:${randomUUID().slice(0, 8)}`; private stopped = false; - private timers = new Map(); + private timers = new Map(); private inFlight: Record = { check: 0, "geo-check": 0, @@ -49,6 +53,7 @@ export class DBQueueWorker implements IWorker { private geoCheckPipeline: ICheckPipeline, private dispatcher: IReactorDispatcher, private helper: IWorkerHelper, + private queueWorkersRepository: IQueueWorkersRepository, private queueMode: QueueMode, private queuePrimaryProcesses: boolean ) {} @@ -68,6 +73,7 @@ export class DBQueueWorker implements IWorker { geoCheckPipeline: ICheckPipeline, dispatcher: IReactorDispatcher, helper: IWorkerHelper, + queueWorkersRepository: IQueueWorkersRepository, queueMode: QueueMode, queuePrimaryProcesses: boolean ): Promise { @@ -82,6 +88,7 @@ export class DBQueueWorker implements IWorker { geoCheckPipeline, dispatcher, helper, + queueWorkersRepository, queueMode, queuePrimaryProcesses ); @@ -247,9 +254,25 @@ export class DBQueueWorker implements IWorker { await this.jobsRepository.upsertJob(this.toCleanupJob("cleanup-retention", now)); }; + // Register worker + private heartbeat = async () => { + try { + await this.queueWorkersRepository.upsert(this.workerId, this.queueMode); + } catch (error: unknown) { + this.logger.warn({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "heartbeat", + }); + } + }; + init = async () => { this.stopped = false; + await this.heartbeat(); + this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS)); + if (this.queueMode === "primary") { await this.reconcile(); } @@ -291,9 +314,12 @@ export class DBQueueWorker implements IWorker { clearTimeout(timer); } this.timers.clear(); // Clear the map out + // Deregister promptly so the worker drops off diagnostics before the TTL would + await this.queueWorkersRepository.deleteById(this.workerId); }; getMetrics = async (): Promise => { const rows = await this.jobsRepository.findAll(); + const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS); const now = Date.now(); return rows.reduce( (acc, job) => { @@ -321,7 +347,7 @@ export class DBQueueWorker implements IWorker { jobsWithFailures: [], totalRuns: 0, totalFailures: 0, - workers: [{ workerId: this.workerId, mode: this.queueMode, lastSeenAt: now }], + workers, } ); }; diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 3b5eb0bd7..ccaa79d12 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -2,6 +2,7 @@ import { Monitor } from "@/domain/monitors/monitor.types.js"; import { Check } from "@/domain/checks/check.type.js"; import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js"; import { MonitorActionDecision } from "@/worker/worker.helper.js"; +import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; export type MonitorEvaluation = { monitor: Monitor; From 142ecaaf8b64fb76352f853283b194e2d4ee439f Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 12:05:15 -0700 Subject: [PATCH 16/75] handle renewing locks for long running jobs --- .../domain/jobs/job.repository.interface.ts | 3 + .../src/domain/jobs/job.repository.mongo.ts | 12 + server/src/worker/worker.db-queue.ts | 28 +- .../unit/services/lessSimpleQueue.test.ts | 791 ------------------ .../unit/services/superSimpleQueue.test.ts | 712 ---------------- ...eueHelper.test.ts => workerHelper.test.ts} | 0 6 files changed, 41 insertions(+), 1505 deletions(-) delete mode 100644 server/test/unit/services/lessSimpleQueue.test.ts delete mode 100644 server/test/unit/services/superSimpleQueue.test.ts rename server/test/unit/services/{superSimpleQueueHelper.test.ts => workerHelper.test.ts} (100%) diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index 5a9bf901e..4def9cd86 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -20,6 +20,9 @@ export interface IJobsRepository { // locks up to `limit` due rows in one batch claimDueBatch(type: JobType, limit: number, now: number): Promise; + // Extends the locks we still hold. Returns how many were renewed + renewLocks(ids: string[], now: number): Promise; + // Completes a repeating job recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index a8c0a5d12..7b409c911 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -76,6 +76,15 @@ class MongoJobsRepository implements IJobsRepository { return docs.map(this.toEntity); }; + renewLocks = async (ids: string[], now: number) => { + if (ids.length === 0) return 0; + const res = await JobModel.updateMany( + { _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed + { $set: { lockedUntil: now + LOCK_MS } } + ); + return res.modifiedCount; + }; + recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => { // Fixed rate job scheduling @@ -89,6 +98,7 @@ class MongoJobsRepository implements IJobsRepository { const res = await JobModel.updateOne( { _id: id, + lockedBy: this.workerId, // Only the worker that claimed can record success }, { $set: { @@ -109,6 +119,7 @@ class MongoJobsRepository implements IJobsRepository { const res = await JobModel.updateOne( { _id: id, + lockedBy: this.workerId, // Only the worker that claimed can record success }, { $set: { @@ -129,6 +140,7 @@ class MongoJobsRepository implements IJobsRepository { const res = await JobModel.updateOne( { _id: id, + lockedBy: this.workerId, // Only the worker that claimed can record success }, { $set: { diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index b26bb4e93..3d106445b 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -1,6 +1,6 @@ import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js"; -import { type Job, type JobSeed, type JobType, jobId } from "@/domain/jobs/job.type.js"; +import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js"; import { randomUUID } from "node:crypto"; import { hostname } from "node:os"; import { ILogger } from "@/utils/logger.js"; @@ -20,6 +20,7 @@ const SERVICE_NAME = "JobQueue"; const POLL_MS = 250; const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale +const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000; const CONCURRENCY: Record = { check: 500, @@ -179,7 +180,28 @@ export class DBQueueWorker implements IWorker { if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff }; + // Extend the lock while a job runs so a slow-but-alive job is never reclaimed. + // Returns false once we no longer hold the lock, so the renewal timer can stop. + private renewLock = async (job: Job): Promise => { + try { + const held = await this.jobsRepository.renewLocks([job.id], Date.now()); + if (held === 0) { + this.logger.warn({ message: `Lost lock on job ${job.id}, stopping renewal`, service: SERVICE_NAME, method: "renewLock" }); + return false; + } + return true; + } catch (error: unknown) { + // Transient failure, keep renewing; a missed renewal is absorbed by the 1/3 margin + this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "renewLock" }); + return true; + } + }; + private runJob = async (job: Job) => { + const renewTimer = setInterval(async () => { + const renewLock = await this.renewLock(job); + if (!renewLock) clearInterval(renewTimer); + }, LOCK_RENEW_MS); try { switch (job.type) { case "check": @@ -212,6 +234,8 @@ export class DBQueueWorker implements IWorker { service: SERVICE_NAME, method: `runJob:${job.type}`, }); + } finally { + clearInterval(renewTimer); } }; @@ -314,7 +338,7 @@ export class DBQueueWorker implements IWorker { clearTimeout(timer); } this.timers.clear(); // Clear the map out - // Deregister promptly so the worker drops off diagnostics before the TTL would + // Remove workers from registry await this.queueWorkersRepository.deleteById(this.workerId); }; getMetrics = async (): Promise => { diff --git a/server/test/unit/services/lessSimpleQueue.test.ts b/server/test/unit/services/lessSimpleQueue.test.ts deleted file mode 100644 index 8b29df1bb..000000000 --- a/server/test/unit/services/lessSimpleQueue.test.ts +++ /dev/null @@ -1,791 +0,0 @@ -import { describe, expect, it, jest } from "@jest/globals"; -import { createMockLogger } from "../../helpers/createMockLogger.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; - -// ── Mock Scheduler / MongoStore ────────────────────────────────────────────── - -const createScheduler = () => { - const listeners: Record = {}; - return { - workerId: "host-1:1234:abcd", - on: jest.fn((event: string, cb: Function) => { - listeners[event] = listeners[event] || []; - listeners[event].push(cb); - }), - off: jest.fn((event: string, cb: Function) => { - listeners[event] = (listeners[event] || []).filter((fn) => fn !== cb); - }), - emit: (event: string, ...args: unknown[]) => { - (listeners[event] || []).forEach((cb) => cb(...args)); - }, - start: jest.fn().mockResolvedValue(true), - stop: jest.fn().mockResolvedValue(true), - addTemplate: jest.fn(), - addJob: jest.fn(), - addJobs: jest.fn().mockResolvedValue({ inserted: 0, upserted: 0, modified: 0, failures: [] }), - removeJob: jest.fn(), - getJob: jest.fn().mockResolvedValue(null), - getJobs: jest.fn().mockResolvedValue([]), - countJobs: jest.fn().mockResolvedValue(0), - getStats: jest.fn().mockResolvedValue({ jobs: 0, activeJobs: 0, failingJobs: 0, totalRuns: 0, totalFailures: 0, jobsWithFailures: [] }), - pauseJob: jest.fn().mockResolvedValue(true), - resumeJob: jest.fn().mockResolvedValue(true), - updateJob: jest.fn(), - flushJobs: jest.fn().mockResolvedValue(true), - }; -}; - -const mockSchedulerInstance = createScheduler(); -const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance); -const MockMongoStore = jest.fn(); - -jest.unstable_mockModule("less-simple-scheduler", () => ({ - Scheduler: MockScheduler, - MongoStore: MockMongoStore, -})); - -const { LessSimpleWorker: LessSimpleQueue } = await import("../../../src/worker/worker.less-simple.ts"); - -const createQueueHelper = () => ({ - getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()), - getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()), - getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()), - getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()), -}); - -const createMonitorsRepo = () => ({ - findAll: jest.fn().mockResolvedValue([]), -}); - -const createWorkersRepo = () => ({ - upsert: jest.fn().mockResolvedValue(undefined), - findRecent: jest.fn().mockResolvedValue([]), - deleteById: jest.fn().mockResolvedValue(undefined), -}); - -const createQueue = (overrides?: Record) => { - const logger = createMockLogger(); - const helper = createQueueHelper(); - const monitorsRepository = createMonitorsRepo(); - const workersRepository = createWorkersRepo(); - const scheduler = createScheduler(); - - const defaults = { logger, helper, monitorsRepository, workersRepository, scheduler, queueMode: "primary", ...overrides }; - - const queue = new LessSimpleQueue( - defaults.logger as any, - defaults.helper as any, - defaults.monitorsRepository as any, - defaults.workersRepository as any, - defaults.scheduler as any, - defaults.queueMode as any - ); - return { queue, ...defaults }; -}; - -const makeMonitor = (overrides?: Partial): Monitor => - ({ - id: "mon-1", - teamId: "team-1", - type: "http", - interval: 60000, - isActive: true, - geoCheckEnabled: false, - geoCheckInterval: 300000, - geoCheckLocations: [], - ...overrides, - }) as Monitor; - -// ── Tests ──────────────────────────────────────────────────────────────────── - -describe("LessSimpleQueue", () => { - describe("serviceName", () => { - it("returns JobQueue", () => { - const { queue } = createQueue(); - expect(queue.serviceName).toBe("JobQueue"); - }); - }); - - // ── init (primary) ───────────────────────────────────────────────────────── - - describe("init - primary mode", () => { - it("starts scheduler, registers all templates, seeds monitors and system jobs", async () => { - const { queue, scheduler, monitorsRepository } = createQueue(); - - const result = await queue.init(); - - expect(result).toBe(true); - expect(scheduler.start).toHaveBeenCalled(); - expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function)); - expect(monitorsRepository.findAll).toHaveBeenCalled(); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" })); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" })); - }); - - it("registers existing monitors in a single bulk write", async () => { - const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })]; - const { queue, scheduler, monitorsRepository } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors); - - await queue.init(); - - expect(scheduler.addJobs).toHaveBeenCalledTimes(1); - expect(scheduler.addJobs).toHaveBeenCalledWith( - expect.arrayContaining([ - expect.objectContaining({ id: "m1", template: "monitor-job", jitter: true }), - expect.objectContaining({ id: "m2", template: "monitor-job", jitter: true }), - ]) - ); - }); - - it("returns true and skips system jobs when findAll returns null", async () => { - const { queue, monitorsRepository, scheduler } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockResolvedValue(null); - - const result = await queue.init(); - - expect(result).toBe(true); - expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" })); - }); - - it("returns false and logs error on failure", async () => { - const { queue, logger, monitorsRepository } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down")); - - const result = await queue.init(); - - expect(result).toBe(false); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") })); - }); - - it("logs error with String(error) for non-Error exceptions", async () => { - const { queue, logger, monitorsRepository } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error"); - - const result = await queue.init(); - - expect(result).toBe(false); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") })); - }); - }); - - // ── init (worker) ────────────────────────────────────────────────────────── - - describe("init - worker mode", () => { - it("starts scheduler and registers all templates", async () => { - const { queue, scheduler } = createQueue(); - - const result = await queue.init("worker"); - - expect(result).toBe(true); - expect(scheduler.start).toHaveBeenCalled(); - expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function)); - }); - - it("does NOT seed monitor jobs (no findAll, primary populates the queue)", async () => { - const { queue, monitorsRepository } = createQueue(); - - await queue.init("worker"); - - expect(monitorsRepository.findAll).not.toHaveBeenCalled(); - }); - - it("does NOT seed cleanup system jobs", async () => { - const { queue, scheduler } = createQueue(); - - await queue.init("worker"); - - expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" })); - expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" })); - expect(scheduler.addJob).not.toHaveBeenCalled(); - }); - - it("logs the initialized worker with its workerId", async () => { - const { queue, logger } = createQueue(); - - await queue.init("worker"); - - expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Worker queue: host-1:1234:abcd initialized" })); - }); - }); - - // ── registerListeners ────────────────────────────────────────────────────── - - describe("registerListeners (via constructor)", () => { - it("logs on scheduler:start", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("scheduler:start", "host-1:1234:abcd"); - expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" })); - }); - - it("logs on scheduler:stop", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("scheduler:stop", "host-1:1234:abcd"); - expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" })); - }); - - it("logs on scheduler:error with Error", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("scheduler:error", "w1", new Error("boom")); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: boom" })); - }); - - it("logs on scheduler:error with non-Error", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("scheduler:error", "w1", "string error"); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: string error", stack: undefined })); - }); - - it("logs on job:locked", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:locked", "w1", { id: "m1" }); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 locked" })); - }); - - it("logs on job:abort", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:abort", "w1", { id: "m1" }, "timeout"); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 aborted: timeout" })); - }); - - it("logs on job:attempt", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:attempt", "w1", { id: "m1" }, 2); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 attempt 2" })); - }); - - it("logs on job:complete", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:complete", "w1", { id: "m1" }); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 completed successfully" })); - }); - - it("logs on job:exhausted with Error", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:exhausted", "w1", { id: "m1" }, new Error("gave up")); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up" })); - }); - - it("logs on job:exhausted with non-Error", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:exhausted", "w1", { id: "m1" }, "gave up"); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up", stack: undefined })); - }); - - it("logs on job:fail with Error", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:fail", "w1", { id: "m1" }, new Error("oops"), 1); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops" })); - }); - - it("logs on job:fail with non-Error", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:fail", "w1", { id: "m1" }, "oops", 1); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops", stack: undefined })); - }); - - it("logs on job:start", () => { - const { scheduler, logger } = createQueue(); - scheduler.emit("job:start", "w1", { id: "m1" }); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 started" })); - }); - }); - - // ── addJob ───────────────────────────────────────────────────────────────── - - describe("addJob", () => { - it("adds monitor job to scheduler", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor()); - expect(scheduler.addJobs).toHaveBeenCalledWith( - expect.arrayContaining([expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true })]) - ); - }); - - it("adds geo check job when geoCheckEnabled and type supports it", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 })); - expect(scheduler.addJobs).toHaveBeenCalledWith( - expect.arrayContaining([expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 })]) - ); - }); - - it("skips geo job when type does not support geo checks", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" })); - expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })])); - }); - - it("skips geo job when geoCheckEnabled is false", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" })); - expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })])); - }); - }); - - // ── deleteJob ────────────────────────────────────────────────────────────── - - describe("deleteJob", () => { - it("removes monitor job", async () => { - const { queue, scheduler } = createQueue(); - await queue.deleteJob(makeMonitor()); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1"); - }); - - it("removes geo job if it exists", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.deleteJob(makeMonitor()); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo"); - }); - - it("does not remove geo job if it does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.deleteJob(makeMonitor()); - expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo"); - }); - }); - - // ── pauseJob ─────────────────────────────────────────────────────────────── - - describe("pauseJob", () => { - it("pauses monitor and geo job when geo exists", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.pauseJob(makeMonitor()); - - expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo"); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" })); - }); - - it("does not pause geo job when it does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.pauseJob(makeMonitor()); - expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo"); - }); - - it("throws when scheduler.pauseJob returns false for the main job", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.pauseJob as jest.Mock).mockResolvedValue(false); - await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor"); - }); - - it("logs error but does not throw when geo pauseJob returns false", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - (scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo"); - - await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined(); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" })); - }); - }); - - // ── resumeJob ────────────────────────────────────────────────────────────── - - describe("resumeJob", () => { - it("resumes monitor and geo job when geo exists", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.resumeJob(makeMonitor()); - - expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo"); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" })); - }); - - it("does not resume geo job when it does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.resumeJob(makeMonitor()); - expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo"); - }); - - it("throws when scheduler.resumeJob returns false for the main job", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.resumeJob as jest.Mock).mockResolvedValue(false); - await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor"); - }); - - it("logs error but does not throw when geo resumeJob returns false", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - (scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo"); - - await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined(); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" })); - }); - }); - - // ── updateJob / syncGeoJob ───────────────────────────────────────────────── - - describe("updateJob", () => { - it("updates monitor job with new interval", async () => { - const { queue, scheduler } = createQueue(); - const monitor = makeMonitor({ interval: 120000 }); - await queue.updateJob(monitor); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor })); - }); - - it("removes existing geo job when geoCheckEnabled is false", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" })); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo"); - }); - - it("removes existing geo job when type does not support geo checks", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" })); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo"); - }); - - it("updates existing geo job when geoCheckEnabled", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }); - await queue.updateJob(monitor); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 })); - }); - - it("creates new geo job when enabled but no existing job", async () => { - const { queue, scheduler } = createQueue(); - const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }); - await queue.updateJob(monitor); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" })); - }); - }); - - // ── shutdown ─────────────────────────────────────────────────────────────── - - describe("shutdown", () => { - it("stops the scheduler", async () => { - const { queue, scheduler } = createQueue(); - await queue.shutdown(); - expect(scheduler.stop).toHaveBeenCalled(); - }); - }); - - // ── getMetrics ───────────────────────────────────────────────────────────── - - describe("getMetrics", () => { - it("returns empty metrics when no jobs", async () => { - const { queue } = createQueue(); - const metrics = await queue.getMetrics(); - expect(metrics).toEqual({ jobs: 0, activeJobs: 0, failingJobs: 0, jobsWithFailures: [], totalRuns: 0, totalFailures: 0, workers: [] }); - }); - - it("maps the store's aggregate counters through", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getStats as jest.Mock).mockResolvedValue({ - jobs: 2, - activeJobs: 1, - failingJobs: 1, - totalRuns: 15, - totalFailures: 2, - jobsWithFailures: [], - }); - const metrics = await queue.getMetrics(); - expect(metrics).toMatchObject({ jobs: 2, activeJobs: 1, failingJobs: 1, totalRuns: 15, totalFailures: 2 }); - }); - - it("derives monitor fields for failing monitor jobs from job data", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getStats as jest.Mock).mockResolvedValue({ - jobs: 1, - activeJobs: 0, - failingJobs: 1, - totalRuns: 10, - totalFailures: 2, - jobsWithFailures: [{ id: "m1", data: { url: "http://a.com", type: "http" }, failedAt: 200, failCount: 2, failReason: "timeout" }], - }); - const metrics = await queue.getMetrics(); - expect(metrics.jobsWithFailures).toEqual([ - { monitorId: "m1", monitorUrl: "http://a.com", monitorType: "http", failedAt: 200, failCount: 2, failReason: "timeout" }, - ]); - }); - - it("leaves monitor fields null for non-monitor (cleanup) failures", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getStats as jest.Mock).mockResolvedValue({ - jobs: 1, - activeJobs: 0, - failingJobs: 1, - totalRuns: 1, - totalFailures: 1, - jobsWithFailures: [{ id: "cleanup-orphaned", data: undefined, failedAt: 100, failCount: 1, failReason: "boom" }], - }); - const metrics = await queue.getMetrics(); - expect(metrics.jobsWithFailures[0]).toEqual({ - monitorId: "cleanup-orphaned", - monitorUrl: null, - monitorType: null, - failedAt: 100, - failCount: 1, - failReason: "boom", - }); - }); - }); - - // ── getMetrics: worker registry ────────────────────────────────────────────── - - describe("getMetrics - worker registry", () => { - it("returns the alive workers from the registry", async () => { - const { queue, workersRepository } = createQueue(); - const alive = [ - { workerId: "host:1:aaa", mode: "primary", lastSeenAt: 1000 }, - { workerId: "host:2:bbb", mode: "worker", lastSeenAt: 2000 }, - ]; - (workersRepository.findRecent as jest.Mock).mockResolvedValue(alive); - - const metrics = await queue.getMetrics(); - - expect(workersRepository.findRecent).toHaveBeenCalledWith(30000); - expect(metrics.workers).toEqual(alive); - }); - }); - - // ── getJobs ──────────────────────────────────────────────────────────────── - - describe("getJobs", () => { - it("maps scheduler jobs to summaries", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { - id: "m1", - template: "monitor-job", - active: true, - repeat: 60000, - lockedAt: null, - runCount: 5, - failCount: 1, - lastError: "timeout", - lastStartedAt: 1000, - lastFinishedAt: 1100, - lastFailedAt: 900, - data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true }, - }, - ]); - const { jobs } = await queue.getJobs({}); - - expect(jobs).toHaveLength(1); - expect(jobs[0]).toEqual({ - monitorId: "m1", - monitorUrl: "http://a.com", - monitorType: "http", - monitorInterval: 60000, - monitorGeoInterval: 300000, - monitorActive: true, - active: true, - repeat: 60000, - lockedBy: null, - lockedUntil: null, - lockedAt: null, - runCount: 5, - failCount: 1, - failReason: "timeout", - lastRunAt: 1000, - lastFinishedAt: 1100, - lastRunTook: 100, - lastFailedAt: 900, - }); - }); - - it("returns null lastRunTook when job is locked", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { - id: "m1", - template: "monitor-job", - active: true, - lockedAt: 999, - runCount: 1, - lastStartedAt: 1000, - lastFinishedAt: 1100, - data: { url: "a", type: "http", interval: 60000 }, - }, - ]); - const { jobs } = await queue.getJobs({}); - expect(jobs[0].lastRunTook).toBeNull(); - }); - - it("leaves monitor fields null for non-monitor jobs", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, data: undefined }]); - const { jobs } = await queue.getJobs({}); - expect(jobs[0].monitorUrl).toBeNull(); - expect(jobs[0].monitorType).toBeNull(); - expect(jobs[0].monitorInterval).toBeNull(); - expect(jobs[0].monitorGeoInterval).toBeNull(); - expect(jobs[0].monitorActive).toBeNull(); - }); - - it("pushes pagination to the scheduler and returns its total count", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { id: "m1", template: "monitor-job", active: true, data: { url: "a", type: "http", interval: 60000 } }, - ]); - (scheduler.countJobs as jest.Mock).mockResolvedValue(42); - - const { jobs, count } = await queue.getJobs({ page: 2, rowsPerPage: 10 }); - - expect(scheduler.getJobs).toHaveBeenCalledWith({ skip: 20, limit: 10 }); - expect(count).toBe(42); - expect(jobs).toHaveLength(1); - }); - - it("requests all jobs when rowsPerPage is omitted", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.countJobs as jest.Mock).mockResolvedValue(7); - - const { count } = await queue.getJobs({}); - - expect(scheduler.getJobs).toHaveBeenCalledWith(undefined); - expect(count).toBe(7); - }); - }); - - // ── flushQueues ──────────────────────────────────────────────────────────── - - describe("flushQueues", () => { - it("stops, flushes, and reinitializes", async () => { - const { queue, scheduler } = createQueue(); - const result = await queue.flushQueues(); - expect(scheduler.stop).toHaveBeenCalled(); - expect(scheduler.flushJobs).toHaveBeenCalled(); - expect(result.success).toBe(true); - }); - - it("flushes before stopping, so the store is still connected for removeAll", async () => { - const { queue, scheduler } = createQueue(); - const order: string[] = []; - (scheduler.flushJobs as jest.Mock).mockImplementation(async () => { - order.push("flush"); - return true; - }); - (scheduler.stop as jest.Mock).mockImplementation(async () => { - order.push("stop"); - return true; - }); - await queue.flushQueues(); - expect(order).toEqual(["flush", "stop"]); - }); - - it("refuses to flush on a non-primary worker without touching the scheduler", async () => { - const { queue, scheduler } = createQueue({ queueMode: "worker" }); - const result = await queue.flushQueues(); - expect(result.success).toBe(false); - expect(scheduler.flushJobs).not.toHaveBeenCalled(); - expect(scheduler.stop).not.toHaveBeenCalled(); - }); - - it("returns false when flush fails", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.flushJobs as jest.Mock).mockResolvedValue(false); - const result = await queue.flushQueues(); - expect(result.success).toBe(false); - }); - }); - - // ── static create ────────────────────────────────────────────────────────── - - describe("static create", () => { - const envSettings = { dbConnectionString: "mongodb://localhost:27017/test_db" } as any; - - it("creates a MongoStore + Scheduler and inits in primary mode", async () => { - const logger = createMockLogger(); - const helper = createQueueHelper(); - const monitorsRepository = createMonitorsRepo(); - const workersRepository = createWorkersRepo(); - - const instance = await LessSimpleQueue.create( - logger as any, - helper as any, - monitorsRepository as any, - workersRepository as any, - envSettings, - "primary" - ); - - expect(MockMongoStore).toHaveBeenCalledWith(expect.objectContaining({ url: "mongodb://localhost:27017/test_db" })); - expect(MockScheduler).toHaveBeenCalled(); - expect(instance).toBeInstanceOf(LessSimpleQueue); - expect(mockSchedulerInstance.start).toHaveBeenCalled(); - expect(monitorsRepository.findAll).toHaveBeenCalled(); - }); - - it("inits in worker mode without seeding monitors", async () => { - const logger = createMockLogger(); - const helper = createQueueHelper(); - const monitorsRepository = createMonitorsRepo(); - const workersRepository = createWorkersRepo(); - - const instance = await LessSimpleQueue.create( - logger as any, - helper as any, - monitorsRepository as any, - workersRepository as any, - envSettings, - "worker" - ); - - expect(instance).toBeInstanceOf(LessSimpleQueue); - expect(monitorsRepository.findAll).not.toHaveBeenCalled(); - }); - }); - - // ── worker heartbeat / deregistration ──────────────────────────────────────── - - describe("worker heartbeat", () => { - it("writes a heartbeat on init with the workerId and mode (primary)", async () => { - const { queue, workersRepository } = createQueue(); - await queue.init("primary"); - expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "primary"); - }); - - it("writes a heartbeat on init with the workerId and mode (worker)", async () => { - const { queue, workersRepository } = createQueue(); - await queue.init("worker"); - expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker"); - }); - - it("beats again on each scheduler:heartbeat event", async () => { - const { queue, scheduler, workersRepository } = createQueue(); - await queue.init("worker"); - (workersRepository.upsert as jest.Mock).mockClear(); - - scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd"); - scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd"); - - expect(workersRepository.upsert).toHaveBeenCalledTimes(2); - expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker"); - }); - - it("stops beating after shutdown", async () => { - const { queue, scheduler, workersRepository } = createQueue(); - await queue.init("worker"); - await queue.shutdown(); - (workersRepository.upsert as jest.Mock).mockClear(); - - scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd"); - - expect(workersRepository.upsert).not.toHaveBeenCalled(); - }); - - it("deregisters the worker and stops the scheduler on shutdown", async () => { - const { queue, scheduler, workersRepository } = createQueue(); - await queue.init("worker"); - await queue.shutdown(); - expect(workersRepository.deleteById).toHaveBeenCalledWith("host-1:1234:abcd"); - expect(scheduler.stop).toHaveBeenCalled(); - }); - }); -}); diff --git a/server/test/unit/services/superSimpleQueue.test.ts b/server/test/unit/services/superSimpleQueue.test.ts deleted file mode 100644 index b9e9b8314..000000000 --- a/server/test/unit/services/superSimpleQueue.test.ts +++ /dev/null @@ -1,712 +0,0 @@ -import { describe, expect, it, jest, beforeEach } from "@jest/globals"; -import { createMockLogger } from "../../helpers/createMockLogger.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; - -// ── Mock Scheduler ─────────────────────────────────────────────────────────── - -const createScheduler = () => { - const listeners: Record = {}; - return { - on: jest.fn((event: string, cb: Function) => { - listeners[event] = listeners[event] || []; - listeners[event].push(cb); - }), - emit: (event: string, ...args: unknown[]) => { - (listeners[event] || []).forEach((cb) => cb(...args)); - }, - start: jest.fn(), - stop: jest.fn().mockResolvedValue(true), - addTemplate: jest.fn(), - addJob: jest.fn(), - removeJob: jest.fn(), - getJob: jest.fn().mockResolvedValue(null), - getJobs: jest.fn().mockResolvedValue([]), - pauseJob: jest.fn().mockResolvedValue(true), - resumeJob: jest.fn().mockResolvedValue(true), - updateJob: jest.fn(), - flushJobs: jest.fn().mockResolvedValue(true), - }; -}; - -const mockSchedulerInstance = createScheduler(); -const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance); - -jest.unstable_mockModule("super-simple-scheduler", () => ({ - default: MockScheduler, -})); - -const { SuperSimpleQueue } = await import("../../../src/worker/worker.super-simple.ts"); - -const createQueueHelper = () => ({ - getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()), - getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()), - getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()), - getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()), -}); - -const createMonitorsRepo = () => ({ - findAll: jest.fn().mockResolvedValue([]), -}); - -const createQueue = (overrides?: Record) => { - const logger = createMockLogger(); - const helper = createQueueHelper(); - const monitorsRepository = createMonitorsRepo(); - const scheduler = createScheduler(); - - const defaults = { logger, helper, monitorsRepository, scheduler, ...overrides }; - - const queue = new SuperSimpleQueue(defaults.logger as any, defaults.helper as any, defaults.monitorsRepository as any, defaults.scheduler as any); - return { queue, ...defaults }; -}; - -const makeMonitor = (overrides?: Partial): Monitor => - ({ - id: "mon-1", - teamId: "team-1", - type: "http", - interval: 60000, - isActive: true, - geoCheckEnabled: false, - geoCheckInterval: 300000, - geoCheckLocations: [], - ...overrides, - }) as Monitor; - -// ── Tests ──────────────────────────────────────────────────────────────────── - -describe("SuperSimpleQueue", () => { - describe("serviceName", () => { - it("returns JobQueue", () => { - const { queue } = createQueue(); - expect(queue.serviceName).toBe("JobQueue"); - }); - }); - - // ── init ───────────────────────────────────────────────────────────────── - - describe("init", () => { - it("registers listeners, starts scheduler, adds templates and system jobs", async () => { - const { queue, scheduler, helper } = createQueue(); - - const result = await queue.init(); - - expect(result).toBe(true); - expect(scheduler.start).toHaveBeenCalled(); - expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function)); - expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function)); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" })); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" })); - }); - - it("schedules existing monitors with staggered offsets", async () => { - jest.useFakeTimers(); - const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })]; - const { queue, scheduler, monitorsRepository } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors); - - await queue.init(); - jest.runAllTimers(); - - // Each monitor gets addJob called (once for the monitor itself via addJob method) - // The init schedules via setTimeout → addJob, plus the two system jobs - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m1", template: "monitor-job" })); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m2", template: "monitor-job" })); - jest.useRealTimers(); - }); - - it("returns true when findAll returns null", async () => { - const { queue, monitorsRepository, scheduler } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockResolvedValue(null); - - const result = await queue.init(); - - expect(result).toBe(true); - // System jobs should not be added when monitors is null (early return) - expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" })); - }); - - it("returns false and logs error on failure", async () => { - const { queue, logger, monitorsRepository } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down")); - - const result = await queue.init(); - - expect(result).toBe(false); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") })); - }); - - it("logs error with String(error) for non-Error exceptions", async () => { - const { queue, logger, monitorsRepository } = createQueue(); - (monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error"); - - const result = await queue.init(); - - expect(result).toBe(false); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") })); - }); - }); - - // ── registerListeners ──────────────────────────────────────────────────── - - describe("registerListeners (via init)", () => { - it("logs on scheduler:start", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("scheduler:start"); - expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" })); - }); - - it("logs on scheduler:stop", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("scheduler:stop"); - expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" })); - }); - - it("logs on scheduler:error with Error", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("scheduler:error", new Error("boom")); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: boom" })); - }); - - it("logs on scheduler:error with non-Error", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("scheduler:error", "string error"); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: string error", stack: undefined })); - }); - - it("logs on job:abort", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:abort", { id: "m1" }, "timeout"); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 aborted: timeout" })); - }); - - it("logs on job:attempt", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:attempt", { id: "m1" }, 2); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 attempt 2" })); - }); - - it("logs on job:complete", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:complete", { id: "m1" }); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 completed successfully" })); - }); - - it("logs on job:exhausted with Error", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:exhausted", { id: "m1" }, new Error("gave up")); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up" })); - }); - - it("logs on job:exhausted with non-Error", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:exhausted", { id: "m1" }, "gave up"); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up", stack: undefined })); - }); - - it("logs on job:fail with Error", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:fail", { id: "m1" }, new Error("oops"), 1); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops" })); - }); - - it("logs on job:fail with non-Error", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:fail", { id: "m1" }, "oops", 1); - expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops", stack: undefined })); - }); - - it("logs on job:start", async () => { - const { queue, scheduler, logger } = createQueue(); - await queue.init(); - scheduler.emit("job:start", { id: "m1" }); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 started" })); - }); - }); - - // ── addJob ─────────────────────────────────────────────────────────────── - - describe("addJob", () => { - it("adds monitor job to scheduler", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor()); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true })); - }); - - it("adds geo check job when geoCheckEnabled and type supports it", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 })); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 })); - }); - - it("skips geo job when type does not support geo checks", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" })); - expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" })); - }); - - it("skips geo job when geoCheckEnabled is false", async () => { - const { queue, scheduler } = createQueue(); - await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" })); - expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" })); - }); - }); - - // ── deleteJob ──────────────────────────────────────────────────────────── - - describe("deleteJob", () => { - it("removes monitor job", async () => { - const { queue, scheduler } = createQueue(); - await queue.deleteJob(makeMonitor()); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1"); - }); - - it("removes geo job if it exists", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.deleteJob(makeMonitor()); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo"); - }); - - it("does not remove geo job if it does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.deleteJob(makeMonitor()); - expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo"); - }); - }); - - // ── pauseJob ───────────────────────────────────────────────────────────── - - describe("pauseJob", () => { - it("pauses monitor and pauses geo job when geo exists", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.pauseJob(makeMonitor()); - - expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo"); - expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo"); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" })); - }); - - it("refreshes scheduler data with current monitor on pause", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - const monitor = makeMonitor({ isActive: false, status: "paused" }); - - await queue.pauseJob(monitor); - - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor }); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor }); - }); - - it("does not pause geo job when it does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.pauseJob(makeMonitor()); - expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo"); - }); - - it("does not refresh geo data when geo job does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.pauseJob(makeMonitor()); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() })); - expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything()); - }); - - it("throws when scheduler.pauseJob returns false for the main job", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.pauseJob as jest.Mock).mockResolvedValue(false); - await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor"); - }); - - it("does not refresh data when main pauseJob throws", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.pauseJob as jest.Mock).mockResolvedValue(false); - await expect(queue.pauseJob(makeMonitor())).rejects.toThrow(); - expect(scheduler.updateJob).not.toHaveBeenCalled(); - }); - - it("logs error but does not throw when geo pauseJob returns false", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - (scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo"); - - await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined(); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" })); - }); - }); - - // ── resumeJob ──────────────────────────────────────────────────────────── - - describe("resumeJob", () => { - it("resumes monitor and geo job when geo exists", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.resumeJob(makeMonitor()); - - expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo"); - expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" })); - }); - - it("refreshes scheduler data with current monitor on resume", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - const monitor = makeMonitor({ isActive: true, status: "initializing" }); - - await queue.resumeJob(monitor); - - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor }); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor }); - }); - - it("does not resume geo job when it does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.resumeJob(makeMonitor()); - expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1"); - expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo"); - }); - - it("does not refresh geo data when geo job does not exist", async () => { - const { queue, scheduler } = createQueue(); - await queue.resumeJob(makeMonitor()); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() })); - expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything()); - }); - - it("throws when scheduler.resumeJob returns false for the main job", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.resumeJob as jest.Mock).mockResolvedValue(false); - await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor"); - }); - - it("does not refresh data when main resumeJob throws", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.resumeJob as jest.Mock).mockResolvedValue(false); - await expect(queue.resumeJob(makeMonitor())).rejects.toThrow(); - expect(scheduler.updateJob).not.toHaveBeenCalled(); - }); - - it("logs error but does not throw when geo resumeJob returns false", async () => { - const { queue, scheduler, logger } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - (scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo"); - - await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined(); - expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" })); - }); - - it("preserves geo job through a pause/resume cycle", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - - await queue.pauseJob(makeMonitor()); - await queue.resumeJob(makeMonitor()); - - expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo"); - expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo"); - expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo"); - }); - }); - - // ── updateJob ──────────────────────────────────────────────────────────── - - describe("updateJob", () => { - it("updates monitor job and syncs geo job", async () => { - const { queue, scheduler } = createQueue(); - const monitor = makeMonitor({ interval: 120000 }); - await queue.updateJob(monitor); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor })); - }); - }); - - // ── syncGeoJob (via updateJob) ─────────────────────────────────────────── - - describe("syncGeoJob (via updateJob)", () => { - it("removes existing geo job when geoCheckEnabled is false", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" })); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo"); - }); - - it("removes existing geo job when type does not support geo checks", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" })); - expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo"); - }); - - it("does not remove when geo disabled and no existing geo job", async () => { - const { queue, scheduler } = createQueue(); - await queue.updateJob(makeMonitor({ geoCheckEnabled: false })); - expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo"); - }); - - it("updates existing geo job when geoCheckEnabled", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" }); - const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }); - await queue.updateJob(monitor); - expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 })); - }); - - it("creates new geo job when enabled but no existing job", async () => { - const { queue, scheduler } = createQueue(); - const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }); - await queue.updateJob(monitor); - expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" })); - }); - }); - - // ── shutdown ────────────────────────────────────────────────────────────── - - describe("shutdown", () => { - it("stops the scheduler", async () => { - const { queue, scheduler } = createQueue(); - await queue.shutdown(); - expect(scheduler.stop).toHaveBeenCalled(); - }); - }); - - // ── getMetrics ─────────────────────────────────────────────────────────── - - describe("getMetrics", () => { - it("returns empty job metrics plus this process as the sole worker when no jobs", async () => { - const { queue } = createQueue(); - const metrics = await queue.getMetrics(); - expect(metrics).toEqual({ - jobs: 0, - activeJobs: 0, - failingJobs: 0, - jobsWithFailures: [], - totalRuns: 0, - totalFailures: 0, - workers: [ - { - // hostname:pid:uuid, matching less-simple-scheduler's workerId format - workerId: expect.stringMatching(/^.+:\d+:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/), - mode: "primary", - lastSeenAt: expect.any(Number), - }, - ], - }); - }); - - it("aggregates metrics from jobs", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { id: "m1", runCount: 10, failCount: 2, lastFailedAt: 200, lastRunAt: 100, lockedAt: null, data: { url: "http://a.com", type: "http" } }, - { id: "m2", runCount: 5, failCount: 0, lastFailedAt: 0, lastRunAt: 50, lockedAt: 999, data: { url: "http://b.com", type: "ping" } }, - ]); - const metrics = await queue.getMetrics(); - - expect(metrics.jobs).toBe(2); - expect(metrics.totalRuns).toBe(15); - expect(metrics.totalFailures).toBe(2); - expect(metrics.activeJobs).toBe(1); - expect(metrics.failingJobs).toBe(1); - expect(metrics.jobsWithFailures).toHaveLength(1); - expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ monitorId: "m1", failCount: 2, monitorUrl: "http://a.com" })); - }); - - it("handles jobs with undefined fields", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", data: null }]); - const metrics = await queue.getMetrics(); - expect(metrics.jobs).toBe(1); - expect(metrics.totalRuns).toBe(0); - }); - - it("handles failing job with null data", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { id: "m1", runCount: 1, failCount: 1, lastFailedAt: 100, lastRunAt: 50, data: null, lastFailReason: "timeout" }, - ]); - const metrics = await queue.getMetrics(); - expect(metrics.jobsWithFailures).toHaveLength(1); - expect(metrics.jobsWithFailures[0].monitorUrl).toBeNull(); - expect(metrics.jobsWithFailures[0].monitorType).toBeNull(); - expect(metrics.jobsWithFailures[0].failReason).toBe("timeout"); - }); - - it("handles failing job with undefined lastFailedAt and lastFailReason", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", runCount: 1, failCount: 1, data: { url: "http://a.com", type: "http" } }]); - const metrics = await queue.getMetrics(); - expect(metrics.jobsWithFailures[0].failedAt).toBeNull(); - expect(metrics.jobsWithFailures[0].failReason).toBeNull(); - }); - - it("does not count as failing when lastFailedAt < lastRunAt", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { id: "m1", runCount: 10, failCount: 1, lastFailedAt: 50, lastRunAt: 100, data: { url: "a" } }, - ]); - const metrics = await queue.getMetrics(); - expect(metrics.failingJobs).toBe(0); - expect(metrics.jobsWithFailures).toHaveLength(1); - }); - }); - - // ── getJobs ────────────────────────────────────────────────────────────── - - describe("getJobs", () => { - it("maps scheduler jobs to summaries", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { - id: "m1", - active: true, - repeat: 60000, - lockedAt: null, - runCount: 5, - failCount: 1, - lastFailReason: "timeout", - lastRunAt: 1000, - lastFinishedAt: 1100, - lastFailedAt: 900, - data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true }, - }, - ]); - const { jobs } = await queue.getJobs({}); - - expect(jobs).toHaveLength(1); - expect(jobs[0]).toEqual({ - monitorId: "m1", - monitorUrl: "http://a.com", - monitorType: "http", - monitorInterval: 60000, - monitorGeoInterval: 300000, - monitorActive: true, - active: true, - repeat: 60000, - lockedBy: null, - lockedUntil: null, - lockedAt: null, - runCount: 5, - failCount: 1, - failReason: "timeout", - lastRunAt: 1000, - lastFinishedAt: 1100, - lastRunTook: 100, - lastFailedAt: 900, - }); - }); - - it("returns null lastRunTook when job is locked", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { id: "m1", active: true, lockedAt: 999, runCount: 1, data: { url: "a", type: "http", interval: 60000 } }, - ]); - const { jobs } = await queue.getJobs({}); - expect(jobs[0].lastRunTook).toBeNull(); - }); - - it("handles jobs with missing data", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", active: true, data: null }]); - const { jobs } = await queue.getJobs({}); - expect(jobs[0].monitorUrl).toBeNull(); - expect(jobs[0].monitorType).toBeNull(); - expect(jobs[0].monitorInterval).toBeNull(); - expect(jobs[0].monitorGeoInterval).toBeNull(); - expect(jobs[0].monitorActive).toBeNull(); - expect(jobs[0].repeat).toBeNull(); - }); - - it("exposes monitor.isActive as monitorActive in the summary", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { id: "m1", active: true, data: { url: "a", type: "http", interval: 60000, isActive: true } }, - { id: "m2", active: false, data: { url: "b", type: "http", interval: 60000, isActive: false } }, - ]); - const { jobs } = await queue.getJobs({}); - expect(jobs[0].monitorActive).toBe(true); - expect(jobs[1].monitorActive).toBe(false); - }); - - it("exposes geo check interval and repeat", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue([ - { id: "m1", active: true, repeat: 120000, data: { url: "a", type: "http", interval: 60000, geoCheckInterval: 600000 } }, - ]); - const { jobs } = await queue.getJobs({}); - expect(jobs[0].repeat).toBe(120000); - expect(jobs[0].monitorGeoInterval).toBe(600000); - }); - - it("paginates jobs and returns the total count", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue( - Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } })) - ); - - const { jobs, count } = await queue.getJobs({ page: 1, rowsPerPage: 10 }); - expect(count).toBe(25); - expect(jobs).toHaveLength(10); - expect(jobs[0].monitorId).toBe("m10"); - }); - - it("returns all jobs when rowsPerPage is omitted", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.getJobs as jest.Mock).mockResolvedValue( - Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } })) - ); - - const { jobs, count } = await queue.getJobs({}); - expect(count).toBe(25); - expect(jobs).toHaveLength(25); - }); - }); - - // ── flushQueues ────────────────────────────────────────────────────────── - - describe("flushQueues", () => { - it("stops, flushes, and reinitializes", async () => { - const { queue, scheduler } = createQueue(); - const result = await queue.flushQueues(); - expect(scheduler.stop).toHaveBeenCalled(); - expect(scheduler.flushJobs).toHaveBeenCalled(); - expect(result.success).toBe(true); - }); - - it("returns false when any step fails", async () => { - const { queue, scheduler } = createQueue(); - (scheduler.flushJobs as jest.Mock).mockResolvedValue(false); - const result = await queue.flushQueues(); - expect(result.success).toBe(false); - }); - }); - - // ── static create ──────────────────────────────────────────────────────── - - describe("static create", () => { - it("creates a SuperSimpleQueue instance with a Scheduler and calls init", async () => { - const logger = createMockLogger(); - const helper = createQueueHelper(); - const monitorsRepository = createMonitorsRepo(); - - const instance = await SuperSimpleQueue.create(logger as any, helper as any, monitorsRepository as any); - - expect(MockScheduler).toHaveBeenCalled(); - expect(instance).toBeInstanceOf(SuperSimpleQueue); - // init was called, which calls scheduler.start - expect(mockSchedulerInstance.start).toHaveBeenCalled(); - }); - }); -}); diff --git a/server/test/unit/services/superSimpleQueueHelper.test.ts b/server/test/unit/services/workerHelper.test.ts similarity index 100% rename from server/test/unit/services/superSimpleQueueHelper.test.ts rename to server/test/unit/services/workerHelper.test.ts From a7d9bb822e08e80145afd5dba26fe478f8f221ce Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 12:05:31 -0700 Subject: [PATCH 17/75] add tests, fix failing tests --- .../test/integration/jobsRepository.test.ts | 80 ++++++++++++++++++- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index f8aedc06f..d09174b84 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -52,6 +52,10 @@ const seedJob = (overrides: Partial = {}): Promise => const readRow = (id: string) => JobModel.findById(id).lean(); +// record*/renewLocks are fenced on the repo's private workerId, so a pre-seeded "held" +// lock must be stamped with the same id the repo will check against. +const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string }).workerId; + describe("MongoJobsRepository", () => { let repo: MongoJobsRepository; @@ -140,7 +144,7 @@ describe("MongoJobsRepository", () => { // ── recordSuccess (fixed-rate reschedule) ──────────────────────────────────── describe("recordSuccess", () => { it("reschedules fixed-rate from the scheduled tick and releases the lease", async () => { - await seedJob({ lockedBy: "me", lockedUntil: NOW + LOCK_MS, runCount: 4 }); + await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, runCount: 4 }); const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_500); @@ -155,7 +159,7 @@ describe("MongoJobsRepository", () => { }); it("skips missed ticks when the worker has fallen behind", async () => { - await seedJob({ lockedBy: "me", lockedUntil: NOW + LOCK_MS }); + await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS }); // scheduled NOW, interval 60s, but completion is 200s late → NOW+60s is already past const now = NOW + 200_000; @@ -169,12 +173,80 @@ describe("MongoJobsRepository", () => { it("returns false for an unknown id", async () => { expect(await repo.recordSuccess("missing", NOW, 60_000, NOW)).toBe(false); }); + + it("does not release a lock owned by another worker (fenced no-op)", async () => { + // A worker whose lease expired and was reclaimed must not clobber the new owner's lock. + await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7 }); + + const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000); + + expect(ok).toBe(false); + const row = await readRow("check:mon-1"); + expect(row?.lockedBy).toBe("other-worker"); // lock untouched + expect(row?.lockedUntil).toBe(NOW + LOCK_MS); + expect(row?.runCount).toBe(7); // not double-counted + }); + }); + + // ── renewLocks (lease renewal) ─────────────────────────────────────────────── + describe("renewLocks", () => { + it("extends the lease on a lock this worker holds", async () => { + await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS }); + + const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000); + + expect(renewed).toBe(1); + expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS); + }); + + it("does not renew a lock held by another worker (fenced)", async () => { + await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS }); + + const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000); + + expect(renewed).toBe(0); + expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + LOCK_MS); // unchanged + }); + + it("renews only the locks this worker holds out of a mixed batch", async () => { + await seedJob({ _id: "check:mine", refId: "mine", lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS }); + await seedJob({ _id: "check:theirs", refId: "theirs", lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS }); + + const renewed = await repo.renewLocks(["check:mine", "check:theirs"], NOW + 1_000); + + expect(renewed).toBe(1); + expect((await readRow("check:mine"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS); + expect((await readRow("check:theirs"))?.lockedUntil).toBe(NOW + LOCK_MS); + }); + + it("returns 0 for an empty id list without touching the db", async () => { + expect(await repo.renewLocks([], NOW)).toBe(0); + }); + + // Regression: a renewed lock keeps a slow-but-alive job from being reclaimed once its + // original lease would have expired — preventing the duplicate-execution path. + it("keeps a job from being reclaimed after the original lease window passes", async () => { + await seedJob(); + await repo.claimDue("check", NOW); // this worker now holds the lease until NOW + LOCK_MS + + // just before expiry, the still-running job renews its own lock + const renewAt = NOW + LOCK_MS - 1_000; + expect(await repo.renewLocks(["check:mon-1"], renewAt)).toBe(1); + + // a second worker scans PAST the original lease end — without renewal it would reclaim + const other = new MongoJobsRepository(); + expect(await other.claimDue("check", NOW + LOCK_MS + 1)).toBeNull(); + + const row = await readRow("check:mon-1"); + expect(row?.lockedBy).toBe(ownedBy(repo)); // still held by the original worker + expect(row?.lockedUntil).toBe(renewAt + LOCK_MS); + }); }); // ── recordFailure (backoff) ────────────────────────────────────────────────── describe("recordFailure", () => { it("backs off, records the reason, bumps failCount, releases the lease", async () => { - await seedJob({ lockedBy: "me", lockedUntil: NOW + LOCK_MS, failCount: 2 }); + await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, failCount: 2 }); const ok = await repo.recordFailure("check:mon-1", new Error("connection refused"), NOW); @@ -188,7 +260,7 @@ describe("MongoJobsRepository", () => { }); it("stringifies a non-Error reason", async () => { - await seedJob(); + await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS }); await repo.recordFailure("check:mon-1", "weird non-error", NOW); const row = await readRow("check:mon-1"); expect(row?.lastFailReason).toBe("weird non-error"); From 7ce6068757c264fd9d65792f0e15a1c0752975f7 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 12:12:14 -0700 Subject: [PATCH 18/75] fix migration --- .../src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts index 8f8de7308..3ab52f581 100644 --- a/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts +++ b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts @@ -3,6 +3,8 @@ import { logger } from "@/utils/logger.js"; /** * Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed. + * + * Stamp the upgrade moment, not 0. If set to 0, all checks will be replayed */ export async function backfillMonitorLastEvaluatedAt(): Promise { const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt"; @@ -10,7 +12,7 @@ export async function backfillMonitorLastEvaluatedAt(): Promise { try { logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" }); - const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: 0 } }); + const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: Date.now() } }); logger.info({ service: SERVICE_NAME, From abf6dc29080d1c0d9e69a06ecfaa89d8b7ee5426 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 12:21:36 -0700 Subject: [PATCH 19/75] single worker ID --- server/src/config/services.ts | 10 +++++++--- server/src/domain/jobs/job.repository.mongo.ts | 4 +--- server/src/worker/worker.db-queue.ts | 12 ++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 2afd263c9..092471b8c 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -1,4 +1,6 @@ import { Mongoose } from "mongoose"; +import { hostname } from "node:os"; +import { randomUUID } from "node:crypto"; import MongoDB from "../db/db.mongo.js"; import { IDb } from "@/db/db.interface.js"; import { EnvConfig, ISettingsService, SettingsService } from "@/domain/app-settings/app-settings.service.js"; @@ -98,7 +100,6 @@ import MongoTeamsRepository from "@/domain/teams/team.repository.model.js"; import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; import MongoUsersRepository from "@/domain/users/user.repository.mongo.js"; import { ILogger } from "@/utils/logger.js"; -import { type QueueMode } from "@/domain/app-settings/app-settings.type.js"; import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; @@ -177,7 +178,9 @@ export const initializeServices = async ({ const incidentsRepository = new MongoIncidentsRepository(); const teamsRepository = new MongoTeamsRepository(); const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); - const jobsRepository = new MongoJobsRepository(); + // One identity per process + const workerId = `${hostname()}:${process.pid}:${randomUUID()}`; + const jobsRepository = new MongoJobsRepository(workerId); const queueWorkersRepository = new MongoQueueWorkersRepository(); // Inject settings repository into settings service (now that DB is connected) @@ -305,7 +308,8 @@ export const initializeServices = async ({ workerHelper, queueWorkersRepository, envSettings.queueMode, - envSettings.queuePrimaryProcesses + envSettings.queuePrimaryProcesses, + workerId ); // Business services diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index 7b409c911..f7609ba01 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -1,11 +1,9 @@ import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js"; import JobModel, { JobDocument } from "@/domain/jobs/job.model.js"; import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js"; -import { hostname } from "node:os"; -import { randomUUID } from "node:crypto"; class MongoJobsRepository implements IJobsRepository { - private readonly workerId = `${hostname()}:${process.pid}:${randomUUID()}`; + constructor(private readonly workerId: string) {} private toEntity(doc: JobDocument): Job { return { diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 3d106445b..d8c997453 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -1,8 +1,6 @@ import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js"; import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js"; -import { randomUUID } from "node:crypto"; -import { hostname } from "node:os"; import { ILogger } from "@/utils/logger.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; @@ -32,7 +30,6 @@ const CONCURRENCY: Record = { export class DBQueueWorker implements IWorker { static SERVICE_NAME = SERVICE_NAME; - private readonly workerId = `${hostname()}:${process.pid}:${randomUUID().slice(0, 8)}`; private stopped = false; private timers = new Map(); private inFlight: Record = { @@ -56,7 +53,8 @@ export class DBQueueWorker implements IWorker { private helper: IWorkerHelper, private queueWorkersRepository: IQueueWorkersRepository, private queueMode: QueueMode, - private queuePrimaryProcesses: boolean + private queuePrimaryProcesses: boolean, + private readonly workerId: string // same id MongoJobsRepository stamps into lockedBy ) {} get serviceName() { @@ -76,7 +74,8 @@ export class DBQueueWorker implements IWorker { helper: IWorkerHelper, queueWorkersRepository: IQueueWorkersRepository, queueMode: QueueMode, - queuePrimaryProcesses: boolean + queuePrimaryProcesses: boolean, + workerId: string ): Promise { const instance = new DBQueueWorker( logger, @@ -91,7 +90,8 @@ export class DBQueueWorker implements IWorker { helper, queueWorkersRepository, queueMode, - queuePrimaryProcesses + queuePrimaryProcesses, + workerId ); await instance.init(); return instance; From 306f2a60a286dd9c60556eac9b02acf912566311 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 15 Jun 2026 12:22:55 -0700 Subject: [PATCH 20/75] update tests --- .../backfillLastEvaluatedAt.test.ts | 50 +++++++++++++++++++ .../test/integration/jobsRepository.test.ts | 12 +++-- 2 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 server/test/integration/backfillLastEvaluatedAt.test.ts diff --git a/server/test/integration/backfillLastEvaluatedAt.test.ts b/server/test/integration/backfillLastEvaluatedAt.test.ts new file mode 100644 index 000000000..53dc6c544 --- /dev/null +++ b/server/test/integration/backfillLastEvaluatedAt.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals"; +import mongoose from "mongoose"; +import { MongoMemoryServer } from "mongodb-memory-server"; +import { MonitorModel } from "../../src/domain/monitors/monitor.model.ts"; +import { backfillMonitorLastEvaluatedAt } from "../../src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts"; + +// ── Real-Mongo harness ───────────────────────────────────────────────────────── +// Legacy monitors are inserted through the raw driver, NOT MonitorModel.create — the +// schema default (lastEvaluatedAt: 0) would otherwise stamp the field and the migration's +// { $exists: false } filter would skip it, defeating the test. + +let mongod: MongoMemoryServer; + +beforeAll(async () => { + mongod = await MongoMemoryServer.create(); + await mongoose.connect(mongod.getUri()); +}, 120_000); + +afterAll(async () => { + await mongoose.disconnect(); + await mongod.stop(); +}); + +beforeEach(async () => { + await MonitorModel.collection.deleteMany({}); +}); + +describe("0008_backfillMonitorLastEvaluatedAt", () => { + it("stamps the upgrade moment (not 0) on monitors missing lastEvaluatedAt", async () => { + await MonitorModel.collection.insertOne({ name: "legacy" }); // pre-upgrade doc, no lastEvaluatedAt + + const before = Date.now(); + await backfillMonitorLastEvaluatedAt(); + + const doc = await MonitorModel.collection.findOne({ name: "legacy" }); + // Must be a recent timestamp — backfilling 0 would replay the whole check history on the first evaluate. + expect(doc?.lastEvaluatedAt).not.toBe(0); + expect(doc?.lastEvaluatedAt).toBeGreaterThanOrEqual(before); + expect(doc?.lastEvaluatedAt).toBeLessThanOrEqual(Date.now()); + }); + + it("leaves monitors that already have lastEvaluatedAt untouched (idempotent)", async () => { + await MonitorModel.collection.insertOne({ name: "current", lastEvaluatedAt: 12345 }); + + await backfillMonitorLastEvaluatedAt(); + + const doc = await MonitorModel.collection.findOne({ name: "current" }); + expect(doc?.lastEvaluatedAt).toBe(12345); + }); +}); diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index d09174b84..431c185d4 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -56,11 +56,13 @@ const readRow = (id: string) => JobModel.findById(id).lean(); // lock must be stamped with the same id the repo will check against. const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string }).workerId; +const WORKER_ID = "worker-under-test"; + describe("MongoJobsRepository", () => { let repo: MongoJobsRepository; beforeEach(() => { - repo = new MongoJobsRepository(); + repo = new MongoJobsRepository(WORKER_ID); }); // ── claimDue ─────────────────────────────────────────────────────────────── @@ -119,7 +121,7 @@ describe("MongoJobsRepository", () => { await seedJob(); // Distinct repo instances = distinct workerIds = realistic competing workers. - const workers = Array.from({ length: 20 }, () => new MongoJobsRepository()); + const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`)); const results = await Promise.all(workers.map((w) => w.claimDue("check", NOW))); const winners = results.filter((r): r is Job => r !== null); @@ -131,7 +133,7 @@ describe("MongoJobsRepository", () => { await seedJob({ _id: "check:a", refId: "a" }); await seedJob({ _id: "check:b", refId: "b" }); - const [w1, w2] = [new MongoJobsRepository(), new MongoJobsRepository()]; + const [w1, w2] = [new MongoJobsRepository("worker-1"), new MongoJobsRepository("worker-2")]; const [r1, r2] = await Promise.all([w1.claimDue("check", NOW), w2.claimDue("check", NOW)]); expect(r1).not.toBeNull(); @@ -234,7 +236,7 @@ describe("MongoJobsRepository", () => { expect(await repo.renewLocks(["check:mon-1"], renewAt)).toBe(1); // a second worker scans PAST the original lease end — without renewal it would reclaim - const other = new MongoJobsRepository(); + const other = new MongoJobsRepository("competing-worker"); expect(await other.claimDue("check", NOW + LOCK_MS + 1)).toBeNull(); const row = await readRow("check:mon-1"); @@ -302,7 +304,7 @@ describe("MongoJobsRepository", () => { }); it("concurrent upserts for one monitor produce exactly one row", async () => { - const workers = Array.from({ length: 20 }, () => new MongoJobsRepository()); + const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`)); await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW))); const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" }); From 2d68e967822a7a961f79d1346fe90ab3a2466334 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 16 Jun 2026 10:50:29 -0700 Subject: [PATCH 21/75] drop jobs in migration --- .../0008_backfillMonitorLastEvaluatedAt.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts index 3ab52f581..deb7cb02e 100644 --- a/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts +++ b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts @@ -1,6 +1,9 @@ +import mongoose from "mongoose"; import { MonitorModel } from "../../domain/monitors/monitor.model.js"; import { logger } from "@/utils/logger.js"; +const JOBS_COLLECTION = "jobs"; + /** * Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed. * @@ -10,6 +13,18 @@ export async function backfillMonitorLastEvaluatedAt(): Promise { const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt"; try { + // Drop the legacy jobs collection first. The old super-simple-scheduler persisted jobs to the + // same "jobs" collection the new db-queue uses, so any leftover rows carry an incompatible schema. + // Migrations run before syncIndexes(), so the new Job indexes are rebuilt right after this drop. + const db = mongoose.connection.db; + if (db) { + const existing = await db.listCollections({ name: JOBS_COLLECTION }).toArray(); + if (existing.length > 0) { + await db.collection(JOBS_COLLECTION).drop(); + logger.info({ service: SERVICE_NAME, message: `Dropped legacy "${JOBS_COLLECTION}" collection` }); + } + } + logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" }); const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: Date.now() } }); From d1b6d08c73600fc0b9a399ee9d4f48ba42492845 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 16 Jun 2026 10:50:39 -0700 Subject: [PATCH 22/75] add tests --- .../integration/backfillLastEvaluatedAt.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/test/integration/backfillLastEvaluatedAt.test.ts b/server/test/integration/backfillLastEvaluatedAt.test.ts index 53dc6c544..ec0459b7d 100644 --- a/server/test/integration/backfillLastEvaluatedAt.test.ts +++ b/server/test/integration/backfillLastEvaluatedAt.test.ts @@ -47,4 +47,18 @@ describe("0008_backfillMonitorLastEvaluatedAt", () => { const doc = await MonitorModel.collection.findOne({ name: "current" }); expect(doc?.lastEvaluatedAt).toBe(12345); }); + + it("drops the legacy jobs collection left behind by the old scheduler", async () => { + const db = mongoose.connection.db!; + await db.collection("jobs").insertOne({ id: "legacy-job", template: "monitor-job" }); // old-scheduler row + + await backfillMonitorLastEvaluatedAt(); + + const exists = await db.listCollections({ name: "jobs" }).toArray(); + expect(exists).toHaveLength(0); + }); + + it("does not throw when there is no jobs collection to drop", async () => { + await expect(backfillMonitorLastEvaluatedAt()).resolves.toBeUndefined(); + }); }); From 8c5be9ef00cdde1c80cc6ee588a0a94a1367567c Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 16 Jun 2026 10:50:50 -0700 Subject: [PATCH 23/75] removed pulsecron and ioredis --- server/package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/package.json b/server/package.json index 8341bfb2c..b893dc29b 100755 --- a/server/package.json +++ b/server/package.json @@ -33,7 +33,6 @@ "dependencies": { "@grpc/grpc-js": "^1.14.3", "@grpc/proto-loader": "^0.8.0", - "@pulsecron/pulse": "1.6.8", "axios": "^1.15.2", "bcryptjs": "3.0.2", "cacheable-lookup": "7.0.0", @@ -48,7 +47,6 @@ "got": "14.4.7", "handlebars": "^4.7.8", "helmet": "^8.0.0", - "ioredis": "^5.4.2", "isomorphic-dompurify": "^2.26.0", "jmespath": "^0.16.0", "jsdom": "^26.1.0", From cb0dd878d6fd1fc125ab30a0a8065905f9e6964a Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 16 Jun 2026 10:59:46 -0700 Subject: [PATCH 24/75] update .env.example --- server/.env.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/.env.example b/server/.env.example index 92798b56b..5a44935b8 100644 --- a/server/.env.example +++ b/server/.env.example @@ -11,7 +11,8 @@ LOG_LEVEL=debug # Database # -------- DB_CONNECTION_STRING=mongodb://localhost:27017/uptime_db -QUEUE_TYPE=superSimpleQueue +QUEUE_MODE="primary" +QUEUE_PRIMARY_PROCESSES="true" # JWT Authentication # ------------------ From f7211711cebda498717e0de692e77420b38024cb Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 16 Jun 2026 10:59:56 -0700 Subject: [PATCH 25/75] add db queue worker tests --- .../test/unit/services/dbQueueWorker.test.ts | 533 ++++++++++++++++++ 1 file changed, 533 insertions(+) create mode 100644 server/test/unit/services/dbQueueWorker.test.ts diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts new file mode 100644 index 000000000..8e0724d1f --- /dev/null +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -0,0 +1,533 @@ +import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals"; +import { DBQueueWorker } from "../../../src/worker/worker.db-queue.ts"; +import { LOCK_MS } from "../../../src/domain/jobs/job.type.ts"; +import type { Job } from "../../../src/domain/jobs/job.type.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; +import type { QueueMode } from "../../../src/domain/app-settings/app-settings.type.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +// ── Notes ────────────────────────────────────────────────────────────────────── +// +// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches +// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and +// renews their locks, and registers itself in the worker heartbeat. The repository +// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we +// test only the orchestration around it, with the repository fully mocked. +// +// Fake timers are mandatory: init() starts self-re-arming setTimeout loops and a +// heartbeat setInterval, so without them the loops would leak across tests. The +// loop re-arms at POLL_MS (250ms) and the lock renews at LOCK_MS / 3 (20s). + +const POLL_MS = 250; +const LOCK_RENEW_MS = LOCK_MS / 3; + +const deferred = () => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +}; + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "m1", + teamId: "team", + type: "http", + interval: 60000, + status: "up", + isActive: true, + geoCheckEnabled: false, + geoCheckInterval: 300000, + lastEvaluatedAt: 0, + ...overrides, + }) as Monitor; + +const makeJob = (overrides?: Partial): Job => ({ + id: "check:m1", + type: "check", + refId: "m1", + isActive: true, + nextScheduledAt: 1000, + intervalMs: 60000, + lockedBy: null, + lockedUntil: null, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + ...overrides, +}); + +const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record }) => { + const jobsRepository = { + claimDueBatch: jest.fn().mockResolvedValue([]), + upsertJob: jest.fn().mockResolvedValue(true), + upsertEvaluate: jest.fn().mockResolvedValue(true), + recordSuccess: jest.fn().mockResolvedValue(true), + recordFailure: jest.fn().mockResolvedValue(true), + recordOneShot: jest.fn().mockResolvedValue(true), + renewLocks: jest.fn().mockResolvedValue(1), + deleteById: jest.fn().mockResolvedValue(true), + deleteByIdAndType: jest.fn().mockResolvedValue(true), + setActiveById: jest.fn().mockResolvedValue(true), + updateScheduleById: jest.fn().mockResolvedValue(true), + findAll: jest.fn().mockResolvedValue([]), + findPage: jest.fn().mockResolvedValue({ jobs: [], count: 0 }), + }; + const monitorsRepository = { + findByIds: jest.fn().mockResolvedValue([makeMonitor()]), + findAll: jest.fn().mockResolvedValue([]), + updateById: jest.fn().mockResolvedValue({}), + }; + const checksRepository = { + findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([]), + }; + const checkService = { + toStatusResponse: jest.fn().mockReturnValue({ status: "up" }), + toLastEvaluatedAt: jest.fn().mockReturnValue(12345), + }; + const checkProducer = { + produce: jest.fn().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }), + }; + const checkEvaluator = { + evaluate: jest.fn().mockResolvedValue({ monitor: makeMonitor(), decision: {} }), + }; + const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) }; + const dispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) }; + const helper = { + getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + }; + const queueWorkersRepository = { + upsert: jest.fn().mockResolvedValue(undefined), + deleteById: jest.fn().mockResolvedValue(undefined), + findRecent: jest.fn().mockResolvedValue([]), + }; + const logger = createMockLogger(); + + const mocks = { + jobsRepository, + monitorsRepository, + checksRepository, + checkService, + checkProducer, + checkEvaluator, + geoCheckPipeline, + dispatcher, + helper, + queueWorkersRepository, + logger, + ...overrides?.mocks, + }; + + const worker = new DBQueueWorker( + mocks.logger as any, + mocks.jobsRepository as any, + mocks.monitorsRepository as any, + mocks.checksRepository as any, + mocks.checkService as any, + mocks.checkProducer as any, + mocks.checkEvaluator as any, + mocks.geoCheckPipeline as any, + mocks.dispatcher as any, + mocks.helper as any, + mocks.queueWorkersRepository as any, + overrides?.queueMode ?? "worker", + overrides?.queuePrimaryProcesses ?? true, + "worker-1" + ); + return { worker, mocks }; +}; + +describe("DBQueueWorker", () => { + let active: DBQueueWorker | null = null; + + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(async () => { + if (active) await active.shutdown(); + active = null; + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + const start = async (opts?: Parameters[0]) => { + const ctx = createWorker(opts); + active = ctx.worker; + await ctx.worker.init(); + await jest.advanceTimersByTimeAsync(1); // flush the immediate tick + any job it launched + return ctx; + }; + + // ── init() wiring ───────────────────────────────────────────────────────── + + describe("init", () => { + it("registers the worker in the heartbeat registry on startup", async () => { + const { mocks } = await start({ queueMode: "worker" }); + expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker"); + }); + + it("primary mode reconciles: seeds a check row per monitor plus the two cleanup rows", async () => { + const { mocks } = await start({ + queueMode: "primary", + queuePrimaryProcesses: false, // isolate reconcile from the polling loops + mocks: { + monitorsRepository: { findByIds: jest.fn(), findAll: jest.fn().mockResolvedValue([makeMonitor()]), updateById: jest.fn() }, + }, + }); + + const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type); + expect(seededTypes).toContain("check"); + expect(seededTypes).toContain("cleanup-orphaned"); + expect(seededTypes).toContain("cleanup-retention"); + }); + + it("primary mode also seeds a geo-check row for geo-enabled monitors", async () => { + const geoMonitor = makeMonitor({ type: "http", geoCheckEnabled: true }); + const { mocks } = await start({ + queueMode: "primary", + queuePrimaryProcesses: false, + mocks: { + monitorsRepository: { findByIds: jest.fn(), findAll: jest.fn().mockResolvedValue([geoMonitor]), updateById: jest.fn() }, + }, + }); + + const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type); + expect(seededTypes).toContain("geo-check"); + }); + + it("worker mode does not reconcile (never seeds jobs)", async () => { + const { mocks } = await start({ queueMode: "worker" }); + expect(mocks.jobsRepository.upsertJob).not.toHaveBeenCalled(); + }); + + it("starts a polling loop per job type, each claiming up to its concurrency cap", async () => { + const { mocks } = await start({ queueMode: "worker" }); + + const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]); + expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"])); + expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number)); + expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number)); + expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number)); + }); + + it("does not start polling loops when primary and queuePrimaryProcesses is false", async () => { + const { mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false }); + expect(mocks.jobsRepository.claimDueBatch).not.toHaveBeenCalled(); + }); + + it("create() constructs the worker and runs init", async () => { + const { mocks } = createWorker(); // reuse the mock set; the un-init'd instance is discarded + const worker = await DBQueueWorker.create( + mocks.logger as any, + mocks.jobsRepository as any, + mocks.monitorsRepository as any, + mocks.checksRepository as any, + mocks.checkService as any, + mocks.checkProducer as any, + mocks.checkEvaluator as any, + mocks.geoCheckPipeline as any, + mocks.dispatcher as any, + mocks.helper as any, + mocks.queueWorkersRepository as any, + "worker", + true, + "worker-1" + ); + active = worker; + await jest.advanceTimersByTimeAsync(1); + + expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker"); + }); + }); + + // ── runJob: the per-type execution paths ──────────────────────────────────── + + describe("runJob", () => { + // Hand the "check" loop exactly one job, then nothing, so the assertion target runs once. + const claimOnce = (job: Job) => { + let handed = false; + return jest.fn(async (type: string) => { + if (type === job.type && !handed) { + handed = true; + return [job]; + } + return []; + }); + }; + + it("runs a check job and reschedules it on success", async () => { + const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 }); + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); + + expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]); + expect(mocks.checkProducer.produce).toHaveBeenCalled(); + expect(mocks.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("m1", expect.any(Number)); + expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); + expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled(); + }); + + it("records a failure (and not a success) when the job handler throws", async () => { + const job = makeJob({ type: "check" }); + const failingProducer = { produce: jest.fn().mockRejectedValue(new Error("network down")) }; + const { mocks } = await start({ + mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: failingProducer }, + }); + + expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number)); + expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled(); + }); + + it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => { + const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null }); + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); + + expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number)); + expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled(); + }); + + it("evaluate job dispatches each unevaluated check and advances lastEvaluatedAt", async () => { + const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null }); + const checksRepository = { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }]) }; + const { mocks } = await start({ + mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository }, + }); + + expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled(); + expect(mocks.dispatcher.dispatch).toHaveBeenCalled(); + expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 }); + }); + + it("cleanup-orphaned job runs the helper's cleanup function", async () => { + const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 }); + const cleanupFn = jest.fn().mockResolvedValue(undefined); + const helper = { + getCleanupOrphanedJob: jest.fn().mockReturnValue(cleanupFn), + getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + }; + await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } }); + + expect(cleanupFn).toHaveBeenCalled(); + }); + + it("cleanup-retention job runs the helper's retention function", async () => { + const job = makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 }); + const retentionFn = jest.fn().mockResolvedValue(undefined); + const helper = { + getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + getCleanupRetentionJob: jest.fn().mockReturnValue(retentionFn), + }; + await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } }); + + expect(retentionFn).toHaveBeenCalled(); + }); + + it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => { + const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 }); + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); + + expect(mocks.geoCheckPipeline.run).toHaveBeenCalled(); + expect(mocks.dispatcher.dispatch).not.toHaveBeenCalled(); + expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); + }); + + it("renews the lock while a slow job is still running", async () => { + const job = makeJob({ type: "check" }); + const gate = deferred<{ status: unknown; check: unknown }>(); + const slowProducer = { produce: jest.fn().mockReturnValue(gate.promise) }; + const { mocks } = await start({ + mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer }, + }); + + // Job is parked on produce(); push past one renewal interval. + await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS); + expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledWith([job.id], expect.any(Number)); + + // Let it finish; success is still recorded. + gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); + await jest.advanceTimersByTimeAsync(1); + expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled(); + }); + + it("stops renewing once the lock is lost to another worker", async () => { + const job = makeJob({ type: "check" }); + const gate = deferred<{ status: unknown; check: unknown }>(); + const slowProducer = { produce: jest.fn().mockReturnValue(gate.promise) }; + const jobsRepository = { + ...createWorker().mocks.jobsRepository, + claimDueBatch: claimOnce(job), + renewLocks: jest.fn().mockResolvedValue(0), + }; + const { mocks } = await start({ mocks: { jobsRepository, checkProducer: slowProducer } }); + + await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS); + expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); // lease lost → renewal timer stops + expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Lost lock") })); + + // A second interval passes without another renewal attempt. + await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS); + expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); + + gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); + await jest.advanceTimersByTimeAsync(1); + }); + }); + + // ── monitor lifecycle delegation ──────────────────────────────────────────── + + describe("job lifecycle methods", () => { + it("addJob seeds a check row, plus a geo row for geo-enabled monitors", async () => { + const { worker, mocks } = createWorker(); + active = worker; + await worker.addJob("m1", makeMonitor({ geoCheckEnabled: true })); + + const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type); + expect(types).toContain("check"); + expect(types).toContain("geo-check"); + }); + + it("addJob seeds only a check row for non-geo monitors", async () => { + const { worker, mocks } = createWorker(); + active = worker; + await worker.addJob("m1", makeMonitor({ geoCheckEnabled: false })); + + const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type); + expect(types).toEqual(["check"]); + }); + + it("deleteJob removes every row for the monitor", async () => { + const { worker, mocks } = createWorker(); + active = worker; + await worker.deleteJob(makeMonitor()); + expect(mocks.jobsRepository.deleteById).toHaveBeenCalledWith("m1"); + }); + + it("pauseJob and resumeJob flip the active flag", async () => { + const { worker, mocks } = createWorker(); + active = worker; + await worker.pauseJob(makeMonitor()); + expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", false); + await worker.resumeJob(makeMonitor()); + expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", true); + }); + + it("updateJob reschedules the check row and drops the geo row when geo is disabled", async () => { + const { worker, mocks } = createWorker(); + active = worker; + await worker.updateJob(makeMonitor({ geoCheckEnabled: false, interval: 30000 })); + expect(mocks.jobsRepository.updateScheduleById).toHaveBeenCalledWith("m1", "check", 30000); + expect(mocks.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith("m1", "geo-check"); + }); + + it("updateJob upserts the geo row when geo is enabled", async () => { + const { worker, mocks } = createWorker(); + active = worker; + await worker.updateJob(makeMonitor({ geoCheckEnabled: true })); + const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type); + expect(types).toContain("geo-check"); + expect(mocks.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled(); + }); + }); + + // ── shutdown ──────────────────────────────────────────────────────────────── + + describe("shutdown", () => { + it("deregisters the worker and stops the polling loops", async () => { + const { worker, mocks } = createWorker({ queueMode: "worker" }); + await worker.init(); + await jest.advanceTimersByTimeAsync(1); + const claimsAfterInit = mocks.jobsRepository.claimDueBatch.mock.calls.length; + + await worker.shutdown(); + expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1"); + + // Loops must not re-arm after shutdown. + await jest.advanceTimersByTimeAsync(POLL_MS * 4); + expect(mocks.jobsRepository.claimDueBatch.mock.calls.length).toBe(claimsAfterInit); + // already shut down; keep afterEach from double-shutting-down + active = null; + }); + }); + + // ── flushQueues ─────────────────────────────────────────────────────────────── + + describe("flushQueues", () => { + it("restarts the worker and reports success", async () => { + const { worker, mocks } = createWorker({ queueMode: "worker" }); + active = worker; + await worker.init(); + await jest.advanceTimersByTimeAsync(1); + + const result = await worker.flushQueues(); + await jest.advanceTimersByTimeAsync(1); + + expect(result).toEqual({ success: true }); + expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1"); // shutdown leg + expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledTimes(2); // init + re-init + }); + }); + + // ── getMetrics ────────────────────────────────────────────────────────────── + + describe("getMetrics", () => { + it("aggregates run/fail counts, active leases, and failing jobs across all rows", async () => { + const now = Date.now(); + const rows = [ + makeJob({ id: "a", runCount: 5, failCount: 0, lockedBy: "w", lockedUntil: now + 10000 }), // active lease + makeJob({ id: "b", runCount: 2, failCount: 3, lastFailReason: "boom", lockedBy: null, lockedUntil: null }), // failing + makeJob({ id: "c", runCount: 1, failCount: 0, lockedBy: "w", lockedUntil: now - 10000 }), // expired lease, not active + ]; + const { worker, mocks } = createWorker(); + active = worker; + mocks.jobsRepository.findAll.mockResolvedValue(rows); + mocks.queueWorkersRepository.findRecent.mockResolvedValue([{ workerId: "w" }]); + + const metrics = await worker.getMetrics(); + + expect(metrics.jobs).toBe(3); + expect(metrics.totalRuns).toBe(8); + expect(metrics.totalFailures).toBe(3); + expect(metrics.activeJobs).toBe(1); + expect(metrics.failingJobs).toBe(1); + expect(metrics.jobsWithFailures).toHaveLength(1); + expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ failCount: 3, failReason: "boom" })); + expect(metrics.workers).toEqual([{ workerId: "w" }]); + }); + }); + + // ── getJobs ───────────────────────────────────────────────────────────────── + + describe("getJobs", () => { + it("maps rows to summaries and passes the total count through", async () => { + const { worker, mocks } = createWorker(); + active = worker; + mocks.jobsRepository.findPage.mockResolvedValue({ + jobs: [makeJob({ id: "check:m1", refId: "m1", type: "check", intervalMs: 60000 })], + count: 1, + }); + + const page = await worker.getJobs({ page: 0, rowsPerPage: 10 }); + + expect(page.count).toBe(1); + expect(page.jobs).toHaveLength(1); + expect(page.jobs[0]).toEqual(expect.objectContaining({ monitorId: "m1", monitorType: "check", monitorInterval: 60000, repeat: 60000 })); + }); + + it("falls back to the job id for monitorId when refId is null (global jobs)", async () => { + const { worker, mocks } = createWorker(); + active = worker; + mocks.jobsRepository.findPage.mockResolvedValue({ + jobs: [makeJob({ id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" })], + count: 1, + }); + + const page = await worker.getJobs({ page: 0, rowsPerPage: 10 }); + expect(page.jobs[0].monitorId).toBe("cleanup-orphaned"); + }); + }); +}); From 699883e0b13ede2a4efe75d7c12dd57e049f36e7 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 16 Jun 2026 11:34:58 -0700 Subject: [PATCH 26/75] add jitter and tests --- server/src/domain/jobs/job.repository.mongo.ts | 4 ++-- server/test/integration/jobsRepository.test.ts | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index f7609ba01..5069c90c4 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -88,9 +88,9 @@ class MongoJobsRepository implements IJobsRepository { let nextScheduledAt = scheduledRunAt + intervalMs; - // If the job fell behind, skipped missed runs + // If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd if (nextScheduledAt <= now) { - nextScheduledAt = now + intervalMs; + nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs); } const res = await JobModel.updateOne( diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index 431c185d4..389cdba1c 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -160,7 +160,7 @@ describe("MongoJobsRepository", () => { expect(row?.runCount).toBe(5); }); - it("skips missed ticks when the worker has fallen behind", async () => { + it("skips missed ticks when the worker has fallen behind, with jitter to avoid a herd", async () => { await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS }); // scheduled NOW, interval 60s, but completion is 200s late → NOW+60s is already past @@ -168,8 +168,10 @@ describe("MongoJobsRepository", () => { await repo.recordSuccess("check:mon-1", NOW, 60_000, now); const row = await readRow("check:mon-1"); - // no catch-up burst: jump to the next aligned tick from now, not NOW+60_000 - expect(row?.nextScheduledAt).toBe(now + 60_000); + // no catch-up burst: rebased to one interval from now, plus up to one interval of jitter + // so behind-jobs don't re-synchronize. Never earlier than now + interval (no over-checking). + expect(row?.nextScheduledAt).toBeGreaterThanOrEqual(now + 60_000); + expect(row?.nextScheduledAt).toBeLessThan(now + 120_000); }); it("returns false for an unknown id", async () => { From 7244b35091b9da4ea2706f4d305ecb9ee5025fad Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 17 Jun 2026 09:38:21 -0700 Subject: [PATCH 27/75] add upsertCleanup to schedule immediate cleanup jobs, cleanup orphaned jobs --- server/nodemon.json | 2 +- server/package-lock.json | 1635 ++++++++--------- server/src/config/services.ts | 1 + .../domain/jobs/job.repository.interface.ts | 6 + .../src/domain/jobs/job.repository.mongo.ts | 31 + server/src/worker/worker.db-queue.ts | 5 +- server/src/worker/worker.helper.ts | 14 + .../test/integration/jobsRepository.test.ts | 42 + .../test/unit/services/dbQueueWorker.test.ts | 7 +- .../test/unit/services/workerHelper.test.ts | 8 + 10 files changed, 871 insertions(+), 880 deletions(-) diff --git a/server/nodemon.json b/server/nodemon.json index 3faf346d3..37e41fc60 100755 --- a/server/nodemon.json +++ b/server/nodemon.json @@ -1,5 +1,5 @@ { "ignore": ["node_modules/*"], - "watch": ["src/**/*.ts", "src/**/*.js", "*.json"], + "watch": ["src/**/*.ts", "src/**/*.js", "*.json", ".env"], "ext": "ts,js,json" } diff --git a/server/package-lock.json b/server/package-lock.json index 7c69b6c1e..0fbe45e08 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -11,7 +11,6 @@ "dependencies": { "@grpc/grpc-js": "^1.14.3", "@grpc/proto-loader": "^0.8.0", - "@pulsecron/pulse": "1.6.8", "axios": "^1.15.2", "bcryptjs": "3.0.2", "cacheable-lookup": "7.0.0", @@ -26,7 +25,6 @@ "got": "14.4.7", "handlebars": "^4.7.8", "helmet": "^8.0.0", - "ioredis": "^5.4.2", "isomorphic-dompurify": "^2.26.0", "jmespath": "^0.16.0", "jsdom": "^26.1.0", @@ -838,12 +836,12 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.28.6.tgz", - "integrity": "sha512-JYgintcMjRiCvS8mMECzaEn+m3PfoQiyqukOMCCVQtoJGYJw8j/8LBJEiqkHLkfwCcs74E3pbAUFNg7d9VNJ+Q==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", + "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -852,9 +850,9 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.6.tgz", - "integrity": "sha512-2lfu57JtzctfIrcGMz992hyLlByuzgIk58+hhGCxjKZ3rWI82NnVLjXcaTqkI2NvlcvOskZaiZ5kjUALo3Lpxg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", "dev": true, "license": "MIT", "engines": { @@ -862,22 +860,22 @@ } }, "node_modules/@babel/core": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.6.tgz", - "integrity": "sha512-H3mcG6ZDLTlYfaSNi0iOKkigqMFvkTKlGUYlD8GW7nNOYRrevuA46iTypPyv+06V3fEmvvazfntkBU34L0azAw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/generator": "^7.28.6", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helpers": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/template": "^7.28.6", - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6", + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", @@ -904,14 +902,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.6.tgz", - "integrity": "sha512-lOoVRwADj8hjf7al89tvQ2a1lf53Z+7tiXMgpZJL3maQPDxh0DgLMN62B2MKUOFcoodBHLMbDM6WAbKgNy5Suw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -921,14 +919,14 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", - "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.28.6", - "@babel/helper-validator-option": "^7.27.1", + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" @@ -958,9 +956,9 @@ } }, "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", "dev": true, "license": "MIT", "engines": { @@ -968,29 +966,29 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", - "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", - "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.28.6" + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -1000,9 +998,9 @@ } }, "node_modules/@babel/helper-plugin-utils": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", - "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", "dev": true, "license": "MIT", "engines": { @@ -1010,9 +1008,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -1020,18 +1018,18 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", "dev": true, "license": "MIT", "engines": { @@ -1039,27 +1037,27 @@ } }, "node_modules/@babel/helpers": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", - "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.6.tgz", - "integrity": "sha512-TeR9zWR18BvbfPmGbLampPMW+uW1NZnJlRuuHso8i87QZNq2JRF9i6RgxRqtEq+wQGsS19NNTWr2duhnE49mfQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.6" + "@babel/types": "^7.29.7" }, "bin": { "parser": "bin/babel-parser.js" @@ -1166,13 +1164,13 @@ } }, "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.28.6.tgz", - "integrity": "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.29.7.tgz", + "integrity": "sha512-TSu8+mHCoEaaCDEZ0I3+6mvTBYR4PCxQwf2z9/r5Tbztv6NaLR3B9thGTTxX2WGuGHJqRiAbKPeGTJ5XWXVg6A==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -1292,13 +1290,13 @@ } }, "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.28.6.tgz", - "integrity": "sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.29.7.tgz", + "integrity": "sha512-ngr+82Sh0xMz25TPCZi+nC2iTzjfCdWS2ONXTp/PtSCHCgaCNBpdMqgvJ2ccdLlClVZ7sisIgB914j/JFe+RZA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" + "@babel/helper-plugin-utils": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -1317,33 +1315,33 @@ } }, "node_modules/@babel/template": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", - "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.6.tgz", - "integrity": "sha512-fgWX62k02qtjqdSNTAGxmKYY/7FSL9WAS1o2Hu5+I5m9T0yxZzr4cnrfXQ/MX0rIifthCSs6FKTlzYbJcPtMNg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/generator": "^7.28.6", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.28.6", - "@babel/template": "^7.28.6", - "@babel/types": "^7.28.6", + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", "debug": "^4.3.1" }, "engines": { @@ -1351,14 +1349,14 @@ } }, "node_modules/@babel/types": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.6.tgz", - "integrity": "sha512-0ZrskXVEHSWIqZM/sQZ4EV3jZJXRkio/WCxaqKZP1g//CEWEPSfeZFcms4XeKBCHU0ZKnIkdJeU/kF+eRp5lBg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -1370,6 +1368,13 @@ "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", "license": "Apache-2.0" }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, "node_modules/@colors/colors": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", @@ -1548,32 +1553,10 @@ "kuler": "^2.0.0" } }, - "node_modules/@emnapi/core": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", - "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.1.0", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.6.0.tgz", - "integrity": "sha512-obtUmAHTMjll499P+D9A3axeJFlhdjOWdKUNs/U6QIGT7V5RjcUW1xToAzjvmgTSQhDbYn/NwfTRoJcQ2rNBxA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", "dev": true, "license": "MIT", "optional": true, @@ -1582,9 +1565,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.11.tgz", - "integrity": "sha512-Xt1dOL13m8u0WE8iplx9Ibbm+hFAO0GsU2P34UNoDGvZYkY8ifSiy6Zuc1lYxfG7svWE2fzqCUmFp5HCn51gJg==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", "cpu": [ "ppc64" ], @@ -1599,9 +1582,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.11.tgz", - "integrity": "sha512-uoa7dU+Dt3HYsethkJ1k6Z9YdcHjTrSb5NUy66ZfZaSV8hEYGD5ZHbEMXnqLFlbBflLsl89Zke7CAdDJ4JI+Gg==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", "cpu": [ "arm" ], @@ -1616,9 +1599,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.11.tgz", - "integrity": "sha512-9slpyFBc4FPPz48+f6jyiXOx/Y4v34TUeDDXJpZqAWQn/08lKGeD8aDp9TMn9jDz2CiEuHwfhRmGBvpnd/PWIQ==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", "cpu": [ "arm64" ], @@ -1633,9 +1616,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.11.tgz", - "integrity": "sha512-Sgiab4xBjPU1QoPEIqS3Xx+R2lezu0LKIEcYe6pftr56PqPygbB7+szVnzoShbx64MUupqoE0KyRlN7gezbl8g==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", "cpu": [ "x64" ], @@ -1650,9 +1633,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.11.tgz", - "integrity": "sha512-VekY0PBCukppoQrycFxUqkCojnTQhdec0vevUL/EDOCnXd9LKWqD/bHwMPzigIJXPhC59Vd1WFIL57SKs2mg4w==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", "cpu": [ "arm64" ], @@ -1667,9 +1650,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.11.tgz", - "integrity": "sha512-+hfp3yfBalNEpTGp9loYgbknjR695HkqtY3d3/JjSRUyPg/xd6q+mQqIb5qdywnDxRZykIHs3axEqU6l1+oWEQ==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", "cpu": [ "x64" ], @@ -1684,9 +1667,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.11.tgz", - "integrity": "sha512-CmKjrnayyTJF2eVuO//uSjl/K3KsMIeYeyN7FyDBjsR3lnSJHaXlVoAK8DZa7lXWChbuOk7NjAc7ygAwrnPBhA==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", "cpu": [ "arm64" ], @@ -1701,9 +1684,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.11.tgz", - "integrity": "sha512-Dyq+5oscTJvMaYPvW3x3FLpi2+gSZTCE/1ffdwuM6G1ARang/mb3jvjxs0mw6n3Lsw84ocfo9CrNMqc5lTfGOw==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", "cpu": [ "x64" ], @@ -1718,9 +1701,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.11.tgz", - "integrity": "sha512-TBMv6B4kCfrGJ8cUPo7vd6NECZH/8hPpBHHlYI3qzoYFvWu2AdTvZNuU/7hsbKWqu/COU7NIK12dHAAqBLLXgw==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", "cpu": [ "arm" ], @@ -1735,9 +1718,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.11.tgz", - "integrity": "sha512-Qr8AzcplUhGvdyUF08A1kHU3Vr2O88xxP0Tm8GcdVOUm25XYcMPp2YqSVHbLuXzYQMf9Bh/iKx7YPqECs6ffLA==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", "cpu": [ "arm64" ], @@ -1752,9 +1735,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.11.tgz", - "integrity": "sha512-TmnJg8BMGPehs5JKrCLqyWTVAvielc615jbkOirATQvWWB1NMXY77oLMzsUjRLa0+ngecEmDGqt5jiDC6bfvOw==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", "cpu": [ "ia32" ], @@ -1769,9 +1752,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.11.tgz", - "integrity": "sha512-DIGXL2+gvDaXlaq8xruNXUJdT5tF+SBbJQKbWy/0J7OhU8gOHOzKmGIlfTTl6nHaCOoipxQbuJi7O++ldrxgMw==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", "cpu": [ "loong64" ], @@ -1786,9 +1769,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.11.tgz", - "integrity": "sha512-Osx1nALUJu4pU43o9OyjSCXokFkFbyzjXb6VhGIJZQ5JZi8ylCQ9/LFagolPsHtgw6himDSyb5ETSfmp4rpiKQ==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", "cpu": [ "mips64el" ], @@ -1803,9 +1786,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.11.tgz", - "integrity": "sha512-nbLFgsQQEsBa8XSgSTSlrnBSrpoWh7ioFDUmwo158gIm5NNP+17IYmNWzaIzWmgCxq56vfr34xGkOcZ7jX6CPw==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", "cpu": [ "ppc64" ], @@ -1820,9 +1803,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.11.tgz", - "integrity": "sha512-HfyAmqZi9uBAbgKYP1yGuI7tSREXwIb438q0nqvlpxAOs3XnZ8RsisRfmVsgV486NdjD7Mw2UrFSw51lzUk1ww==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", "cpu": [ "riscv64" ], @@ -1837,9 +1820,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.11.tgz", - "integrity": "sha512-HjLqVgSSYnVXRisyfmzsH6mXqyvj0SA7pG5g+9W7ESgwA70AXYNpfKBqh1KbTxmQVaYxpzA/SvlB9oclGPbApw==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", "cpu": [ "s390x" ], @@ -1854,9 +1837,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.11.tgz", - "integrity": "sha512-HSFAT4+WYjIhrHxKBwGmOOSpphjYkcswF449j6EjsjbinTZbp8PJtjsVK1XFJStdzXdy/jaddAep2FGY+wyFAQ==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", "cpu": [ "x64" ], @@ -1871,9 +1854,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.11.tgz", - "integrity": "sha512-hr9Oxj1Fa4r04dNpWr3P8QKVVsjQhqrMSUzZzf+LZcYjZNqhA3IAfPQdEh1FLVUJSiu6sgAwp3OmwBfbFgG2Xg==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", "cpu": [ "arm64" ], @@ -1888,9 +1871,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.11.tgz", - "integrity": "sha512-u7tKA+qbzBydyj0vgpu+5h5AeudxOAGncb8N6C9Kh1N4n7wU1Xw1JDApsRjpShRpXRQlJLb9wY28ELpwdPcZ7A==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", "cpu": [ "x64" ], @@ -1905,9 +1888,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.11.tgz", - "integrity": "sha512-Qq6YHhayieor3DxFOoYM1q0q1uMFYb7cSpLD2qzDSvK1NAvqFi8Xgivv0cFC6J+hWVw2teCYltyy9/m/14ryHg==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", "cpu": [ "arm64" ], @@ -1922,9 +1905,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.11.tgz", - "integrity": "sha512-CN+7c++kkbrckTOz5hrehxWN7uIhFFlmS/hqziSFVWpAzpWrQoAG4chH+nN3Be+Kzv/uuo7zhX716x3Sn2Jduw==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", "cpu": [ "x64" ], @@ -1939,9 +1922,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.11.tgz", - "integrity": "sha512-rOREuNIQgaiR+9QuNkbkxubbp8MSO9rONmwP5nKncnWJ9v5jQ4JxFnLu4zDSRPf3x4u+2VN4pM4RdyIzDty/wQ==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", "cpu": [ "arm64" ], @@ -1956,9 +1939,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.11.tgz", - "integrity": "sha512-nq2xdYaWxyg9DcIyXkZhcYulC6pQ2FuCgem3LI92IwMgIZ69KHeY8T4Y88pcwoLIjbed8n36CyKoYRDygNSGhA==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", "cpu": [ "x64" ], @@ -1973,9 +1956,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.11.tgz", - "integrity": "sha512-3XxECOWJq1qMZ3MN8srCJ/QfoLpL+VaxD/WfNRm1O3B4+AZ/BnLVgFbUV3eiRYDMXetciH16dwPbbHqwe1uU0Q==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", "cpu": [ "arm64" ], @@ -1990,9 +1973,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.11.tgz", - "integrity": "sha512-3ukss6gb9XZ8TlRyJlgLn17ecsK4NSQTmdIXRASVsiS2sQ6zPPZklNJT5GR5tE/MUarymmy8kCEf5xPCNCqVOA==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", "cpu": [ "ia32" ], @@ -2007,9 +1990,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.11.tgz", - "integrity": "sha512-D7Hpz6A2L4hzsRpPaCYkQnGOotdUpDzSGRIv9I+1ITdHROSFUWW95ZPZWQmGka1Fg7W3zFJowyn9WGwMJ0+KPA==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", "cpu": [ "x64" ], @@ -2181,9 +2164,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", - "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", "license": "Apache-2.0", "dependencies": { "@grpc/proto-loader": "^0.8.0", @@ -2624,12 +2607,6 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@ioredis/commands": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz", - "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==", - "license": "MIT" - }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -2765,17 +2742,17 @@ } }, "node_modules/@jest/console": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.2.0.tgz", - "integrity": "sha512-+O1ifRjkvYIkBqASKWgLxrpEhQAAE7hY77ALLUufSk5717KfOShg6IbqLmdsLMPdUiFvA2kTs0R7YZy+l0IzZQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.4.1.tgz", + "integrity": "sha512-v3bhyxUh9Hgmo5p6hAOXe14/R3ZxZDOsvHleh4B07z3m/x4/ngPUXEm9XwK4sF4u+f+P2ORb0Ge+MgpaqRMVDA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "@types/node": "*", "chalk": "^4.1.2", - "jest-message-util": "30.2.0", - "jest-util": "30.2.0", + "jest-message-util": "30.4.1", + "jest-util": "30.4.1", "slash": "^3.0.0" }, "engines": { @@ -2783,39 +2760,39 @@ } }, "node_modules/@jest/core": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.2.0.tgz", - "integrity": "sha512-03W6IhuhjqTlpzh/ojut/pDB2LPRygyWX8ExpgHtQA8H/3K7+1vKmcINx5UzeOX1se6YEsBsOHQ1CRzf3fOwTQ==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.4.2.tgz", + "integrity": "sha512-TZJA6cPJUFxoWhxaLo8t0VX/MZX2wPWr0uIDvLSHIvN4gu9h02vSzqI2kBADG1ExqQlC+cY09xKMSreivvrChQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.2.0", - "@jest/pattern": "30.0.1", - "@jest/reporters": "30.2.0", - "@jest/test-result": "30.2.0", - "@jest/transform": "30.2.0", - "@jest/types": "30.2.0", + "@jest/console": "30.4.1", + "@jest/pattern": "30.4.0", + "@jest/reporters": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", "ansi-escapes": "^4.3.2", "chalk": "^4.1.2", "ci-info": "^4.2.0", "exit-x": "^0.2.2", + "fast-json-stable-stringify": "^2.1.0", "graceful-fs": "^4.2.11", - "jest-changed-files": "30.2.0", - "jest-config": "30.2.0", - "jest-haste-map": "30.2.0", - "jest-message-util": "30.2.0", - "jest-regex-util": "30.0.1", - "jest-resolve": "30.2.0", - "jest-resolve-dependencies": "30.2.0", - "jest-runner": "30.2.0", - "jest-runtime": "30.2.0", - "jest-snapshot": "30.2.0", - "jest-util": "30.2.0", - "jest-validate": "30.2.0", - "jest-watcher": "30.2.0", - "micromatch": "^4.0.8", - "pretty-format": "30.2.0", + "jest-changed-files": "30.4.1", + "jest-config": "30.4.2", + "jest-haste-map": "30.4.1", + "jest-message-util": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-resolve": "30.4.1", + "jest-resolve-dependencies": "30.4.2", + "jest-runner": "30.4.2", + "jest-runtime": "30.4.2", + "jest-snapshot": "30.4.1", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", + "jest-watcher": "30.4.1", + "pretty-format": "30.4.1", "slash": "^3.0.0" }, "engines": { @@ -2831,9 +2808,9 @@ } }, "node_modules/@jest/diff-sequences": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", - "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.4.0.tgz", + "integrity": "sha512-zOpzlfUs45l6u7jm39qr87JCHUDsaeCtvL+kQe/Vn9jSnRB4/5IPXISm0h9I1vZW/o00Kn4UTJ2MOlhnUGwv3g==", "dev": true, "license": "MIT", "engines": { @@ -2841,39 +2818,39 @@ } }, "node_modules/@jest/environment": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.2.0.tgz", - "integrity": "sha512-/QPTL7OBJQ5ac09UDRa3EQes4gt1FTEG/8jZ/4v5IVzx+Cv7dLxlVIvfvSVRiiX2drWyXeBjkMSR8hvOWSog5g==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.4.1.tgz", + "integrity": "sha512-AK9yNRqgKxiabqMoe4oW+3/TSSeV8vkdC7BGaxZdU0AFXfOpofTLqdru2GXKZghP3sdgwE9XXpnVwfZ8JnFV4w==", "dev": true, "license": "MIT", "dependencies": { - "@jest/fake-timers": "30.2.0", - "@jest/types": "30.2.0", + "@jest/fake-timers": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", - "jest-mock": "30.2.0" + "jest-mock": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.2.0.tgz", - "integrity": "sha512-V9yxQK5erfzx99Sf+7LbhBwNWEZ9eZay8qQ9+JSC0TrMR1pMDHLMY+BnVPacWU6Jamrh252/IKo4F1Xn/zfiqA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.4.1.tgz", + "integrity": "sha512-ginrj6TMgh2GshLUGCjO94Ptx9HhdZA/I6A9iUfyeLKFtdAjnKzHDgzgP9HYQgbxM1lbXScQ2eUBz2lGeVDPWA==", "dev": true, "license": "MIT", "dependencies": { - "expect": "30.2.0", - "jest-snapshot": "30.2.0" + "expect": "30.4.1", + "jest-snapshot": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect-utils": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.2.0.tgz", - "integrity": "sha512-1JnRfhqpD8HGpOmQp180Fo9Zt69zNtC+9lR+kT7NVL05tNXIi+QC8Csz7lfidMoVLPD3FnOtcmp0CEFnxExGEA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.4.1.tgz", + "integrity": "sha512-ZBn5CglH8fBsQsvs4VWNzD4aWfUYks+IdOOQU3MEK71ol/BcVm+P+rtb1KpiFBpSWSCE27uOahyyf1vfqOVbcQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2884,18 +2861,18 @@ } }, "node_modules/@jest/fake-timers": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.2.0.tgz", - "integrity": "sha512-HI3tRLjRxAbBy0VO8dqqm7Hb2mIa8d5bg/NJkyQcOk7V118ObQML8RC5luTF/Zsg4474a+gDvhce7eTnP4GhYw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.4.1.tgz", + "integrity": "sha512-iW5umdmfPeWzehrVhugFQZqCchSCud5S1l2YT0O9ZhjRR0ExclANDZkiSBwzqtnlOn0J1JXvO+HZ6rkuyOVOgQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.2.0", - "@sinonjs/fake-timers": "^13.0.0", + "@jest/types": "30.4.1", + "@sinonjs/fake-timers": "^15.4.0", "@types/node": "*", - "jest-message-util": "30.2.0", - "jest-mock": "30.2.0", - "jest-util": "30.2.0" + "jest-message-util": "30.4.1", + "jest-mock": "30.4.1", + "jest-util": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -2912,62 +2889,62 @@ } }, "node_modules/@jest/globals": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.2.0.tgz", - "integrity": "sha512-b63wmnKPaK+6ZZfpYhz9K61oybvbI1aMcIs80++JI1O1rR1vaxHUCNqo3ITu6NU0d4V34yZFoHMn/uoKr/Rwfw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.4.1.tgz", + "integrity": "sha512-ZbuY4cmXC8DkxYjfvT2DbcHWL2T6vmsMhXCDcmTB2T0y0gaezBI77ufq5ZAIdcRkYZ7NEQEDg1xFeKbxUJ5v5Q==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.2.0", - "@jest/expect": "30.2.0", - "@jest/types": "30.2.0", - "jest-mock": "30.2.0" + "@jest/environment": "30.4.1", + "@jest/expect": "30.4.1", + "@jest/types": "30.4.1", + "jest-mock": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/pattern": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.0.1.tgz", - "integrity": "sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/@jest/pattern/-/pattern-30.4.0.tgz", + "integrity": "sha512-RAWn3+f9u8BsHijKJ71uHcFp6vmyEt6VvoWXkl6hKF3qVIuWNmudVjg12DlBPGup/frIl5UcUlH5HfEuvHpEXg==", "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", - "jest-regex-util": "30.0.1" + "jest-regex-util": "30.4.0" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/reporters": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.2.0.tgz", - "integrity": "sha512-DRyW6baWPqKMa9CzeiBjHwjd8XeAyco2Vt8XbcLFjiwCOEKOvy82GJ8QQnJE9ofsxCMPjH4MfH8fCWIHHDKpAQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.4.1.tgz", + "integrity": "sha512-/SnkPCzEQpUaBH81kjdEdDdo2WZl5hxw+BmLDGWjRkm8o7XlhjwsU36cqwe5PGBE5WYpBvDzRSdXx9rbGuJtNA==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "30.2.0", - "@jest/test-result": "30.2.0", - "@jest/transform": "30.2.0", - "@jest/types": "30.2.0", + "@jest/console": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", "@jridgewell/trace-mapping": "^0.3.25", "@types/node": "*", "chalk": "^4.1.2", "collect-v8-coverage": "^1.0.2", "exit-x": "^0.2.2", - "glob": "^10.3.10", + "glob": "^10.5.0", "graceful-fs": "^4.2.11", "istanbul-lib-coverage": "^3.0.0", "istanbul-lib-instrument": "^6.0.0", "istanbul-lib-report": "^3.0.0", "istanbul-lib-source-maps": "^5.0.0", "istanbul-reports": "^3.1.3", - "jest-message-util": "30.2.0", - "jest-util": "30.2.0", - "jest-worker": "30.2.0", + "jest-message-util": "30.4.1", + "jest-util": "30.4.1", + "jest-worker": "30.4.1", "slash": "^3.0.0", "string-length": "^4.0.2", "v8-to-istanbul": "^9.0.1" @@ -2984,17 +2961,10 @@ } } }, - "node_modules/@jest/reporters/node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true, - "license": "MIT" - }, "node_modules/@jest/schemas": { - "version": "30.0.5", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", - "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.4.1.tgz", + "integrity": "sha512-i6b4qw5qnP8c5FEeBJg/uZQ4ddrkN6Ca8qISJh0pr7a5hfn3h3v5x60BEbOC7OYAGZNMs1LfFLwnW2CuK8F57Q==", "dev": true, "license": "MIT", "dependencies": { @@ -3005,13 +2975,13 @@ } }, "node_modules/@jest/snapshot-utils": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.2.0.tgz", - "integrity": "sha512-0aVxM3RH6DaiLcjj/b0KrIBZhSX1373Xci4l3cW5xiUWPctZ59zQ7jj4rqcJQ/Z8JuN/4wX3FpJSa3RssVvCug==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.4.1.tgz", + "integrity": "sha512-ObY4ljvQ95mt6iwKtVLetR/4yXiAgl3H4nJxhztr0MTjrN97TwDYrnCp/kF60Ec9HdhkWTHSu+Hg05aXfngpOA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "chalk": "^4.1.2", "graceful-fs": "^4.2.11", "natural-compare": "^1.4.0" @@ -3036,14 +3006,14 @@ } }, "node_modules/@jest/test-result": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.2.0.tgz", - "integrity": "sha512-RF+Z+0CCHkARz5HT9mcQCBulb1wgCP3FBvl9VFokMX27acKphwyQsNuWH3c+ojd1LeWBLoTYoxF0zm6S/66mjg==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.4.1.tgz", + "integrity": "sha512-/ZG7pgEiOmmWkN9TplKbOu4id2N5lh7FHwRwlkgBVAzGdRH+OkkQ8wX/kIxg4zmd3ZQvAL1RwL2yWsvNYYECTw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.2.0", - "@jest/types": "30.2.0", + "@jest/console": "30.4.1", + "@jest/types": "30.4.1", "@types/istanbul-lib-coverage": "^2.0.6", "collect-v8-coverage": "^1.0.2" }, @@ -3052,15 +3022,15 @@ } }, "node_modules/@jest/test-sequencer": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.2.0.tgz", - "integrity": "sha512-wXKgU/lk8fKXMu/l5Hog1R61bL4q5GCdT6OJvdAFz1P+QrpoFuLU68eoKuVc4RbrTtNnTL5FByhWdLgOPSph+Q==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.4.1.tgz", + "integrity": "sha512-PeYE+4td5rKjoRPxztObrXU+H8hsjZfxKMXOcmrr34JerSyB/ROOxbbicz8B7A5j9R9VayDnVPvBmedqCsFCdw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.2.0", + "@jest/test-result": "30.4.1", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.2.0", + "jest-haste-map": "30.4.1", "slash": "^3.0.0" }, "engines": { @@ -3068,24 +3038,23 @@ } }, "node_modules/@jest/transform": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.2.0.tgz", - "integrity": "sha512-XsauDV82o5qXbhalKxD7p4TZYYdwcaEXC77PPD2HixEFF+6YGppjrAAQurTl2ECWcEomHBMMNS9AH3kcCFx8jA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.4.1.tgz", + "integrity": "sha512-Wz0LyktlTvRefoymh+n64hQ84KNXsRGcwdoZ8CSa0Ea+fgYcHZlnk+hDP7v2MS7il2bQ5uTEIxf4/NNfhMN4KQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.27.4", - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "@jridgewell/trace-mapping": "^0.3.25", "babel-plugin-istanbul": "^7.0.1", "chalk": "^4.1.2", "convert-source-map": "^2.0.0", "fast-json-stable-stringify": "^2.1.0", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.2.0", - "jest-regex-util": "30.0.1", - "jest-util": "30.2.0", - "micromatch": "^4.0.8", + "jest-haste-map": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-util": "30.4.1", "pirates": "^4.0.7", "slash": "^3.0.0", "write-file-atomic": "^5.0.1" @@ -3095,14 +3064,14 @@ } }, "node_modules/@jest/types": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.2.0.tgz", - "integrity": "sha512-H9xg1/sfVvyfU7o3zMfBEjQ1gcsdeTMgqHoYdN79tuLqfTtuu7WckRA1R5whDwOzxaZAeMKTYWqP+WCAi0CHsg==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-30.4.1.tgz", + "integrity": "sha512-f1x/vJXIfjOlEmejYpbkbgw1gOqpPECwMvMEtBqe47j7H2Hg8h8w3o3ikhSXq3MI15kg+oQ0exWO0uCtTNJLoQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/pattern": "30.0.1", - "@jest/schemas": "30.0.5", + "@jest/pattern": "30.4.0", + "@jest/schemas": "30.4.1", "@types/istanbul-lib-coverage": "^2.0.6", "@types/istanbul-reports": "^3.0.4", "@types/node": "*", @@ -3183,16 +3152,22 @@ } }, "node_modules/@napi-rs/wasm-runtime": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", - "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz", + "integrity": "sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "^1.4.3", - "@emnapi/runtime": "^1.4.3", - "@tybys/wasm-util": "^0.10.0" + "@tybys/wasm-util": "^0.10.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" } }, "node_modules/@nodable/entities": { @@ -3257,13 +3232,13 @@ } }, "node_modules/@pkgr/core": { - "version": "0.2.9", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", - "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.3.6.tgz", + "integrity": "sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==", "dev": true, "license": "MIT", "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + "node": "^14.18.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/pkgr" @@ -3308,12 +3283,6 @@ "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", "license": "BSD-3-Clause" }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", - "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", - "license": "BSD-3-Clause" - }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", @@ -3332,20 +3301,6 @@ "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", "license": "BSD-3-Clause" }, - "node_modules/@pulsecron/pulse": { - "version": "1.6.8", - "resolved": "https://registry.npmjs.org/@pulsecron/pulse/-/pulse-1.6.8.tgz", - "integrity": "sha512-fg9sT/pfpZTlUYr/ktDu6om6XEm2zmulcZ8TP4JZa+Dmol7/35mMKzDwT1H5a0AUcxiFIRfB4KOpkJ+BI2fgmg==", - "license": "MIT", - "dependencies": { - "cron-parser": "^4.9.0", - "date.js": "~0.3.3", - "debug": "~4.3.4", - "human-interval": "~2.0.1", - "moment-timezone": "^0.5.45", - "mongodb": "^6.5.0" - } - }, "node_modules/@scarf/scarf": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", @@ -3360,9 +3315,9 @@ "license": "MIT" }, "node_modules/@sinclair/typebox": { - "version": "0.34.47", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.47.tgz", - "integrity": "sha512-ZGIBQ+XDvO5JQku9wmwtabcVTHJsgSWAHYtVuM9pBNNR5E88v6Jcj/llpmsjivig5X8A8HHOb4/mbEKPS5EvAw==", + "version": "0.34.49", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", + "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", "dev": true, "license": "MIT" }, @@ -3389,9 +3344,9 @@ } }, "node_modules/@sinonjs/fake-timers": { - "version": "13.0.5", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", - "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", + "version": "15.4.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.4.0.tgz", + "integrity": "sha512-DsG+8/LscQIQg68J6Ef3dv10u6nVyetYn923s3/sus5eaGfTo1of5WMZSLf0UJc9KDuKPilPH0UDJCjvNbDNCA==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -4115,9 +4070,9 @@ "license": "MIT" }, "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", "dev": true, "license": "MIT", "optional": true, @@ -4975,9 +4930,9 @@ "license": "ISC" }, "node_modules/@unrs/resolver-binding-android-arm-eabi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.11.1.tgz", - "integrity": "sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm-eabi/-/resolver-binding-android-arm-eabi-1.12.2.tgz", + "integrity": "sha512-g5T90pqg1bo/7mytQx6F4iBNC0Wsh9cu+z9veDbFjc7HjpesJFWD7QMS0NGStXM075+7dJPPVvBbpZlnrdpi/w==", "cpu": [ "arm" ], @@ -4989,9 +4944,9 @@ ] }, "node_modules/@unrs/resolver-binding-android-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.11.1.tgz", - "integrity": "sha512-lCxkVtb4wp1v+EoN+HjIG9cIIzPkX5OtM03pQYkG+U5O/wL53LC4QbIeazgiKqluGeVEeBlZahHalCaBvU1a2g==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-android-arm64/-/resolver-binding-android-arm64-1.12.2.tgz", + "integrity": "sha512-YGCRZv/9GLhwmz6mYDeTsm/92BAyR28l6c2ReweVW5pWgfsitWLY8upvfRlGdoyD8HjeTHSYJWyZGD4KJA/nFQ==", "cpu": [ "arm64" ], @@ -5003,9 +4958,9 @@ ] }, "node_modules/@unrs/resolver-binding-darwin-arm64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.11.1.tgz", - "integrity": "sha512-gPVA1UjRu1Y/IsB/dQEsp2V1pm44Of6+LWvbLc9SDk1c2KhhDRDBUkQCYVWe6f26uJb3fOK8saWMgtX8IrMk3g==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-arm64/-/resolver-binding-darwin-arm64-1.12.2.tgz", + "integrity": "sha512-u9DiNT1auQMO20A9SyTuG3wUgQWB9Z7KjAg0uFuCDR1FsAY8A0CG2S6JpHS1xwm/w1G08bjXZDcyOCjv1WAm2w==", "cpu": [ "arm64" ], @@ -5017,9 +4972,9 @@ ] }, "node_modules/@unrs/resolver-binding-darwin-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.11.1.tgz", - "integrity": "sha512-cFzP7rWKd3lZaCsDze07QX1SC24lO8mPty9vdP+YVa3MGdVgPmFc59317b2ioXtgCMKGiCLxJ4HQs62oz6GfRQ==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-darwin-x64/-/resolver-binding-darwin-x64-1.12.2.tgz", + "integrity": "sha512-f7rPLi/T1HVKZu/u6t87lroib16n8vrSzcyxI7lg4BGO9UF26KhQL44sd9eOUgrTYhvRXtWOIZT5PejdPyJfUA==", "cpu": [ "x64" ], @@ -5031,9 +4986,9 @@ ] }, "node_modules/@unrs/resolver-binding-freebsd-x64": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.11.1.tgz", - "integrity": "sha512-fqtGgak3zX4DCB6PFpsH5+Kmt/8CIi4Bry4rb1ho6Av2QHTREM+47y282Uqiu3ZRF5IQioJQ5qWRV6jduA+iGw==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-freebsd-x64/-/resolver-binding-freebsd-x64-1.12.2.tgz", + "integrity": "sha512-BpcOjWCJub6nRZUS2zA20pmLvjtqAtGejETaIyRLiZiQf++cbrjltLA5NN/xaXfqeOBOSlMFbemIl5/S5tljmg==", "cpu": [ "x64" ], @@ -5045,9 +5000,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-arm-gnueabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.11.1.tgz", - "integrity": "sha512-u92mvlcYtp9MRKmP+ZvMmtPN34+/3lMHlyMj7wXJDeXxuM0Vgzz0+PPJNsro1m3IZPYChIkn944wW8TYgGKFHw==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-gnueabihf/-/resolver-binding-linux-arm-gnueabihf-1.12.2.tgz", + "integrity": "sha512-vZTDvdSISZjJx66OzJqtsOhzifbqRjbmI1Mnu49fQDwog5GtDI4QidRiEAYbZCRj9C8YZEW+3ZjqsyS9GR4k2A==", "cpu": [ "arm" ], @@ -5059,9 +5014,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-arm-musleabihf": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.11.1.tgz", - "integrity": "sha512-cINaoY2z7LVCrfHkIcmvj7osTOtm6VVT16b5oQdS4beibX2SYBwgYLmqhBjA1t51CarSaBuX5YNsWLjsqfW5Cw==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm-musleabihf/-/resolver-binding-linux-arm-musleabihf-1.12.2.tgz", + "integrity": "sha512-BiPI+IrIlwcW4nLLMM21+B1dFPzd55yAVgVGrdgDjNef+ch03GdxrcyaIz8X9SsQirh/kCQ7mviyWlMxdh2D7g==", "cpu": [ "arm" ], @@ -5073,9 +5028,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-arm64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz", - "integrity": "sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.12.2.tgz", + "integrity": "sha512-zJc0H99FEPoFfSrNpa91HYfxzfAJCr502oxNK1cfdC9hlaFI43RT+JFCann9JUgZmLzzntChHyn13Sgn9ljHNg==", "cpu": [ "arm64" ], @@ -5087,9 +5042,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-arm64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz", - "integrity": "sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.12.2.tgz", + "integrity": "sha512-KQ3Lki6l+Pz1k/eBipN41ES+YUK30beLGb9YqcB1O542cyLCNE6GaxrfcY3T6EezmGGk84wb5XyO9loTM9tkcA==", "cpu": [ "arm64" ], @@ -5100,10 +5055,38 @@ "linux" ] }, + "node_modules/@unrs/resolver-binding-linux-loong64-gnu": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-loong64-gnu/-/resolver-binding-linux-loong64-gnu-1.12.2.tgz", + "integrity": "sha512-3SJGEh1DborhG6pyxvhPzCT4bbSIVihsvgJc13P1bHG7KLdNDaF9T3gsTwFc7Jw/5Y5/iWOjkEx7Zy0NvCGX3Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@unrs/resolver-binding-linux-loong64-musl": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-loong64-musl/-/resolver-binding-linux-loong64-musl-1.12.2.tgz", + "integrity": "sha512-jiuG/Obbel7uw1PwHNFfrkiKhLAF6mnyZ6aWlOAVN9WqKm8v0OFGnciJIHu8+CMvXLQ8AD51LPzAoUfT21D5Ew==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@unrs/resolver-binding-linux-ppc64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz", - "integrity": "sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.12.2.tgz", + "integrity": "sha512-q7xRvVpmcfeL+LlZg8Pbbo6QaTZwDU5BaGZbwfhkEsXJn3Was8xYfE0RBH266xZt0rM6B7i8xAYIvjthuUIWHg==", "cpu": [ "ppc64" ], @@ -5115,9 +5098,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-riscv64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz", - "integrity": "sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.12.2.tgz", + "integrity": "sha512-0CVdx6lcnT3Q9inOH8tsMIOJ6ImndllMjqJHg8RLVdB7Vq4SfkEXl9mCSsVNuNA4MCYycRicCUxPCabVHJRr6A==", "cpu": [ "riscv64" ], @@ -5129,9 +5112,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-riscv64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz", - "integrity": "sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.12.2.tgz", + "integrity": "sha512-iOwlRo9vnp6R6ohHQS11n0NnfdXx/omhkocmIfaPRpQhKZ+3BDMkkdRVh53qjkFkpPddf+FETA28NwGN7l5l+w==", "cpu": [ "riscv64" ], @@ -5143,9 +5126,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-s390x-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz", - "integrity": "sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.12.2.tgz", + "integrity": "sha512-HYJtLfXq94q8iZNFT1lknx258wlkkWhZeUXJRqzKBBUJ00CvZ+N33zgbCqimLjsyw5Va6uUxhVa12mI+kaveEw==", "cpu": [ "s390x" ], @@ -5157,9 +5140,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-x64-gnu": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz", - "integrity": "sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.12.2.tgz", + "integrity": "sha512-mPsUhunKKDih5O96Y6enDQyHc1SqBPlY1E/SfMWDM3EdJ95Z9CArPeCVwCCqbP45ljvivdEk8Fxn+SIb1rDAJQ==", "cpu": [ "x64" ], @@ -5171,9 +5154,9 @@ ] }, "node_modules/@unrs/resolver-binding-linux-x64-musl": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz", - "integrity": "sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.12.2.tgz", + "integrity": "sha512-azrt6+5ydLd8Vt210AAFis/lZevSfPw93EJRIJG+xPu4WCJ8K0kppCTpMyLPcKT7H15M4Jnt2tMp5bOvCkRC6A==", "cpu": [ "x64" ], @@ -5184,10 +5167,24 @@ "linux" ] }, + "node_modules/@unrs/resolver-binding-openharmony-arm64": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-openharmony-arm64/-/resolver-binding-openharmony-arm64-1.12.2.tgz", + "integrity": "sha512-YZ9hP4O0X9PQb8eO980qmLNGH4zT3I9+SZTdt0Pr0YyuGQhYKoOZkV02VzrzyOZJ5xIJ3UFIenKkUkGg8GjgWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, "node_modules/@unrs/resolver-binding-wasm32-wasi": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz", - "integrity": "sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.12.2.tgz", + "integrity": "sha512-tYFDIkMxSflfEc/h92ZWNsZlHSwgimbNHSO3PL2JWQHfCuC2q316jMyYU9TIWZsFK2bQwyK5VAdYgn8ygPj69A==", "cpu": [ "wasm32" ], @@ -5195,16 +5192,18 @@ "license": "MIT", "optional": true, "dependencies": { - "@napi-rs/wasm-runtime": "^0.2.11" + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" }, "engines": { "node": ">=14.0.0" } }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", - "integrity": "sha512-nRcz5Il4ln0kMhfL8S3hLkxI85BXs3o8EYoattsJNdsX4YUU89iOkVn7g0VHSRxFuVMdM4Q1jEpIId1Ihim/Uw==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz", + "integrity": "sha512-qzNyg3xL0VPQmCaUh+N5jSitce6k+uCBfMDesWRnlULOZaqUkaJ0ybdT+UqlAWJoQjuqfIU/0Ptx9bteN4D82g==", "cpu": [ "arm64" ], @@ -5216,9 +5215,9 @@ ] }, "node_modules/@unrs/resolver-binding-win32-ia32-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.11.1.tgz", - "integrity": "sha512-DCEI6t5i1NmAZp6pFonpD5m7i6aFrpofcp4LA2i8IIq60Jyo28hamKBxNrZcyOwVOZkgsRp9O2sXWBWP8MnvIQ==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-ia32-msvc/-/resolver-binding-win32-ia32-msvc-1.12.2.tgz", + "integrity": "sha512-WD9sY00OfpHVGfsnHZoA8jVT+esS/Bg8z8jzxp5BnDCjjwsuKsPQrzswwpFy4J1AUJbXPRfkpcX0mXrzeXW79g==", "cpu": [ "ia32" ], @@ -5230,9 +5229,9 @@ ] }, "node_modules/@unrs/resolver-binding-win32-x64-msvc": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz", - "integrity": "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.12.2.tgz", + "integrity": "sha512-nAB74NfSNKknqQ1RrYj6uz8FcXEomu/MATJZxh/x+BArzN2U3JbOYC0APYzUIGhVY3m5hRxA8VPNdPBoG8txlA==", "cpu": [ "x64" ], @@ -5531,16 +5530,16 @@ } }, "node_modules/babel-jest": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.2.0.tgz", - "integrity": "sha512-0YiBEOxWqKkSQWL9nNGGEgndoeL0ZpWrbLMNL5u/Kaxrli3Eaxlt3ZtIDktEvXt4L/R9r3ODr2zKwGM/2BjxVw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.4.1.tgz", + "integrity": "sha512-fATAbM8piYxkiXQp3RBXmZHxZVNJZAVXXfyeyCN2Tida3+qJ8ea9UxhiJ2y4fLO90ZImKt6k9FlcH2+rLkJGhw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/transform": "30.2.0", + "@jest/transform": "30.4.1", "@types/babel__core": "^7.20.5", "babel-plugin-istanbul": "^7.0.1", - "babel-preset-jest": "30.2.0", + "babel-preset-jest": "30.4.0", "chalk": "^4.1.2", "graceful-fs": "^4.2.11", "slash": "^3.0.0" @@ -5610,9 +5609,9 @@ } }, "node_modules/babel-plugin-jest-hoist": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.2.0.tgz", - "integrity": "sha512-ftzhzSGMUnOzcCXd6WHdBGMyuwy15Wnn0iyyWGKgBDLxf9/s5ABuraCSpBX2uG0jUg4rqJnxsLc5+oYBqoxVaA==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-30.4.0.tgz", + "integrity": "sha512-9EdtWM/sSfXLOGLwSn+GS6pIXyBnL07/8gyJlwFXjWy4DxMOyItqyUT29d4lQiS380EZwYlX7/At4PgBS+m2aA==", "dev": true, "license": "MIT", "dependencies": { @@ -5650,13 +5649,13 @@ } }, "node_modules/babel-preset-jest": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.2.0.tgz", - "integrity": "sha512-US4Z3NOieAQumwFnYdUWKvUKh8+YSnS/gB3t6YBiz0bskpu7Pine8pPCheNxlPEW4wnUkma2a94YuW2q3guvCQ==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-30.4.0.tgz", + "integrity": "sha512-lBY4jxsNmCnSiu7kquw8ZC9F4+XLMOKypT3RnNHPvU2Kpd4W0xaPuLr5ZkRyOsvLYAY4yaW1ZwTW4xB7NIiZzg==", "dev": true, "license": "MIT", "dependencies": { - "babel-plugin-jest-hoist": "30.2.0", + "babel-plugin-jest-hoist": "30.4.0", "babel-preset-current-node-syntax": "^1.2.0" }, "engines": { @@ -6470,15 +6469,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/cluster-key-slot": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", - "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -6804,18 +6794,6 @@ "dev": true, "license": "MIT" }, - "node_modules/cron-parser": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", - "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", - "license": "MIT", - "dependencies": { - "luxon": "^3.2.1" - }, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -7040,30 +7018,6 @@ "node": ">=18" } }, - "node_modules/date.js": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/date.js/-/date.js-0.3.3.tgz", - "integrity": "sha512-HgigOS3h3k6HnW011nAb43c5xx5rBXk8P2v/WIT9Zv4koIaVXiH2BURguI78VVp+5Qc076T7OR378JViCnZtBw==", - "license": "MIT", - "dependencies": { - "debug": "~3.1.0" - } - }, - "node_modules/date.js/node_modules/debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/date.js/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -7115,9 +7069,9 @@ } }, "node_modules/dedent": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.1.tgz", - "integrity": "sha512-9JmrhGZpOlEgOLdQgSm0zxFaYoQon408V1v49aqTWuXENVlnCuY9JBZcXZiCsZQWDjTm5Qf/nIvAy77mXDAjEg==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", + "integrity": "sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==", "dev": true, "license": "MIT", "peerDependencies": { @@ -7164,15 +7118,6 @@ "node": ">=0.4.0" } }, - "node_modules/denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.10" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -7332,9 +7277,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.0.tgz", - "integrity": "sha512-nolgK9JcaUXMSmW+j1yaSvaEaoXYHwWyGJlkoCTghc97KgGDDSnpoU/PlEnw63Ah+TGKFOyY+X5LnxaWbCSfXg==", + "version": "3.4.10", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.10.tgz", + "integrity": "sha512-0xzNv0e7oYC6yyuOGZIABPM4qtg3QxLFniDNPP4ZP90wR8Yq3zgwpRbrNiT4N3IKqDbbYFEJLV+JWEs19aZ//w==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -7564,9 +7509,9 @@ } }, "node_modules/esbuild": { - "version": "0.25.11", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.11.tgz", - "integrity": "sha512-KohQwyzrKTQmhXDW1PjCv3Tyspn9n5GcY2RTDqeORIdIJY8yKIF7sTSopFmn/wpMPW4rdPXI0UE5LJLuq3bx0Q==", + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -7577,32 +7522,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.11", - "@esbuild/android-arm": "0.25.11", - "@esbuild/android-arm64": "0.25.11", - "@esbuild/android-x64": "0.25.11", - "@esbuild/darwin-arm64": "0.25.11", - "@esbuild/darwin-x64": "0.25.11", - "@esbuild/freebsd-arm64": "0.25.11", - "@esbuild/freebsd-x64": "0.25.11", - "@esbuild/linux-arm": "0.25.11", - "@esbuild/linux-arm64": "0.25.11", - "@esbuild/linux-ia32": "0.25.11", - "@esbuild/linux-loong64": "0.25.11", - "@esbuild/linux-mips64el": "0.25.11", - "@esbuild/linux-ppc64": "0.25.11", - "@esbuild/linux-riscv64": "0.25.11", - "@esbuild/linux-s390x": "0.25.11", - "@esbuild/linux-x64": "0.25.11", - "@esbuild/netbsd-arm64": "0.25.11", - "@esbuild/netbsd-x64": "0.25.11", - "@esbuild/openbsd-arm64": "0.25.11", - "@esbuild/openbsd-x64": "0.25.11", - "@esbuild/openharmony-arm64": "0.25.11", - "@esbuild/sunos-x64": "0.25.11", - "@esbuild/win32-arm64": "0.25.11", - "@esbuild/win32-ia32": "0.25.11", - "@esbuild/win32-x64": "0.25.11" + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" } }, "node_modules/escalade": { @@ -8001,18 +7946,18 @@ } }, "node_modules/expect": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/expect/-/expect-30.2.0.tgz", - "integrity": "sha512-u/feCi0GPsI+988gU2FLcsHyAHTU0MX1Wg68NhAnN7z/+C5wqG+CY8J53N9ioe8RXgaoz0nBR/TYMf3AycUuPw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/expect/-/expect-30.4.1.tgz", + "integrity": "sha512-PMARsyh/JtqC20HoGqlFcIlQAyqUtW4PlI1rup1uhYJtKuwAjbvWi3GQMAn+STdHum/dk8xrKfUM1+5SAwpolA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/expect-utils": "30.2.0", + "@jest/expect-utils": "30.4.1", "@jest/get-type": "30.1.0", - "jest-matcher-utils": "30.2.0", - "jest-message-util": "30.2.0", - "jest-mock": "30.2.0", - "jest-util": "30.2.0" + "jest-matcher-utils": "30.4.1", + "jest-message-util": "30.4.1", + "jest-mock": "30.4.1", + "jest-util": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -8433,16 +8378,16 @@ } }, "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", + "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "hasown": "^2.0.4", + "mime-types": "^2.1.35" }, "engines": { "node": ">= 6" @@ -8993,9 +8938,9 @@ } }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -9159,15 +9104,6 @@ "node": ">= 14" } }, - "node_modules/human-interval": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/human-interval/-/human-interval-2.0.1.tgz", - "integrity": "sha512-r4Aotzf+OtKIGQCB3odUowy4GfUDTy3aTWTfLd7ZF2gBCy3XW3v/dJLRefZnOFFnjqs5B1TypvS8WarpBkYUNQ==", - "license": "MIT", - "dependencies": { - "numbered": "^1.1.0" - } - }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -9291,30 +9227,6 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, - "node_modules/ioredis": { - "version": "5.8.2", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz", - "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==", - "license": "MIT", - "dependencies": { - "@ioredis/commands": "1.4.0", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.1.0", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - }, - "engines": { - "node": ">=12.22.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, "node_modules/ip-address": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", @@ -9740,17 +9652,17 @@ } }, "node_modules/jest": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest/-/jest-30.2.0.tgz", - "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest/-/jest-30.4.2.tgz", + "integrity": "sha512-Yi1jqNC/Oq0N4hBgNH/YvBpP1P57QqundgytzYqy3yqAa7NZPNjSoi4SGbRAXDMdBzNE6xBCi5U7RgfrvMEUVQ==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@jest/core": "30.2.0", - "@jest/types": "30.2.0", + "@jest/core": "30.4.2", + "@jest/types": "30.4.1", "import-local": "^3.2.0", - "jest-cli": "30.2.0" + "jest-cli": "30.4.2" }, "bin": { "jest": "bin/jest.js" @@ -9768,14 +9680,14 @@ } }, "node_modules/jest-changed-files": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.2.0.tgz", - "integrity": "sha512-L8lR1ChrRnSdfeOvTrwZMlnWV8G/LLjQ0nG9MBclwWZidA2N5FviRki0Bvh20WRMOX31/JYvzdqTJrk5oBdydQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-30.4.1.tgz", + "integrity": "sha512-IuctmYrxi21iOSOaIXpJWalHyPAsVv0GeBHKDn8C1CA4W5htHn7INL+wdnL4Bo0+olEndvAFkmb++tIQJG+vvg==", "dev": true, "license": "MIT", "dependencies": { "execa": "^5.1.1", - "jest-util": "30.2.0", + "jest-util": "30.4.1", "p-limit": "^3.1.0" }, "engines": { @@ -9783,29 +9695,29 @@ } }, "node_modules/jest-circus": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.2.0.tgz", - "integrity": "sha512-Fh0096NC3ZkFx05EP2OXCxJAREVxj1BcW/i6EWqqymcgYKWjyyDpral3fMxVcHXg6oZM7iULer9wGRFvfpl+Tg==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.4.2.tgz", + "integrity": "sha512-rvHH7VlY6LgbJXJTQ87GW62g1FntOtbhh0zT+v04kC+pgL6aBKyYINXxWukCpj3dcIBMw5/XUbtDS9dU9JTXeQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.2.0", - "@jest/expect": "30.2.0", - "@jest/test-result": "30.2.0", - "@jest/types": "30.2.0", + "@jest/environment": "30.4.1", + "@jest/expect": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", "chalk": "^4.1.2", "co": "^4.6.0", "dedent": "^1.6.0", "is-generator-fn": "^2.1.0", - "jest-each": "30.2.0", - "jest-matcher-utils": "30.2.0", - "jest-message-util": "30.2.0", - "jest-runtime": "30.2.0", - "jest-snapshot": "30.2.0", - "jest-util": "30.2.0", + "jest-each": "30.4.1", + "jest-matcher-utils": "30.4.1", + "jest-message-util": "30.4.1", + "jest-runtime": "30.4.2", + "jest-snapshot": "30.4.1", + "jest-util": "30.4.1", "p-limit": "^3.1.0", - "pretty-format": "30.2.0", + "pretty-format": "30.4.1", "pure-rand": "^7.0.0", "slash": "^3.0.0", "stack-utils": "^2.0.6" @@ -9815,21 +9727,21 @@ } }, "node_modules/jest-cli": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.2.0.tgz", - "integrity": "sha512-Os9ukIvADX/A9sLt6Zse3+nmHtHaE6hqOsjQtNiugFTbKRHYIYtZXNGNK9NChseXy7djFPjndX1tL0sCTlfpAA==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.4.2.tgz", + "integrity": "sha512-jfA2ocvVHMXS2QijrJ0d31ektP+d/W0T5RpcTX2Pq+3sVqHlsXVCM2+FmwpL+bdY8OfHpIg9xMxLF17Zg0U49Q==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "30.2.0", - "@jest/test-result": "30.2.0", - "@jest/types": "30.2.0", + "@jest/core": "30.4.2", + "@jest/test-result": "30.4.1", + "@jest/types": "30.4.1", "chalk": "^4.1.2", "exit-x": "^0.2.2", "import-local": "^3.2.0", - "jest-config": "30.2.0", - "jest-util": "30.2.0", - "jest-validate": "30.2.0", + "jest-config": "30.4.2", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", "yargs": "^17.7.2" }, "bin": { @@ -9848,34 +9760,33 @@ } }, "node_modules/jest-config": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.2.0.tgz", - "integrity": "sha512-g4WkyzFQVWHtu6uqGmQR4CQxz/CH3yDSlhzXMWzNjDx843gYjReZnMRanjRCq5XZFuQrGDxgUaiYWE8BRfVckA==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.4.2.tgz", + "integrity": "sha512-rNHAShJQqQwFNoL0hbf3BphSBOWnpOUAKvidLS/AjNVLPfoj5mSf4jQMfW3cYOs6hXeZC7nF7mDHaBnbxELOzg==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.27.4", "@jest/get-type": "30.1.0", - "@jest/pattern": "30.0.1", - "@jest/test-sequencer": "30.2.0", - "@jest/types": "30.2.0", - "babel-jest": "30.2.0", + "@jest/pattern": "30.4.0", + "@jest/test-sequencer": "30.4.1", + "@jest/types": "30.4.1", + "babel-jest": "30.4.1", "chalk": "^4.1.2", "ci-info": "^4.2.0", "deepmerge": "^4.3.1", - "glob": "^10.3.10", + "glob": "^10.5.0", "graceful-fs": "^4.2.11", - "jest-circus": "30.2.0", - "jest-docblock": "30.2.0", - "jest-environment-node": "30.2.0", - "jest-regex-util": "30.0.1", - "jest-resolve": "30.2.0", - "jest-runner": "30.2.0", - "jest-util": "30.2.0", - "jest-validate": "30.2.0", - "micromatch": "^4.0.8", + "jest-circus": "30.4.2", + "jest-docblock": "30.4.0", + "jest-environment-node": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-resolve": "30.4.1", + "jest-runner": "30.4.2", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", "parse-json": "^5.2.0", - "pretty-format": "30.2.0", + "pretty-format": "30.4.1", "slash": "^3.0.0", "strip-json-comments": "^3.1.1" }, @@ -9900,25 +9811,25 @@ } }, "node_modules/jest-diff": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", - "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.4.1.tgz", + "integrity": "sha512-CRpFK0RtLriVDGcPPAnR6HMVI8bSR2jnUIgralhauzYQZIb4RH9AtEInTuQr65LmmGggGcRT6HIASxwqsVsmlA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/diff-sequences": "30.0.1", + "@jest/diff-sequences": "30.4.0", "@jest/get-type": "30.1.0", "chalk": "^4.1.2", - "pretty-format": "30.2.0" + "pretty-format": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-docblock": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.2.0.tgz", - "integrity": "sha512-tR/FFgZKS1CXluOQzZvNH3+0z9jXr3ldGSD8bhyuxvlVUwbeLOGynkunvlTMxchC5urrKndYiwCFC0DLVjpOCA==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-30.4.0.tgz", + "integrity": "sha512-ZPMabUZCx5MpbZ2eBYSvZ0J8fvo3dR9oM+eeUpb3aKNQFuS2tu3Duw1TNlMoP8k3WQgKGJuhcMFvwcVuq6T7oA==", "dev": true, "license": "MIT", "dependencies": { @@ -9929,57 +9840,57 @@ } }, "node_modules/jest-each": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.2.0.tgz", - "integrity": "sha512-lpWlJlM7bCUf1mfmuqTA8+j2lNURW9eNafOy99knBM01i5CQeY5UH1vZjgT9071nDJac1M4XsbyI44oNOdhlDQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.4.1.tgz", + "integrity": "sha512-/8MJbH6fuj48TstjrMf+u/pd06Qezz5xOXvZA6442heNOWr8bdeoGZX2d9fCn028CoMgYmroH9//zky5GfyYmA==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "chalk": "^4.1.2", - "jest-util": "30.2.0", - "pretty-format": "30.2.0" + "jest-util": "30.4.1", + "pretty-format": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-environment-node": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.2.0.tgz", - "integrity": "sha512-ElU8v92QJ9UrYsKrxDIKCxu6PfNj4Hdcktcn0JX12zqNdqWHB0N+hwOnnBBXvjLd2vApZtuLUGs1QSY+MsXoNA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.4.1.tgz", + "integrity": "sha512-4FZYVOk85hz2AyT6BbarKy9u37g6DbrDyCdFhsnDdXqyrueYQvB+0zO4f/kqLCRD0BsPRXPMNJeQwihKZV8naw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.2.0", - "@jest/fake-timers": "30.2.0", - "@jest/types": "30.2.0", + "@jest/environment": "30.4.1", + "@jest/fake-timers": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", - "jest-mock": "30.2.0", - "jest-util": "30.2.0", - "jest-validate": "30.2.0" + "jest-mock": "30.4.1", + "jest-util": "30.4.1", + "jest-validate": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-haste-map": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.2.0.tgz", - "integrity": "sha512-sQA/jCb9kNt+neM0anSj6eZhLZUIhQgwDt7cPGjumgLM4rXsfb9kpnlacmvZz3Q5tb80nS+oG/if+NBKrHC+Xw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.4.1.tgz", + "integrity": "sha512-rFrcONd8jeFsyw+Z9CrScJgglRf2+NFmNam8dKu7n+SoHqNYT47mn0DdEcVUZJpvh7Iz6/si7f7yUH7GJHVgnw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "@types/node": "*", "anymatch": "^3.1.3", "fb-watchman": "^2.0.2", "graceful-fs": "^4.2.11", - "jest-regex-util": "30.0.1", - "jest-util": "30.2.0", - "jest-worker": "30.2.0", - "micromatch": "^4.0.8", + "jest-regex-util": "30.4.0", + "jest-util": "30.4.1", + "jest-worker": "30.4.1", + "picomatch": "^4.0.3", "walker": "^1.0.8" }, "engines": { @@ -9989,50 +9900,64 @@ "fsevents": "^2.3.3" } }, + "node_modules/jest-haste-map/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/jest-leak-detector": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.2.0.tgz", - "integrity": "sha512-M6jKAjyzjHG0SrQgwhgZGy9hFazcudwCNovY/9HPIicmNSBuockPSedAP9vlPK6ONFJ1zfyH/M2/YYJxOz5cdQ==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.4.1.tgz", + "integrity": "sha512-IpmyiioeHxiWDhesHnUFmOxcTzwCwKpgACgWajtAP+nYQXiY7DakTxB6Bx9JFiRMljr0AX1PvnQdaU1KFoz6NQ==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", - "pretty-format": "30.2.0" + "pretty-format": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-matcher-utils": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.2.0.tgz", - "integrity": "sha512-dQ94Nq4dbzmUWkQ0ANAWS9tBRfqCrn0bV9AMYdOi/MHW726xn7eQmMeRTpX2ViC00bpNaWXq+7o4lIQ3AX13Hg==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.4.1.tgz", + "integrity": "sha512-zvYfX5CaeEkFrrLS9suWe9rvJrm9J1Iv3ua8kIBv9GEPzcnsfBf0bob37la7s67fs0nlBC3EuvkOLnXQKxtx4A==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", "chalk": "^4.1.2", - "jest-diff": "30.2.0", - "pretty-format": "30.2.0" + "jest-diff": "30.4.1", + "pretty-format": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-message-util": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.2.0.tgz", - "integrity": "sha512-y4DKFLZ2y6DxTWD4cDe07RglV88ZiNEdlRfGtqahfbIjfsw1nMCPx49Uev4IA/hWn3sDKyAnSPwoYSsAEdcimw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.4.1.tgz", + "integrity": "sha512-kwCKIvq0MCW1HzLoGola9Te6JUdzgV0loyKJ3Qghrkz9i5/RRIHsL95BMQc2HBBhlBKC4j22K9p11TGHH8RBpQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.27.1", - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "@types/stack-utils": "^2.0.3", "chalk": "^4.1.2", "graceful-fs": "^4.2.11", - "micromatch": "^4.0.8", - "pretty-format": "30.2.0", + "jest-util": "30.4.1", + "picomatch": "^4.0.3", + "pretty-format": "30.4.1", "slash": "^3.0.0", "stack-utils": "^2.0.6" }, @@ -10040,16 +9965,29 @@ "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, + "node_modules/jest-message-util/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/jest-mock": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.2.0.tgz", - "integrity": "sha512-JNNNl2rj4b5ICpmAcq+WbLH83XswjPbjH4T7yvGzfAGCPh1rw+xVNbtk+FnRslvt9lkCcdn9i1oAoKUuFsOxRw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-30.4.1.tgz", + "integrity": "sha512-/i8SVb8/NSB7RfNi8gfqu8gxLV23KaL5EpAttyb9iz8qWRIqXRLflycz/32wXsYkOnaUlx8NAKnJYtpsmXUmfw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "@types/node": "*", - "jest-util": "30.2.0" + "jest-util": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -10074,9 +10012,9 @@ } }, "node_modules/jest-regex-util": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.0.1.tgz", - "integrity": "sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA==", + "version": "30.4.0", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.4.0.tgz", + "integrity": "sha512-mWlvLviKIgIQ8VCuM1xRdD0TWp3zlzionlmDBjuXVBs+VkmXq6FgW9T4Emr7oGz/Rk6feDCGyiugolcQEyp3mg==", "dev": true, "license": "MIT", "engines": { @@ -10084,18 +10022,18 @@ } }, "node_modules/jest-resolve": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.2.0.tgz", - "integrity": "sha512-TCrHSxPlx3tBY3hWNtRQKbtgLhsXa1WmbJEqBlTBrGafd5fiQFByy2GNCEoGR+Tns8d15GaL9cxEzKOO3GEb2A==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.4.1.tgz", + "integrity": "sha512-Zry8Yq/yJcNAZ7dJ5F2heic8AheXvbFZ7XI5V+h28nrYZ7Qoyy4dItq8OodjnYD270mvX+ZudmrNV9cysqhW5Q==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^4.1.2", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.2.0", + "jest-haste-map": "30.4.1", "jest-pnp-resolver": "^1.2.3", - "jest-util": "30.2.0", - "jest-validate": "30.2.0", + "jest-util": "30.4.1", + "jest-validate": "30.4.1", "slash": "^3.0.0", "unrs-resolver": "^1.7.11" }, @@ -10104,46 +10042,46 @@ } }, "node_modules/jest-resolve-dependencies": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.2.0.tgz", - "integrity": "sha512-xTOIGug/0RmIe3mmCqCT95yO0vj6JURrn1TKWlNbhiAefJRWINNPgwVkrVgt/YaerPzY3iItufd80v3lOrFJ2w==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.4.2.tgz", + "integrity": "sha512-gDiVh1I+GxYzz9oXlyw+1wv6VOYX1WYxMOfjsA3iGKePV2oxmbHhwxfkALxNxYy1ciw6APWwkW2zZONwP97aEQ==", "dev": true, "license": "MIT", "dependencies": { - "jest-regex-util": "30.0.1", - "jest-snapshot": "30.2.0" + "jest-regex-util": "30.4.0", + "jest-snapshot": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-runner": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.2.0.tgz", - "integrity": "sha512-PqvZ2B2XEyPEbclp+gV6KO/F1FIFSbIwewRgmROCMBo/aZ6J1w8Qypoj2pEOcg3G2HzLlaP6VUtvwCI8dM3oqQ==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.4.2.tgz", + "integrity": "sha512-2dw0PslVYXxffXGpLo+Ejad+KcI1Qkjn7f4X4619gf21oCUmL+SPfjqIa/losUem3yEOvfNZe/F1HWUcNpODcg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.2.0", - "@jest/environment": "30.2.0", - "@jest/test-result": "30.2.0", - "@jest/transform": "30.2.0", - "@jest/types": "30.2.0", + "@jest/console": "30.4.1", + "@jest/environment": "30.4.1", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", "chalk": "^4.1.2", "emittery": "^0.13.1", "exit-x": "^0.2.2", "graceful-fs": "^4.2.11", - "jest-docblock": "30.2.0", - "jest-environment-node": "30.2.0", - "jest-haste-map": "30.2.0", - "jest-leak-detector": "30.2.0", - "jest-message-util": "30.2.0", - "jest-resolve": "30.2.0", - "jest-runtime": "30.2.0", - "jest-util": "30.2.0", - "jest-watcher": "30.2.0", - "jest-worker": "30.2.0", + "jest-docblock": "30.4.0", + "jest-environment-node": "30.4.1", + "jest-haste-map": "30.4.1", + "jest-leak-detector": "30.4.1", + "jest-message-util": "30.4.1", + "jest-resolve": "30.4.1", + "jest-runtime": "30.4.2", + "jest-util": "30.4.1", + "jest-watcher": "30.4.1", + "jest-worker": "30.4.1", "p-limit": "^3.1.0", "source-map-support": "0.5.13" }, @@ -10152,32 +10090,32 @@ } }, "node_modules/jest-runtime": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.2.0.tgz", - "integrity": "sha512-p1+GVX/PJqTucvsmERPMgCPvQJpFt4hFbM+VN3n8TMo47decMUcJbt+rgzwrEme0MQUA/R+1de2axftTHkKckg==", + "version": "30.4.2", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.4.2.tgz", + "integrity": "sha512-3/5e8iPz2k/VLqlr8DgTftYyLUv8Su3FkCAO2/Od81UsUTpSxOrS6O5x5KkoQwyUjmpYyDJKeyAvg2T2nvpNkQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.2.0", - "@jest/fake-timers": "30.2.0", - "@jest/globals": "30.2.0", + "@jest/environment": "30.4.1", + "@jest/fake-timers": "30.4.1", + "@jest/globals": "30.4.1", "@jest/source-map": "30.0.1", - "@jest/test-result": "30.2.0", - "@jest/transform": "30.2.0", - "@jest/types": "30.2.0", + "@jest/test-result": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", "chalk": "^4.1.2", "cjs-module-lexer": "^2.1.0", "collect-v8-coverage": "^1.0.2", - "glob": "^10.3.10", + "glob": "^10.5.0", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.2.0", - "jest-message-util": "30.2.0", - "jest-mock": "30.2.0", - "jest-regex-util": "30.0.1", - "jest-resolve": "30.2.0", - "jest-snapshot": "30.2.0", - "jest-util": "30.2.0", + "jest-haste-map": "30.4.1", + "jest-message-util": "30.4.1", + "jest-mock": "30.4.1", + "jest-regex-util": "30.4.0", + "jest-resolve": "30.4.1", + "jest-snapshot": "30.4.1", + "jest-util": "30.4.1", "slash": "^3.0.0", "strip-bom": "^4.0.0" }, @@ -10186,9 +10124,9 @@ } }, "node_modules/jest-snapshot": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.2.0.tgz", - "integrity": "sha512-5WEtTy2jXPFypadKNpbNkZ72puZCa6UjSr/7djeecHWOu7iYhSXSnHScT8wBz3Rn8Ena5d5RYRcsyKIeqG1IyA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.4.1.tgz", + "integrity": "sha512-tEOkkfOMppUyeiHwjZswOQ3lcnoTnws/q5FnGIaeIh/jmoU0ZlgMYRR8sTlTj+nNGCoJ0RDq6SfxGxCsyMTPmw==", "dev": true, "license": "MIT", "dependencies": { @@ -10197,20 +10135,20 @@ "@babel/plugin-syntax-jsx": "^7.27.1", "@babel/plugin-syntax-typescript": "^7.27.1", "@babel/types": "^7.27.3", - "@jest/expect-utils": "30.2.0", + "@jest/expect-utils": "30.4.1", "@jest/get-type": "30.1.0", - "@jest/snapshot-utils": "30.2.0", - "@jest/transform": "30.2.0", - "@jest/types": "30.2.0", + "@jest/snapshot-utils": "30.4.1", + "@jest/transform": "30.4.1", + "@jest/types": "30.4.1", "babel-preset-current-node-syntax": "^1.2.0", "chalk": "^4.1.2", - "expect": "30.2.0", + "expect": "30.4.1", "graceful-fs": "^4.2.11", - "jest-diff": "30.2.0", - "jest-matcher-utils": "30.2.0", - "jest-message-util": "30.2.0", - "jest-util": "30.2.0", - "pretty-format": "30.2.0", + "jest-diff": "30.4.1", + "jest-matcher-utils": "30.4.1", + "jest-message-util": "30.4.1", + "jest-util": "30.4.1", + "pretty-format": "30.4.1", "semver": "^7.7.2", "synckit": "^0.11.8" }, @@ -10219,18 +10157,18 @@ } }, "node_modules/jest-util": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.2.0.tgz", - "integrity": "sha512-QKNsM0o3Xe6ISQU869e+DhG+4CK/48aHYdJZGlFQVTjnbvgpcKyxpzk29fGiO7i/J8VENZ+d2iGnSsvmuHywlA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-30.4.1.tgz", + "integrity": "sha512-vjQb1sACEiv13DKJMDToJpzVW0joCsIQrmbg0fi7CyOOt+g9jTuQl2A216pWRBYhOVt53XbL/2LbMKg1BECWOw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "@types/node": "*", "chalk": "^4.1.2", "ci-info": "^4.2.0", "graceful-fs": "^4.2.11", - "picomatch": "^4.0.2" + "picomatch": "^4.0.3" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -10250,18 +10188,18 @@ } }, "node_modules/jest-validate": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.2.0.tgz", - "integrity": "sha512-FBGWi7dP2hpdi8nBoWxSsLvBFewKAg0+uSQwBaof4Y4DPgBabXgpSYC5/lR7VmnIlSpASmCi/ntRWPbv7089Pw==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.4.1.tgz", + "integrity": "sha512-PDWi4SOwLnwqNDfHZjOcsEFyZ4fc/2W2gVL3DEoyqnB6jCQMLRtfBong8s6omIw3lI0HWOus12xfnFmQtjW3fw==", "dev": true, "license": "MIT", "dependencies": { "@jest/get-type": "30.1.0", - "@jest/types": "30.2.0", + "@jest/types": "30.4.1", "camelcase": "^6.3.0", "chalk": "^4.1.2", "leven": "^3.1.0", - "pretty-format": "30.2.0" + "pretty-format": "30.4.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -10281,19 +10219,19 @@ } }, "node_modules/jest-watcher": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.2.0.tgz", - "integrity": "sha512-PYxa28dxJ9g777pGm/7PrbnMeA0Jr7osHP9bS7eJy9DuAjMgdGtxgf0uKMyoIsTWAkIbUW5hSDdJ3urmgXBqxg==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.4.1.tgz", + "integrity": "sha512-/l9UonmvCwjHH7d2h3iAwIloLc1H0S8mJZ/LNK3i86hqwPAz8otUJjP9MfYtz9Tt77Su5FD2xGjZn8d31IZHlw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.2.0", - "@jest/types": "30.2.0", + "@jest/test-result": "30.4.1", + "@jest/types": "30.4.1", "@types/node": "*", "ansi-escapes": "^4.3.2", "chalk": "^4.1.2", "emittery": "^0.13.1", - "jest-util": "30.2.0", + "jest-util": "30.4.1", "string-length": "^4.0.2" }, "engines": { @@ -10301,15 +10239,15 @@ } }, "node_modules/jest-worker": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.2.0.tgz", - "integrity": "sha512-0Q4Uk8WF7BUwqXHuAjc23vmopWJw5WH7w2tqBoUOZpOjW/ZnR44GXXd1r82RvnmI2GZge3ivrYXk/BE2+VtW2g==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.4.1.tgz", + "integrity": "sha512-SHynN/q/QD++iNyvMdy+WMmbCGk8jIsNcRxycXbWubSOhvo6T+j2afcfUSl+3hYsiBebOTo0cT7c2H7CXugu1g==", "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", "@ungap/structured-clone": "^1.3.0", - "jest-util": "30.2.0", + "jest-util": "30.4.1", "merge-stream": "^2.0.0", "supports-color": "^8.1.1" }, @@ -10349,9 +10287,19 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -10785,24 +10733,12 @@ "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", "license": "MIT" }, - "node_modules/lodash.defaults": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", - "license": "MIT" - }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", "license": "MIT" }, - "node_modules/lodash.isarguments": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", - "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", - "license": "MIT" - }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -10991,15 +10927,6 @@ "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", "license": "ISC" }, - "node_modules/luxon": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", - "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", - "license": "MIT", - "engines": { - "node": ">=12" - } - }, "node_modules/mailersend": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/mailersend/-/mailersend-2.6.0.tgz", @@ -11684,27 +11611,6 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "license": "MIT" }, - "node_modules/moment": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", - "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/moment-timezone": { - "version": "0.5.48", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.48.tgz", - "integrity": "sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw==", - "license": "MIT", - "dependencies": { - "moment": "^2.29.4" - }, - "engines": { - "node": "*" - } - }, "node_modules/mongodb": { "version": "6.20.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.20.0.tgz", @@ -12278,12 +12184,6 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, - "node_modules/numbered": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/numbered/-/numbered-1.1.0.tgz", - "integrity": "sha512-pv/ue2Odr7IfYOO0byC1KgBI10wo5YDauLhxY6/saNzAdAs0r1SotGCPzzCLNPL0xtrAwWRialLu23AAu9xO1g==", - "license": "MIT" - }, "node_modules/nwsapi": { "version": "2.2.22", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.22.tgz", @@ -13404,15 +13304,16 @@ } }, "node_modules/pretty-format": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", - "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", + "version": "30.4.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.4.1.tgz", + "integrity": "sha512-K6KiKMHTL4jjX4u3Kir2EW07nRfcqVTXIImx50wbjHQTcZPgg+gjVeNTIT3l3L1Rd4UefxfogquC9J37SoFyyw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/schemas": "30.0.5", + "@jest/schemas": "30.4.1", "ansi-styles": "^5.2.0", - "react-is": "^18.3.1" + "react-is-18": "npm:react-is@^18.3.1", + "react-is-19": "npm:react-is@^19.2.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" @@ -13438,9 +13339,9 @@ "license": "MIT" }, "node_modules/protobufjs": { - "version": "7.6.1", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.1.tgz", - "integrity": "sha512-4K0myLaWL5EteuSAro91EGFgcfVgxb64Jx+7oDAY6GOkXD4M69yuSEljNcInGVCA5sOPxmZ/EqDLj2x0Q0+Ygg==", + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -13450,7 +13351,6 @@ "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", @@ -13644,13 +13544,22 @@ "node": ">= 0.8" } }, - "node_modules/react-is": { + "node_modules/react-is-18": { + "name": "react-is", "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true, "license": "MIT" }, + "node_modules/react-is-19": { + "name": "react-is", + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.7.tgz", + "integrity": "sha512-kZFnouyVv7eP/Phmrlo9FK+zcAdriZJvzxXHF1Sl1P377WSGe2G/JxVolhTrB/jeV47lKImhNUsijjHAAbcl/A==", + "dev": true, + "license": "MIT" + }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -13677,27 +13586,6 @@ "node": ">=8.10.0" } }, - "node_modules/redis-errors": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/redis-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", - "license": "MIT", - "dependencies": { - "redis-errors": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -13939,9 +13827,9 @@ } }, "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.4.tgz", + "integrity": "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -14368,12 +14256,6 @@ "node": ">=8" } }, - "node_modules/standard-as-callback": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", - "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", - "license": "MIT" - }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -14707,13 +14589,13 @@ "license": "MIT" }, "node_modules/synckit": { - "version": "0.11.12", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.12.tgz", - "integrity": "sha512-Bh7QjT8/SuKUIfObSXNHNSK6WHo6J1tHCqJsuaFDP7gP0fkzSfTxI8y85JrppZ0h8l0maIgc2tfuZQ6/t3GtnQ==", + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.13.tgz", + "integrity": "sha512-eNRKgb3z66Yp3D2CixVujOUvXLFUTij/zVnV8KRyvFdQwpz7I5DS8UfRkTeLzb64u+dkzDSdelE24izu+zSSUg==", "dev": true, "license": "MIT", "dependencies": { - "@pkgr/core": "^0.2.9" + "@pkgr/core": "^0.3.6" }, "engines": { "node": "^14.18.0 || >=16.0.0" @@ -14928,19 +14810,19 @@ } }, "node_modules/ts-jest": { - "version": "29.4.6", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.6.tgz", - "integrity": "sha512-fSpWtOO/1AjSNQguk43hb/JCo16oJDnMJf3CdEGNkqsEX3t0KX96xvyX1D7PfLCpVoKu4MfVrqUkFyblYoY4lA==", + "version": "29.4.11", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.11.tgz", + "integrity": "sha512-IrFl7l9AuB/qrNw5quqvAv/hmKMb8dhWOH4jQOGo0Oq8tCeo1O86/iTFG1FaRimgUkF13l4PcepO8ATFT6Ns4g==", "dev": true, "license": "MIT", "dependencies": { "bs-logger": "^0.2.6", "fast-json-stable-stringify": "^2.1.0", - "handlebars": "^4.7.8", + "handlebars": "^4.7.9", "json5": "^2.2.3", "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", - "semver": "^7.7.3", + "semver": "^7.8.0", "type-fest": "^4.41.0", "yargs-parser": "^21.1.1" }, @@ -14957,7 +14839,7 @@ "babel-jest": "^29.0.0 || ^30.0.0", "jest": "^29.0.0 || ^30.0.0", "jest-util": "^29.0.0 || ^30.0.0", - "typescript": ">=4.3 <6" + "typescript": ">=4.3 <7" }, "peerDependenciesMeta": { "@babel/core": { @@ -15237,38 +15119,41 @@ } }, "node_modules/unrs-resolver": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.11.1.tgz", - "integrity": "sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.12.2.tgz", + "integrity": "sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==", "dev": true, "hasInstallScript": true, "license": "MIT", "dependencies": { - "napi-postinstall": "^0.3.0" + "napi-postinstall": "^0.3.4" }, "funding": { "url": "https://opencollective.com/unrs-resolver" }, "optionalDependencies": { - "@unrs/resolver-binding-android-arm-eabi": "1.11.1", - "@unrs/resolver-binding-android-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-arm64": "1.11.1", - "@unrs/resolver-binding-darwin-x64": "1.11.1", - "@unrs/resolver-binding-freebsd-x64": "1.11.1", - "@unrs/resolver-binding-linux-arm-gnueabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm-musleabihf": "1.11.1", - "@unrs/resolver-binding-linux-arm64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-arm64-musl": "1.11.1", - "@unrs/resolver-binding-linux-ppc64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-riscv64-musl": "1.11.1", - "@unrs/resolver-binding-linux-s390x-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-gnu": "1.11.1", - "@unrs/resolver-binding-linux-x64-musl": "1.11.1", - "@unrs/resolver-binding-wasm32-wasi": "1.11.1", - "@unrs/resolver-binding-win32-arm64-msvc": "1.11.1", - "@unrs/resolver-binding-win32-ia32-msvc": "1.11.1", - "@unrs/resolver-binding-win32-x64-msvc": "1.11.1" + "@unrs/resolver-binding-android-arm-eabi": "1.12.2", + "@unrs/resolver-binding-android-arm64": "1.12.2", + "@unrs/resolver-binding-darwin-arm64": "1.12.2", + "@unrs/resolver-binding-darwin-x64": "1.12.2", + "@unrs/resolver-binding-freebsd-x64": "1.12.2", + "@unrs/resolver-binding-linux-arm-gnueabihf": "1.12.2", + "@unrs/resolver-binding-linux-arm-musleabihf": "1.12.2", + "@unrs/resolver-binding-linux-arm64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-arm64-musl": "1.12.2", + "@unrs/resolver-binding-linux-loong64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-loong64-musl": "1.12.2", + "@unrs/resolver-binding-linux-ppc64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-riscv64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-riscv64-musl": "1.12.2", + "@unrs/resolver-binding-linux-s390x-gnu": "1.12.2", + "@unrs/resolver-binding-linux-x64-gnu": "1.12.2", + "@unrs/resolver-binding-linux-x64-musl": "1.12.2", + "@unrs/resolver-binding-openharmony-arm64": "1.12.2", + "@unrs/resolver-binding-wasm32-wasi": "1.12.2", + "@unrs/resolver-binding-win32-arm64-msvc": "1.12.2", + "@unrs/resolver-binding-win32-ia32-msvc": "1.12.2", + "@unrs/resolver-binding-win32-x64-msvc": "1.12.2" } }, "node_modules/update-browserslist-db": { diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 092471b8c..a60505c25 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -285,6 +285,7 @@ export const initializeServices = async ({ checkService, settingsService, monitorsRepository, + jobsRepository, teamsRepository, monitorStatsRepository, checksRepository, diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index 4def9cd86..3ec95d4c2 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -43,6 +43,9 @@ export interface IJobsRepository { // Create jobs. Takes a JobSeed upsertJob(job: JobSeed): Promise; + // Creates cleanup jobs, these are executed immediately so need their own method + upsertCleanupJob(job: JobSeed): Promise; + // Pause/Resume setActiveById(refId: string, isActive: boolean): Promise; @@ -55,6 +58,9 @@ export interface IJobsRepository { // Delete a single job of type deleteByIdAndType(refId: string, type: JobType): Promise; + // Delete jobs that reference monitors that no longer exist + deleteByMonitorIdsNotIn(monitorIds: string[]): Promise; + // Get job findById(refId: string): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index 5069c90c4..e2dfd2e2a 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -214,6 +214,31 @@ class MongoJobsRepository implements IJobsRepository { return res.acknowledged; }; + upsertCleanupJob = async (job: JobSeed) => { + const res = await JobModel.updateOne( + { _id: job.id }, + { + $set: { + type: job.type, + refId: job.refId, + isActive: job.isActive, + intervalMs: job.intervalMs, + nextScheduledAt: job.nextScheduledAt, + }, + $setOnInsert: { + lockedBy: null, + lockedUntil: null, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + }, + }, + { upsert: true } + ); + return res.acknowledged; + }; + setActiveById = async (refId: string, isActive: boolean) => { const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together return res.modifiedCount > 0; @@ -234,6 +259,12 @@ class MongoJobsRepository implements IJobsRepository { return res.deletedCount > 0; }; + deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise => { + // remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs) + const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } }); + return res.deletedCount ?? 0; + }; + findById = async (refId: string) => { const docs = await JobModel.find({ refId }).lean(); return docs.map(this.toEntity); diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index d8c997453..f44d91c98 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -274,8 +274,9 @@ export class DBQueueWorker implements IWorker { await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now)); } } - await this.jobsRepository.upsertJob(this.toCleanupJob("cleanup-orphaned", now)); - await this.jobsRepository.upsertJob(this.toCleanupJob("cleanup-retention", now)); + // Both cleanup jobs run immediately on every startup, then once a day + await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now)); + await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now)); }; // Register worker diff --git a/server/src/worker/worker.helper.ts b/server/src/worker/worker.helper.ts index 29de9f2a3..7b60a26ba 100644 --- a/server/src/worker/worker.helper.ts +++ b/server/src/worker/worker.helper.ts @@ -8,6 +8,7 @@ import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.i import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js"; import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; import { ILogger } from "@/utils/logger.js"; import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; @@ -42,6 +43,7 @@ export class WorkerHelper implements IWorkerHelper { private checkService: ICheckService; private settingsService: ISettingsService; private monitorsRepository: IMonitorsRepository; + private jobsRepository: IJobsRepository; private teamsRepository: ITeamsRepository; private monitorStatsRepository: IMonitorStatsRepository; private checksRepository: IChecksRepository; @@ -56,6 +58,7 @@ export class WorkerHelper implements IWorkerHelper { checkService: ICheckService, settingsService: ISettingsService, monitorsRepository: IMonitorsRepository, + jobsRepository: IJobsRepository, teamsRepository: ITeamsRepository, monitorStatsRepository: IMonitorStatsRepository, checksRepository: IChecksRepository, @@ -69,6 +72,7 @@ export class WorkerHelper implements IWorkerHelper { this.checkService = checkService; this.settingsService = settingsService; this.monitorsRepository = monitorsRepository; + this.jobsRepository = jobsRepository; this.teamsRepository = teamsRepository; this.monitorStatsRepository = monitorStatsRepository; this.checksRepository = checksRepository; @@ -190,6 +194,16 @@ export class WorkerHelper implements IWorkerHelper { }); } + // Remove orphaned jobs + const deletedJobsCount = await this.jobsRepository.deleteByMonitorIdsNotIn(allMonitorIds); + if (deletedJobsCount > 0) { + this.logger.info({ + message: `Deleted ${deletedJobsCount} orphaned jobs`, + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + } + this.logger.info({ message: "Cleanup of orphaned data completed", service: SERVICE_NAME, diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index 389cdba1c..9c2f54f94 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -347,6 +347,34 @@ describe("MongoJobsRepository", () => { expect(row?.runCount).toBe(7); }); + it("upsertCleanupJob resets nextScheduledAt on an existing row so it is due again on restart", async () => { + const job: Job = { + id: "cleanup-orphaned", + type: "cleanup-orphaned", + refId: null, + isActive: true, + nextScheduledAt: NOW, + intervalMs: 86_400_000, + lockedBy: null, + lockedUntil: null, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + }; + await repo.upsertCleanupJob(job); + + // row has already advanced a day into the future after a previous run + await JobModel.updateOne({ _id: "cleanup-orphaned" }, { $set: { nextScheduledAt: NOW + 86_400_000, runCount: 3 } }); + + // a restart reconcile re-upserts and pulls it due now + await repo.upsertCleanupJob({ ...job, nextScheduledAt: NOW + 1_000 }); + + const row = await readRow("cleanup-orphaned"); + expect(row?.nextScheduledAt).toBe(NOW + 1_000); // schedule reset ($set) + expect(row?.runCount).toBe(3); // run history untouched ($setOnInsert) + }); + it("setActiveById flips check and geo rows together", async () => { await seedJob({ _id: "check:mon-1", type: "check" }); await seedJob({ _id: "geo:mon-1", type: "geo-check" }); @@ -378,6 +406,20 @@ describe("MongoJobsRepository", () => { expect(await JobModel.countDocuments({ refId: "mon-1" })).toBe(0); }); + it("deleteByMonitorIdsNotIn drops jobs for missing monitors but keeps valid and global rows", async () => { + await seedJob({ _id: "check:keep", refId: "keep", type: "check" }); + await seedJob({ _id: "geo:keep", refId: "keep", type: "geo-check" }); + await seedJob({ _id: "check:gone", refId: "gone", type: "check" }); + await seedJob({ _id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" }); + await seedJob({ _id: "cleanup-retention", refId: null, type: "cleanup-retention" }); + + const deleted = await repo.deleteByMonitorIdsNotIn(["keep"]); + + expect(deleted).toBe(1); // only check:gone + const remaining = (await JobModel.find().lean()).map((d) => d._id).sort(); + expect(remaining).toEqual(["check:keep", "cleanup-orphaned", "cleanup-retention", "geo:keep"]); + }); + it("findById returns all of a monitor's rows", async () => { await seedJob({ _id: "check:mon-1", type: "check" }); await seedJob({ _id: "geo:mon-1", type: "geo-check" }); diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index 8e0724d1f..e24c5790d 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -65,6 +65,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses const jobsRepository = { claimDueBatch: jest.fn().mockResolvedValue([]), upsertJob: jest.fn().mockResolvedValue(true), + upsertCleanupJob: jest.fn().mockResolvedValue(true), upsertEvaluate: jest.fn().mockResolvedValue(true), recordSuccess: jest.fn().mockResolvedValue(true), recordFailure: jest.fn().mockResolvedValue(true), @@ -183,8 +184,9 @@ describe("DBQueueWorker", () => { const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type); expect(seededTypes).toContain("check"); - expect(seededTypes).toContain("cleanup-orphaned"); - expect(seededTypes).toContain("cleanup-retention"); + const cleanupTypes = mocks.jobsRepository.upsertCleanupJob.mock.calls.map((c: any[]) => c[0].type); + expect(cleanupTypes).toContain("cleanup-orphaned"); + expect(cleanupTypes).toContain("cleanup-retention"); }); it("primary mode also seeds a geo-check row for geo-enabled monitors", async () => { @@ -204,6 +206,7 @@ describe("DBQueueWorker", () => { it("worker mode does not reconcile (never seeds jobs)", async () => { const { mocks } = await start({ queueMode: "worker" }); expect(mocks.jobsRepository.upsertJob).not.toHaveBeenCalled(); + expect(mocks.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled(); }); it("starts a polling loop per job type, each claiming up to its concurrency cap", async () => { diff --git a/server/test/unit/services/workerHelper.test.ts b/server/test/unit/services/workerHelper.test.ts index 1c6f28861..27f06ab6c 100644 --- a/server/test/unit/services/workerHelper.test.ts +++ b/server/test/unit/services/workerHelper.test.ts @@ -17,6 +17,9 @@ const createHelper = (overrides?: Record) => { findAllMonitorIds: jest.fn().mockResolvedValue(["m1"]), deleteByTeamIdsNotIn: jest.fn().mockResolvedValue(0), }; + const jobsRepository = { + deleteByMonitorIdsNotIn: jest.fn().mockResolvedValue(0), + }; const teamsRepository = { findAllTeamIds: jest.fn().mockResolvedValue(["team"]), }; @@ -48,6 +51,7 @@ const createHelper = (overrides?: Record) => { checkService, settingsService, monitorsRepository, + jobsRepository, teamsRepository, monitorStatsRepository, checksRepository, @@ -64,6 +68,7 @@ const createHelper = (overrides?: Record) => { defaults.checkService as any, defaults.settingsService as any, defaults.monitorsRepository as any, + defaults.jobsRepository as any, defaults.teamsRepository as any, defaults.monitorStatsRepository as any, defaults.checksRepository as any, @@ -170,6 +175,7 @@ describe("WorkerHelper", () => { (defaults.checksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(4); (defaults.incidentsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(1); (defaults.geoChecksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(5); + (defaults.jobsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(6); const job = helper.getCleanupOrphanedJob(); await job(); @@ -181,11 +187,13 @@ describe("WorkerHelper", () => { expect(defaults.checksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]); expect(defaults.incidentsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]); expect(defaults.geoChecksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]); + expect(defaults.jobsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("2 orphaned monitors") })); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("3 orphaned monitor stats") })); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("4 orphaned checks") })); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 orphaned incidents") })); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("5 orphaned geo checks") })); + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("6 orphaned jobs") })); }); it("skips info logs when no orphaned data is found", async () => { From 64b8aab4d62c299c3d3a849e80b2c12430b83601 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 17 Jun 2026 09:40:51 -0700 Subject: [PATCH 28/75] repair tests --- server/test/helpers/heartbeatTestHarness.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index 88187ac15..baf7eeee3 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -172,6 +172,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { checkService as any, { getDBSettings: jest.fn() } as any, monitorsRepo as any, + { deleteByMonitorIdsNotIn: jest.fn() } as any, { findAllTeamIds: jest.fn() } as any, createStubMonitorStatsRepo() as any, createStubChecksRepo() as any, From f65238da6cefdc48853839af344f5b3061734ed7 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 17 Jun 2026 09:45:59 -0700 Subject: [PATCH 29/75] reformat --- server/src/api/middleware/handleErrors.ts | 3 +-- server/src/api/routes/authRoute.ts | 8 +++++++- server/src/config/envValidation.ts | 2 +- server/test/unit/middleware/upload.test.ts | 5 ++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/server/src/api/middleware/handleErrors.ts b/server/src/api/middleware/handleErrors.ts index 2f817b2af..239349b90 100755 --- a/server/src/api/middleware/handleErrors.ts +++ b/server/src/api/middleware/handleErrors.ts @@ -7,8 +7,7 @@ import { AppError } from "@/utils/AppError.js"; const multerErrorStatus = (error: MulterError): number => (error.code === "LIMIT_FILE_SIZE" ? 413 : 400); const handleErrors = (error: unknown, req: Request, res: Response, _next: NextFunction) => { - const status = - error instanceof MulterError ? multerErrorStatus(error) : error instanceof AppError ? error.status || 500 : 500; + const status = error instanceof MulterError ? multerErrorStatus(error) : error instanceof AppError ? error.status || 500 : 500; const message = error instanceof AppError ? error.message : error instanceof Error ? error.message : "Server error"; const service = error instanceof AppError ? error.service : error instanceof MulterError ? "uploadMiddleware" : "unknownService"; const method = error instanceof AppError ? error.method : "unknownMethod"; diff --git a/server/src/api/routes/authRoute.ts b/server/src/api/routes/authRoute.ts index e8ed31f2e..3b202a502 100644 --- a/server/src/api/routes/authRoute.ts +++ b/server/src/api/routes/authRoute.ts @@ -30,7 +30,13 @@ class AuthRoutes { this.router.patch("/users/:userId/password", verifyJWT, isAllowed(["superadmin"]), this.authController.editUserPasswordById); this.router.delete("/users/:userId", verifyJWT, isAllowed(["admin", "superadmin"]), this.authController.deleteUserById); - this.router.patch("/user", verifyJWT, imageUpload.single("profileImage"), isAllowed(["admin", "superadmin", "user"]), this.authController.editUser); + this.router.patch( + "/user", + verifyJWT, + imageUpload.single("profileImage"), + isAllowed(["admin", "superadmin", "user"]), + this.authController.editUser + ); this.router.delete("/user", verifyJWT, isAllowed(["admin", "superadmin", "user"]), this.authController.deleteUser); } diff --git a/server/src/config/envValidation.ts b/server/src/config/envValidation.ts index 4497119f8..b65ef5165 100644 --- a/server/src/config/envValidation.ts +++ b/server/src/config/envValidation.ts @@ -1,4 +1,4 @@ -import { boolean, z } from "zod"; +import { z } from "zod"; import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js"; import { booleanCoercion } from "@/api/validation/shared.js"; diff --git a/server/test/unit/middleware/upload.test.ts b/server/test/unit/middleware/upload.test.ts index bbdfb205c..dc172d3da 100644 --- a/server/test/unit/middleware/upload.test.ts +++ b/server/test/unit/middleware/upload.test.ts @@ -6,10 +6,9 @@ import { AppError } from "../../../src/utils/AppError.ts"; // Reaches into the multer instance config to exercise the fileFilter/limits in isolation. // Guards against regressing GHSA-9xvg-x28f-m78m (pre-auth DoS via unbounded uploads). -const config = (imageUpload as unknown as { fileFilter: Function; limits: { fileSize: number; files: number } }); +const config = imageUpload as unknown as { fileFilter: Function; limits: { fileSize: number; files: number } }; -const makeFile = (mimetype: string): Express.Multer.File => - ({ fieldname: "profileImage", originalname: "x", mimetype }) as Express.Multer.File; +const makeFile = (mimetype: string): Express.Multer.File => ({ fieldname: "profileImage", originalname: "x", mimetype }) as Express.Multer.File; const runFilter = (mimetype: string): Promise<{ error: unknown; accepted: unknown }> => new Promise((resolve) => { From ad9ff7f5df05613803483ee393ac76577196720e Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 17 Jun 2026 10:55:39 -0700 Subject: [PATCH 30/75] add mongo read/write per second stats --- server/scripts/seed-monitors.js | 2 +- .../domain/diagnostics/diagnostic.service.ts | 72 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/server/scripts/seed-monitors.js b/server/scripts/seed-monitors.js index e8419f73f..f62fe2eb3 100644 --- a/server/scripts/seed-monitors.js +++ b/server/scripts/seed-monitors.js @@ -14,7 +14,7 @@ import { UserModel } from "../dist/domain/users/user.model.js"; // NOTE: the running queue only schedules monitors it finds at init(), so restart // the server after seeding for the new monitors to actually start checking. -const DEFAULT_COUNT = 5_000; +const DEFAULT_COUNT = 500; const DEFAULT_BATCH = 1_000; const DEFAULT_INTERVAL = 60_000; const NAME_PREFIX = "Seed"; diff --git a/server/src/domain/diagnostics/diagnostic.service.ts b/server/src/domain/diagnostics/diagnostic.service.ts index 408b19cb5..f26fa02e3 100644 --- a/server/src/domain/diagnostics/diagnostic.service.ts +++ b/server/src/domain/diagnostics/diagnostic.service.ts @@ -6,6 +6,22 @@ import { AppError } from "@/utils/AppError.js"; const SERVICE_NAME = "diagnosticService"; +export interface MongoStats { + reads: number; + inserts: number; + updates: number; + deletes: number; + timestamp: number; +} + +export interface MongoOpsPerSecond { + readsPerSecond: number; + insertsPerSecond: number; + updatesPerSecond: number; + deletesPerSecond: number; + writesPerSecond: number; +} + export interface CollectionDiagnostics { name: string; documentCount: number; @@ -17,6 +33,11 @@ export interface CollectionDiagnostics { export interface MongoDiagnostics { readyState: number; + readsPerSecond: number; + insertsPerSecond: number; + updatesPerSecond: number; + deletesPerSecond: number; + writesPerSecond: number; host: string; port: number; dbName: string; @@ -44,6 +65,8 @@ export interface IDiagnosticService { export class DiagnosticService implements IDiagnosticService { static SERVICE_NAME = SERVICE_NAME; + private lastMongoOpsPerSecond: MongoStats | null = null; + constructor(private db: IDb) { /** * Performance Observer for monitoring system performance metrics. @@ -68,6 +91,48 @@ export class DiagnosticService implements IDiagnosticService { return ((endUsage.user + endUsage.system) / 1000 / timingPeriod) * 100; }; + private getMongoOpsPerSecond = async (): Promise => { + try { + const mongo = await this.db.getConnection(); + const db = mongo.connection.db; + if (!db) { + throw new AppError({ message: "Database connection is not available", service: SERVICE_NAME, method: "getMongoDBStats" }); + } + const sample = async (): Promise => { + const res = await db.command({ serverStatus: 1 }); + return { + reads: (res.opcounters?.query ?? 0) + (res.opcounters?.getmore ?? 0), + inserts: res.opcounters?.insert ?? 0, + updates: res.opcounters?.update ?? 0, + deletes: res.opcounters?.delete ?? 0, + timestamp: Date.now(), + }; + }; + + // First call has no baseline, seed one so we always have a rate + if (this.lastMongoOpsPerSecond === null) { + this.lastMongoOpsPerSecond = await sample(); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + const previous = this.lastMongoOpsPerSecond; + const current = await sample(); + this.lastMongoOpsPerSecond = current; + + const elapsedSeconds = (current.timestamp - previous.timestamp) / 1000; + return { + readsPerSecond: (current.reads - previous.reads) / elapsedSeconds, + insertsPerSecond: (current.inserts - previous.inserts) / elapsedSeconds, + updatesPerSecond: (current.updates - previous.updates) / elapsedSeconds, + deletesPerSecond: (current.deletes - previous.deletes) / elapsedSeconds, + writesPerSecond: + (current.inserts - previous.inserts + current.updates - previous.updates + current.deletes - previous.deletes) / elapsedSeconds, + }; + } catch { + return null; + } + }; + private getMongoDBStats = async (): Promise => { const mongo = await this.db.getConnection(); const db = mongo.connection.db; @@ -75,6 +140,8 @@ export class DiagnosticService implements IDiagnosticService { throw new AppError({ message: "Database connection is not available", service: SERVICE_NAME, method: "getMongoDBStats" }); } + const opsPerSecond = await this.getMongoOpsPerSecond(); + const dbStats = await db.stats(); const rawCollections = await db.collections(); @@ -100,6 +167,11 @@ export class DiagnosticService implements IDiagnosticService { return { readyState: mongo.connection.readyState, + readsPerSecond: opsPerSecond?.readsPerSecond ?? 0, + insertsPerSecond: opsPerSecond?.insertsPerSecond ?? 0, + updatesPerSecond: opsPerSecond?.updatesPerSecond ?? 0, + deletesPerSecond: opsPerSecond?.deletesPerSecond ?? 0, + writesPerSecond: opsPerSecond?.writesPerSecond ?? 0, host: mongo.connection.host, port: mongo.connection.port, dbName: mongo.connection.name, From bffc4ee813bd9a11ef4c2c05d7fab2c6e6456cc0 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 17 Jun 2026 10:55:49 -0700 Subject: [PATCH 31/75] add stats to fe --- .../src/Pages/Logs/components/MongoStats.tsx | 20 +++++++++++++++++++ client/src/Types/Diagnostics.ts | 5 +++++ client/src/locales/en.json | 5 +++++ 3 files changed, 30 insertions(+) diff --git a/client/src/Pages/Logs/components/MongoStats.tsx b/client/src/Pages/Logs/components/MongoStats.tsx index ed7b9cc53..8dc2f0231 100644 --- a/client/src/Pages/Logs/components/MongoStats.tsx +++ b/client/src/Pages/Logs/components/MongoStats.tsx @@ -33,6 +33,26 @@ export const MongoStats = ({ diagnostics }: MongoStatsProps) => { title={t("pages.logs.diagnostics.mongoDBStats.readyState")} subtitle={mongoStats.readyState.toString()} /> + + + + + Date: Wed, 17 Jun 2026 11:01:56 -0700 Subject: [PATCH 32/75] update tests --- .../unit/services/diagnosticService.test.ts | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/server/test/unit/services/diagnosticService.test.ts b/server/test/unit/services/diagnosticService.test.ts index b44de1986..07962a68e 100644 --- a/server/test/unit/services/diagnosticService.test.ts +++ b/server/test/unit/services/diagnosticService.test.ts @@ -104,9 +104,8 @@ describe("DiagnosticService", () => { it("returns only the slimmed diagnostic shape (no serverStatus/serverInfo/memoryUsage)", async () => { const service = new DiagnosticService(makeDb()); const promise = service.getSystemStats(); - // getCPUUsagePercentage waits 1s, then event loop delay waits 0ms - jest.advanceTimersByTime(1000); - await jest.advanceTimersByTimeAsync(10); + // getCPUUsagePercentage waits 1s, the first mongo ops sample waits another 1s, then event loop delay waits 0ms + await jest.advanceTimersByTimeAsync(2500); const result = await promise; expect(Object.keys(result).sort()).toEqual(["cpuUsage", "eventLoopDelayMs", "mongoStats", "osStats", "uptimeMs", "v8HeapStats"].sort()); @@ -131,11 +130,24 @@ describe("DiagnosticService", () => { }); const service = new DiagnosticService(db); const promise = service.getSystemStats(); - jest.advanceTimersByTime(1000); - await jest.advanceTimersByTimeAsync(10); + await jest.advanceTimersByTimeAsync(2500); const result = await promise; - expect(Object.keys(result.mongoStats).sort()).toEqual(["collections", "dbName", "host", "port", "readyState", "totalSize"].sort()); + expect(Object.keys(result.mongoStats).sort()).toEqual( + [ + "collections", + "dbName", + "host", + "port", + "readyState", + "totalSize", + "readsPerSecond", + "insertsPerSecond", + "updatesPerSecond", + "deletesPerSecond", + "writesPerSecond", + ].sort() + ); expect(result.mongoStats.totalSize).toBe(9999); expect(result.mongoStats.collections).toHaveLength(2); expect(result.mongoStats.collections[0]).toEqual({ @@ -154,8 +166,7 @@ describe("DiagnosticService", () => { }); const service = new DiagnosticService(db); const promise = service.getSystemStats(); - jest.advanceTimersByTime(1000); - await jest.advanceTimersByTimeAsync(10); + await jest.advanceTimersByTimeAsync(2500); const result = await promise; expect(result.mongoStats.totalSize).toBe(376832 + 1155072); @@ -169,8 +180,7 @@ describe("DiagnosticService", () => { }); const service = new DiagnosticService(db); const promise = service.getSystemStats(); - jest.advanceTimersByTime(1000); - await jest.advanceTimersByTimeAsync(10); + await jest.advanceTimersByTimeAsync(2500); const result = await promise; const coll = result.mongoStats.collections[0]; @@ -190,8 +200,7 @@ describe("DiagnosticService", () => { }); const service = new DiagnosticService(db); const promise = service.getSystemStats(); - jest.advanceTimersByTime(1000); - await jest.advanceTimersByTimeAsync(10); + await jest.advanceTimersByTimeAsync(2500); const result = await promise; expect(result.mongoStats.collections[0]?.bucketCount).toBe(12); @@ -199,8 +208,7 @@ describe("DiagnosticService", () => { const db2 = makeDb({ collections: [{ collectionName: "monitors" }] }); const service2 = new DiagnosticService(db2); const promise2 = service2.getSystemStats(); - jest.advanceTimersByTime(1000); - await jest.advanceTimersByTimeAsync(10); + await jest.advanceTimersByTimeAsync(2500); const result2 = await promise2; expect(result2.mongoStats.collections[0]).not.toHaveProperty("bucketCount"); }); @@ -235,8 +243,7 @@ describe("DiagnosticService", () => { performance.getEntriesByName = jest.fn().mockReturnValue(entries) as any; const promise = service.getSystemStats(); - jest.advanceTimersByTime(1000); - await jest.advanceTimersByTimeAsync(10); + await jest.advanceTimersByTimeAsync(2500); const result = await promise; expect(result.eventLoopDelayMs).toBe(0); From 380fdf470f2e8958606540b09da8d16a289d7d3b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 17 Jun 2026 11:35:15 -0700 Subject: [PATCH 33/75] backoff excessive polling --- server/src/worker/worker.db-queue.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index f44d91c98..2b42b8ca2 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -15,7 +15,8 @@ import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.rep import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; const SERVICE_NAME = "JobQueue"; -const POLL_MS = 250; +const POLL_MS = 250; // base poll interval while a loop is actively claiming work +const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals @@ -240,6 +241,7 @@ export class DBQueueWorker implements IWorker { }; private startLoop = (type: JobType) => { + let pollMs = POLL_MS; const tick = async () => { if (this.stopped) return; try { @@ -252,6 +254,8 @@ export class DBQueueWorker implements IWorker { this.inFlight[type]--; // job done, free the slot }); } + // Back off if no jobs founds, reset to base if jobs found + pollMs = capacity > 0 && jobs.length === 0 ? Math.min(pollMs * 2, POLL_MAX_MS) : POLL_MS; } catch (error: unknown) { this.logger.error({ message: error instanceof Error ? error.message : String(error), @@ -259,7 +263,7 @@ export class DBQueueWorker implements IWorker { method: `loop:${type}`, }); } - if (!this.stopped) this.timers.set(type, setTimeout(tick, POLL_MS)); // re-arm unless shutting down + if (!this.stopped) this.timers.set(type, setTimeout(tick, pollMs)); // re-arm unless shutting down }; tick(); }; From 158d976ccb23299ec0b7ba7f343850899966a1bc Mon Sep 17 00:00:00 2001 From: harsh-aghara Date: Tue, 9 Jun 2026 20:39:34 +0530 Subject: [PATCH 34/75] feat(client): add HTTP status codes constants for monitor options --- client/src/Utils/statusCode.ts | 80 ++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/client/src/Utils/statusCode.ts b/client/src/Utils/statusCode.ts index 33a5f3de1..73ee2a892 100644 --- a/client/src/Utils/statusCode.ts +++ b/client/src/Utils/statusCode.ts @@ -74,3 +74,83 @@ export const getStatusCodeValueType = (code: number): ValueType => { if (code < 400) return "neutral"; return "negative"; }; + +export interface HttpStatusCodeOption { + id: number; + name: string; +} + +export const ALL_HTTP_STATUS_CODES: HttpStatusCodeOption[] = [ + { id: 100, name: "100 - Continue" }, + { id: 101, name: "101 - Switching Protocols" }, + { id: 102, name: "102 - Processing" }, + { id: 103, name: "103 - Early Hints" }, + { id: 200, name: "200 - OK" }, + { id: 201, name: "201 - Created" }, + { id: 202, name: "202 - Accepted" }, + { id: 203, name: "203 - Non-Authoritative Information" }, + { id: 204, name: "204 - No Content" }, + { id: 205, name: "205 - Reset Content" }, + { id: 206, name: "206 - Partial Content" }, + { id: 207, name: "207 - Multi-Status" }, + { id: 208, name: "208 - Already Reported" }, + { id: 226, name: "226 - IM Used" }, + { id: 300, name: "300 - Multiple Choices" }, + { id: 301, name: "301 - Moved Permanently" }, + { id: 302, name: "302 - Found" }, + { id: 303, name: "303 - See Other" }, + { id: 304, name: "304 - Not Modified" }, + { id: 305, name: "305 - Use Proxy" }, + { id: 307, name: "307 - Temporary Redirect" }, + { id: 308, name: "308 - Permanent Redirect" }, + { id: 400, name: "400 - Bad Request" }, + { id: 401, name: "401 - Unauthorized" }, + { id: 402, name: "402 - Payment Required" }, + { id: 403, name: "403 - Forbidden" }, + { id: 404, name: "404 - Not Found" }, + { id: 405, name: "405 - Method Not Allowed" }, + { id: 406, name: "406 - Not Acceptable" }, + { id: 407, name: "407 - Proxy Authentication Required" }, + { id: 408, name: "408 - Request Timeout" }, + { id: 409, name: "409 - Conflict" }, + { id: 410, name: "410 - Gone" }, + { id: 411, name: "411 - Length Required" }, + { id: 412, name: "412 - Precondition Failed" }, + { id: 413, name: "413 - Payload Too Large" }, + { id: 414, name: "414 - URI Too Long" }, + { id: 415, name: "415 - Unsupported Media Type" }, + { id: 416, name: "416 - Range Not Satisfiable" }, + { id: 417, name: "417 - Expectation Failed" }, + { id: 418, name: "418 - I'm a Teapot" }, + { id: 419, name: "419 - Page Expired (Laravel)" }, + { id: 421, name: "421 - Misdirected Request" }, + { id: 422, name: "422 - Unprocessable Entity" }, + { id: 423, name: "423 - Locked" }, + { id: 424, name: "424 - Failed Dependency" }, + { id: 425, name: "425 - Too Early" }, + { id: 426, name: "426 - Upgrade Required" }, + { id: 428, name: "428 - Precondition Required" }, + { id: 429, name: "429 - Too Many Requests" }, + { id: 431, name: "431 - Request Header Fields Too Large" }, + { id: 451, name: "451 - Unavailable For Legal Reasons" }, + { id: 499, name: "499 - Client Closed Request (NGINX)" }, + { id: 500, name: "500 - Internal Server Error" }, + { id: 501, name: "501 - Not Implemented" }, + { id: 502, name: "502 - Bad Gateway" }, + { id: 503, name: "503 - Service Unavailable" }, + { id: 504, name: "504 - Gateway Timeout" }, + { id: 505, name: "505 - HTTP Version Not Supported" }, + { id: 506, name: "506 - Variant Also Negotiates" }, + { id: 507, name: "507 - Insufficient Storage" }, + { id: 508, name: "508 - Loop Detected" }, + { id: 510, name: "510 - Not Extended" }, + { id: 511, name: "511 - Network Authentication Required" }, + { id: 520, name: "520 - Web Server Returned an Unknown Error (Cloudflare)" }, + { id: 521, name: "521 - Web Server Is Down (Cloudflare)" }, + { id: 522, name: "522 - Connection Timed Out (Cloudflare)" }, + { id: 523, name: "523 - Origin Is Unreachable (Cloudflare)" }, + { id: 524, name: "524 - A Timeout Occurred (Cloudflare)" }, + { id: 525, name: "525 - SSL Handshake Failed (Cloudflare)" }, + { id: 526, name: "526 - Invalid SSL Certificate (Cloudflare)" }, + { id: 527, name: "527 - Railgun Error (Cloudflare)" }, +]; From 603bd1d814a982a5cc02c028d012dc992b6ef86e Mon Sep 17 00:00:00 2001 From: harsh-aghara Date: Tue, 9 Jun 2026 20:40:06 +0530 Subject: [PATCH 35/75] feat(client): add customUpCodes to Monitor type and validation schema --- client/src/Types/Monitor.ts | 1 + client/src/Validation/monitor.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index 1e3dafb99..b2d90cb0b 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -71,6 +71,7 @@ export interface Monitor { uptimePercentage?: number; notifications: string[]; tags: string[]; + customUpCodes?: number[]; secret?: string; cpuAlertThreshold: number; cpuAlertCounter: number; diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 80d0d77c4..a6006d543 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -64,6 +64,10 @@ const httpSchema = baseSchema.extend({ .register(monitorStepRegistry, { step: 2 }), expectedValue: z.string().optional().register(monitorStepRegistry, { step: 2 }), jsonPath: z.string().optional().register(monitorStepRegistry, { step: 2 }), + customUpCodes: z + .array(z.number().min(100).max(599)) + .optional() + .register(monitorStepRegistry, { step: 2 }), ...geoCheckFields, }); From 1b1d8ea229cc05ed666ab8334cef654c7be3a89f Mon Sep 17 00:00:00 2001 From: harsh-aghara Date: Tue, 9 Jun 2026 20:40:29 +0530 Subject: [PATCH 36/75] feat(client): implement accepted status codes UI in create monitor form --- client/src/Hooks/useMonitorForm.ts | 2 + client/src/Pages/CreateMonitor/index.tsx | 63 ++++++++++++++++++++++++ client/src/locales/en.json | 4 ++ 3 files changed, 69 insertions(+) diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 50fa62ec3..0ae463911 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -45,6 +45,7 @@ export const getMonitorDefaults = ( matchMethod: data?.matchMethod || "", expectedValue: data?.expectedValue || "", jsonPath: data?.jsonPath || "", + customUpCodes: data?.customUpCodes || [], }; break; case "ping": @@ -138,6 +139,7 @@ export const getMonitorDefaults = ( matchMethod: "", expectedValue: "", jsonPath: "", + customUpCodes: [], }; } diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 94bed9355..2bda92e62 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -1,6 +1,7 @@ import { useMemo, useState } from "react"; import { useEffect } from "react"; import { logger } from "@/Utils/logger"; +import { ALL_HTTP_STATUS_CODES } from "@/Utils/statusCode"; import { useParams, useLocation, useNavigate } from "react-router"; import { useForm, Controller } from "react-hook-form"; import { zodResolver } from "@hookform/resolvers/zod"; @@ -1090,6 +1091,68 @@ const CreateMonitorPage = () => { subtitle={t("pages.createMonitor.form.advanced.description")} rightContent={ + { + const selectedOptions = ALL_HTTP_STATUS_CODES.filter((option) => + field.value?.includes(option.id) + ); + return ( + + option.name} + isOptionEqualToValue={(option, value) => option.id === value.id} + onChange={( + _: unknown, + newValue: typeof ALL_HTTP_STATUS_CODES + ) => { + field.onChange(newValue?.map((v) => v.id) || []); + }} + fieldLabel={t( + "pages.createMonitor.form.advanced.option.customUpCodes.label" + )} + /> + {selectedOptions.length > 0 && ( + + {selectedOptions.map((option, index) => ( + + {option.name} + { + field.onChange( + field.value?.filter((id) => id !== option.id) + ); + }} + aria-label={t( + "pages.createMonitor.form.advanced.option.customUpCodes.removeAriaLabel" + )} + > + + + {index < selectedOptions.length - 1 && } + + ))} + + )} + + ); + }} + /> Date: Thu, 18 Jun 2026 09:56:22 -0700 Subject: [PATCH 37/75] inject job repo into buffer service --- server/src/config/services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index a60505c25..ebc17eaa0 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -227,7 +227,7 @@ export const initializeServices = async ({ monitorsRepository, }); - const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService); + const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository); const statusService = new StatusService(logger, bufferService, monitorsRepository, monitorStatsRepository, checksRepository); From 1ea6dafb1a88f5f9b43e46c2d2cb67544513799e Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 09:56:38 -0700 Subject: [PATCH 38/75] evaluate checks on buffer flush --- server/src/service/bufferService.ts | 39 ++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/server/src/service/bufferService.ts b/server/src/service/bufferService.ts index 38dfb3545..989ebf936 100755 --- a/server/src/service/bufferService.ts +++ b/server/src/service/bufferService.ts @@ -3,6 +3,7 @@ import type { GeoCheck } from "@/domain/geo-checks/geo-check.type.js"; import type { IGeoChecksService } from "../domain/geo-checks/geo-check.service.js"; import type { ILogger } from "@/utils/logger.js"; import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; +import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { ICheckService } from "../domain/checks/check.service.js"; const SERVICE_NAME = "BufferService"; @@ -24,12 +25,20 @@ export class BufferService implements IBufferService { private bufferTimer: NodeJS.Timeout | null = null; private checksService: ICheckService; private geoChecksService: IGeoChecksService; + private jobsRepository: IJobsRepository; - constructor(logger: ILogger, checkService: ICheckService, geoChecksService: IGeoChecksService, settingsService: ISettingsService) { + constructor( + logger: ILogger, + checkService: ICheckService, + geoChecksService: IGeoChecksService, + settingsService: ISettingsService, + jobsRepository: IJobsRepository + ) { this.BUFFER_TIMEOUT = settingsService.getSettings().nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute this.logger = logger; this.checksService = checkService; this.geoChecksService = geoChecksService; + this.jobsRepository = jobsRepository; this.SERVICE_NAME = SERVICE_NAME; this.buffer = []; this.geoBuffer = []; @@ -93,16 +102,23 @@ export class BufferService implements IBufferService { }, this.BUFFER_TIMEOUT); } async flushBuffer() { + if (this.buffer.length === 0) { + return; + } + // Take the batch first so a write that drains it can't be appended to mid-flush + const batch = this.buffer; + this.buffer = []; try { - if (this.buffer.length > 0) { - this.logger.debug({ - message: `Flushing ${this.buffer.length} checks to database`, - service: this.SERVICE_NAME, - method: "flushBuffer", - }); - await this.checksService.createChecks(this.buffer); - this.buffer = []; - } + this.logger.debug({ + message: `Flushing ${batch.length} checks to database`, + service: this.SERVICE_NAME, + method: "flushBuffer", + }); + await this.checksService.createChecks(batch); + // Need to evaluate checks when they are flushed + const monitorIds = [...new Set(batch.map((check) => check.metadata.monitorId))]; + const now = Date.now(); + await Promise.all(monitorIds.map((monitorId) => this.jobsRepository.upsertEvaluate(monitorId, now))); } catch (error: unknown) { this.logger.error({ message: error instanceof Error ? error.message : "Unknown error", @@ -110,8 +126,7 @@ export class BufferService implements IBufferService { method: "flushBuffer", stack: error instanceof Error ? error.stack : undefined, }); - // Clear buffer even on error to prevent infinite retry loops - this.buffer = []; + // Batch is already cleared from the buffer; dropping it prevents infinite retry loops } } From 53ee45172dccd6e635332e226f71b539b2fb9922 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 09:56:56 -0700 Subject: [PATCH 39/75] remove evluation from worker --- server/src/worker/worker.db-queue.ts | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index f44d91c98..9f1a7bcf5 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -97,18 +97,18 @@ export class DBQueueWorker implements IWorker { return instance; } - // Helpers, builds job seeds - private toCheckJob = (monitor: Monitor, now: number): JobSeed => ({ + // Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd. + private toCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ id: jobId("check", monitor.id), type: "check", refId: monitor.id, isActive: monitor.isActive, - nextScheduledAt: now + Math.floor(Math.random() * monitor.interval), // Avoid herd + nextScheduledAt: immediate ? now : now + Math.floor(Math.random() * monitor.interval), // Avoid herd intervalMs: monitor.interval, }); - private toGeoCheckJob = (monitor: Monitor, now: number): JobSeed => ({ - ...this.toCheckJob(monitor, now), + private toGeoCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ + ...this.toCheckJob(monitor, now, immediate), id: jobId("geo-check", monitor.id), type: "geo-check", intervalMs: monitor.geoCheckInterval!, @@ -152,10 +152,7 @@ export class DBQueueWorker implements IWorker { if (!job.refId) return; const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId if (!monitor) return; - const produced = await this.checkProducer.produce(monitor); - if (produced) { - await this.jobsRepository.upsertEvaluate(monitor.id, Date.now()); - } + await this.checkProducer.produce(monitor); }; // ******************** @@ -181,7 +178,6 @@ export class DBQueueWorker implements IWorker { }; // Extend the lock while a job runs so a slow-but-alive job is never reclaimed. - // Returns false once we no longer hold the lock, so the renewal timer can stop. private renewLock = async (job: Job): Promise => { try { const held = await this.jobsRepository.renewLocks([job.id], Date.now()); @@ -191,7 +187,6 @@ export class DBQueueWorker implements IWorker { } return true; } catch (error: unknown) { - // Transient failure, keep renewing; a missed renewal is absorbed by the 1/3 margin this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "renewLock" }); return true; } @@ -312,8 +307,8 @@ export class DBQueueWorker implements IWorker { addJob = async (_monitorId: string, monitor: Monitor) => { const now = Date.now(); - await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now)); - if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now)); + await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true)); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); }; deleteJob = async (monitor: Monitor) => { await this.jobsRepository.deleteById(monitor.id); @@ -327,7 +322,7 @@ export class DBQueueWorker implements IWorker { updateJob = async (monitor: Monitor) => { await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval); if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { - await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now())); + await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away } else { await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check"); } From 398e5d7314d7e88f1ee1192bebc8f2b25375312c Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 09:57:05 -0700 Subject: [PATCH 40/75] add tests --- .../test/unit/services/bufferService.test.ts | 34 +++++++++++++++++-- .../test/unit/services/dbQueueWorker.test.ts | 14 +++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/server/test/unit/services/bufferService.test.ts b/server/test/unit/services/bufferService.test.ts index 5a6418c70..7c344c086 100644 --- a/server/test/unit/services/bufferService.test.ts +++ b/server/test/unit/services/bufferService.test.ts @@ -4,6 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; import type { ICheckService } from "../../../src/domain/checks/check.service.ts"; import type { IGeoChecksService } from "../../../src/domain/geo-checks/geo-check.service.ts"; import type { ISettingsService } from "../../../src/domain/app-settings/app-settings.service.ts"; +import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts"; import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { GeoCheck } from "../../../src/domain/geo-checks/geo-check.type.ts"; @@ -24,6 +25,11 @@ const createMockSettingsService = (nodeEnv: string = "development") => getSettings: jest.fn().mockReturnValue({ nodeEnv }), }) as unknown as jest.Mocked; +const createMockJobsRepository = () => + ({ + upsertEvaluate: jest.fn().mockResolvedValue(true), + }) as unknown as jest.Mocked; + const makeCheck = (overrides?: Partial): Check => ({ id: "check-1", @@ -47,8 +53,9 @@ const createService = (nodeEnv: string = "development") => { const checkService = createMockCheckService(); const geoChecksService = createMockGeoChecksService(); const settingsService = createMockSettingsService(nodeEnv); - const service = new BufferService(logger as any, checkService, geoChecksService, settingsService); - return { service, logger, checkService, geoChecksService }; + const jobsRepository = createMockJobsRepository(); + const service = new BufferService(logger as any, checkService, geoChecksService, settingsService, jobsRepository); + return { service, logger, checkService, geoChecksService, jobsRepository }; }; // ── Tests ──────────────────────────────────────────────────────────────────── @@ -344,6 +351,29 @@ describe("BufferService", () => { ); }); + it("arms the evaluate stage once per distinct monitor after the checks are stored", async () => { + const { service, jobsRepository } = createService(); + service.addToBuffer(makeCheck({ id: "c1", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } })); + service.addToBuffer(makeCheck({ id: "c2", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } })); + service.addToBuffer(makeCheck({ id: "c3", metadata: { monitorId: "mon-2", teamId: "team-1", type: "http" } })); + + await service.flushBuffer(); + + expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2); + expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-1", expect.any(Number)); + expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", expect.any(Number)); + }); + + it("does not arm the evaluate stage when the check write fails", async () => { + const { service, checkService, jobsRepository } = createService(); + (checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed")); + service.addToBuffer(makeCheck()); + + await service.flushBuffer(); + + expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); + }); + it("clears buffer even on error to prevent infinite retries", async () => { const { service, checkService, logger } = createService(); (checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed")); diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index e24c5790d..e60f82a95 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -270,7 +270,8 @@ describe("DBQueueWorker", () => { expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]); expect(mocks.checkProducer.produce).toHaveBeenCalled(); - expect(mocks.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("m1", expect.any(Number)); + // The check stage only produces; the buffer arms evaluate once the check is durably stored + expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled(); }); @@ -394,6 +395,17 @@ describe("DBQueueWorker", () => { expect(types).toContain("geo-check"); }); + it("addJob schedules new jobs to run immediately (no startup jitter)", async () => { + const { worker, mocks } = createWorker(); + active = worker; + const now = Date.now(); + await worker.addJob("m1", makeMonitor({ geoCheckEnabled: true })); + + for (const [seed] of mocks.jobsRepository.upsertJob.mock.calls as any[][]) { + expect(seed.nextScheduledAt).toBe(now); + } + }); + it("addJob seeds only a check row for non-geo monitors", async () => { const { worker, mocks } = createWorker(); active = worker; From d8dafa311a03d917775e621d03a72f4329a09610 Mon Sep 17 00:00:00 2001 From: harsh-aghara Date: Tue, 9 Jun 2026 23:45:13 +0530 Subject: [PATCH 41/75] feat: sync custom http status codes between frontend and backend - Add standard custom HTTP status codes (Cloudflare, AWS ELB, etc.) to both frontend and backend allowlists. - Enforce strict validation of custom status codes using Zod sets on both sides, ensuring they match perfectly. --- client/src/Utils/statusCode.ts | 10 +++++++++ client/src/Validation/monitor.ts | 8 ++++++- server/src/domain/monitors/monitor.types.ts | 24 ++++++++++++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/client/src/Utils/statusCode.ts b/client/src/Utils/statusCode.ts index 73ee2a892..7b5448d2e 100644 --- a/client/src/Utils/statusCode.ts +++ b/client/src/Utils/statusCode.ts @@ -123,6 +123,7 @@ export const ALL_HTTP_STATUS_CODES: HttpStatusCodeOption[] = [ { id: 417, name: "417 - Expectation Failed" }, { id: 418, name: "418 - I'm a Teapot" }, { id: 419, name: "419 - Page Expired (Laravel)" }, + { id: 420, name: "420 - Enhance Your Calm (Twitter)" }, { id: 421, name: "421 - Misdirected Request" }, { id: 422, name: "422 - Unprocessable Entity" }, { id: 423, name: "423 - Locked" }, @@ -132,7 +133,12 @@ export const ALL_HTTP_STATUS_CODES: HttpStatusCodeOption[] = [ { id: 428, name: "428 - Precondition Required" }, { id: 429, name: "429 - Too Many Requests" }, { id: 431, name: "431 - Request Header Fields Too Large" }, + { id: 440, name: "440 - Login Time-out (IIS)" }, + { id: 449, name: "449 - Retry With (IIS)" }, { id: 451, name: "451 - Unavailable For Legal Reasons" }, + { id: 460, name: "460 - Client Closed Connection (AWS ELB)" }, + { id: 463, name: "463 - X-Forwarded-For Too Large (AWS ELB)" }, + { id: 497, name: "497 - HTTP Request Sent to HTTPS Port (NGINX)" }, { id: 499, name: "499 - Client Closed Request (NGINX)" }, { id: 500, name: "500 - Internal Server Error" }, { id: 501, name: "501 - Not Implemented" }, @@ -143,6 +149,7 @@ export const ALL_HTTP_STATUS_CODES: HttpStatusCodeOption[] = [ { id: 506, name: "506 - Variant Also Negotiates" }, { id: 507, name: "507 - Insufficient Storage" }, { id: 508, name: "508 - Loop Detected" }, + { id: 509, name: "509 - Bandwidth Limit Exceeded (Apache)" }, { id: 510, name: "510 - Not Extended" }, { id: 511, name: "511 - Network Authentication Required" }, { id: 520, name: "520 - Web Server Returned an Unknown Error (Cloudflare)" }, @@ -153,4 +160,7 @@ export const ALL_HTTP_STATUS_CODES: HttpStatusCodeOption[] = [ { id: 525, name: "525 - SSL Handshake Failed (Cloudflare)" }, { id: 526, name: "526 - Invalid SSL Certificate (Cloudflare)" }, { id: 527, name: "527 - Railgun Error (Cloudflare)" }, + { id: 529, name: "529 - Site is overloaded" }, + { id: 530, name: "530 - Site is frozen (Cloudflare)" }, + { id: 561, name: "561 - Unauthorized (AWS ELB)" }, ]; diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index a6006d543..e0c4eae07 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import type { FieldPath } from "react-hook-form"; import { GeoContinents } from "@/Types/GeoCheck"; import { DnsRecordTypes, PageSpeedStrategies, type MonitorType } from "@/Types/Monitor"; +import { ALL_HTTP_STATUS_CODES } from "@/Utils/statusCode"; // Wizard step a field is validated on. Attached inline to each field below so // the grouping lives next to the field definition; unannotated fields default @@ -53,6 +54,11 @@ const geoCheckFields = { }; // HTTP monitor schema +const httpStatusCodeSet = new Set(ALL_HTTP_STATUS_CODES.map((c) => c.id)); +const httpStatusCode = z.number().refine((code) => httpStatusCodeSet.has(code), { + message: "Must be a valid HTTP status code", +}); + const httpSchema = baseSchema.extend({ type: z.literal("http"), url: urlSchema, @@ -65,7 +71,7 @@ const httpSchema = baseSchema.extend({ expectedValue: z.string().optional().register(monitorStepRegistry, { step: 2 }), jsonPath: z.string().optional().register(monitorStepRegistry, { step: 2 }), customUpCodes: z - .array(z.number().min(100).max(599)) + .array(httpStatusCode) .optional() .register(monitorStepRegistry, { step: 2 }), ...geoCheckFields, diff --git a/server/src/domain/monitors/monitor.types.ts b/server/src/domain/monitors/monitor.types.ts index 37d477704..b614d3191 100644 --- a/server/src/domain/monitors/monitor.types.ts +++ b/server/src/domain/monitors/monitor.types.ts @@ -4,7 +4,29 @@ import type { GeoContinent, GroupedGeoCheck } from "@/domain/geo-checks/geo-chec export type { GeoContinent } from "@/domain/geo-checks/geo-check.type.js"; import http from "node:http"; -export const HttpStatusCodes = Object.keys(http.STATUS_CODES).map(Number); +export const HttpStatusCodes = [ + ...Object.keys(http.STATUS_CODES).map(Number), + 419, // Page Expired (Laravel) + 420, // Enhance Your Calm (Twitter) + 440, // Login Time-out (IIS) + 449, // Retry With (IIS) + 460, // Client Closed Connection (AWS ELB) + 463, // X-Forwarded-For Too Large (AWS ELB) + 497, // HTTP Request Sent to HTTPS Port (NGINX) + 499, // Client Closed Request (NGINX) + 509, // Bandwidth Limit Exceeded (Apache) + 520, // Web Server Returned an Unknown Error (Cloudflare) + 521, // Web Server Is Down (Cloudflare) + 522, // Connection Timed Out (Cloudflare) + 523, // Origin Is Unreachable (Cloudflare) + 524, // A Timeout Occurred (Cloudflare) + 525, // SSL Handshake Failed (Cloudflare) + 526, // Invalid SSL Certificate (Cloudflare) + 527, // Railgun Error (Cloudflare) + 529, // Site is overloaded + 530, // Site is frozen (Cloudflare) + 561, // Unauthorized (AWS ELB) +]; export const HttpStatusCodeSet = new Set(HttpStatusCodes); export type HttpStatusCode = number; From 6d433fd0ac34349081656775ea8229ba0b9a70ce Mon Sep 17 00:00:00 2001 From: harsh-aghara Date: Thu, 18 Jun 2026 21:37:02 +0530 Subject: [PATCH 42/75] feat(client): polish customUpCodes ui --- client/src/Pages/CreateMonitor/index.tsx | 9 +++++++++ client/src/Utils/statusCode.ts | 3 +++ client/src/locales/en.json | 1 + 3 files changed, 13 insertions(+) diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 2bda92e62..465b9d15d 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -1117,6 +1117,15 @@ const CreateMonitorPage = () => { "pages.createMonitor.form.advanced.option.customUpCodes.label" )} /> + + {t( + "pages.createMonitor.form.advanced.option.customUpCodes.description" + )} + {selectedOptions.length > 0 && ( Date: Thu, 18 Jun 2026 10:17:33 -0700 Subject: [PATCH 43/75] rearm jobs on maintenance window pause/delete --- server/src/config/services.ts | 1 + .../domain/jobs/job.repository.interface.ts | 3 ++ .../src/domain/jobs/job.repository.mongo.ts | 18 +++++++++ .../maintenance-window.service.ts | 7 ++++ .../test/integration/jobsRepository.test.ts | 38 +++++++++++++++++++ .../services/maintenanceWindowService.test.ts | 16 ++++++-- 6 files changed, 80 insertions(+), 3 deletions(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index ebc17eaa0..346f77209 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -338,6 +338,7 @@ export const initializeServices = async ({ const maintenanceWindowService = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, + jobsRepository, }); const monitorService = new MonitorService({ worker, diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index 3ec95d4c2..190323eed 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -52,6 +52,9 @@ export interface IJobsRepository { // Edit job schedule updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise; + // Re-arm the check/geo-check rows. Add jitter to prevent herd + markMonitorsDue(monitorIds: string[], now: number): Promise; + // Delete, drop all rows deleteById(refId: string): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index e2dfd2e2a..1dd912d10 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -2,6 +2,9 @@ import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.inte import JobModel, { JobDocument } from "@/domain/jobs/job.model.js"; import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js"; +// Cap for rearm interval, never want to wait more than 15s. +const REARM_JITTER_MAX_MS = 15_000; + class MongoJobsRepository implements IJobsRepository { constructor(private readonly workerId: string) {} @@ -249,6 +252,21 @@ class MongoJobsRepository implements IJobsRepository { return res.modifiedCount === 1; }; + markMonitorsDue = async (monitorIds: string[], now: number) => { + if (monitorIds.length === 0) return 0; + const res = await JobModel.updateMany({ refId: { $in: monitorIds }, type: { $in: ["check", "geo-check"] } }, [ + { + // Spread rearmed jobs, but never for more than rearm jitter max + $set: { + nextScheduledAt: { + $add: [now, { $floor: { $multiply: [{ $rand: {} }, { $min: ["$intervalMs", REARM_JITTER_MAX_MS] }] } }], + }, + }, + }, + ]); + return res.modifiedCount; + }; + deleteById = async (refId: string) => { const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor return res.deletedCount > 0; diff --git a/server/src/domain/maintenance-windows/maintenance-window.service.ts b/server/src/domain/maintenance-windows/maintenance-window.service.ts index e1158dec9..aef0f9fb4 100644 --- a/server/src/domain/maintenance-windows/maintenance-window.service.ts +++ b/server/src/domain/maintenance-windows/maintenance-window.service.ts @@ -1,5 +1,6 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js"; import { AppError } from "@/utils/AppError.js"; import { isWindowActive } from "@/utils/maintenanceWindow.js"; @@ -40,16 +41,20 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { static SERVICE_NAME = SERVICE_NAME; private monitorsRepository: IMonitorsRepository; private maintenanceWindowsRepository: IMaintenanceWindowsRepository; + private jobsRepository: IJobsRepository; constructor({ monitorsRepository, maintenanceWindowsRepository, + jobsRepository, }: { monitorsRepository: IMonitorsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; + jobsRepository: IJobsRepository; }) { this.monitorsRepository = monitorsRepository; this.maintenanceWindowsRepository = maintenanceWindowsRepository; + this.jobsRepository = jobsRepository; } get serviceName() { @@ -71,6 +76,8 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { const toInitializing = monitorIds.filter((monitorId) => !stillCovered.has(monitorId)); if (toInitializing.length > 0) { await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]); + // Rearm jobs to run soon (now + jitter to avoid herding) + await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime()); } }; diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index 9c2f54f94..41781114a 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -396,6 +396,44 @@ describe("MongoJobsRepository", () => { expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched }); + it("markMonitorsDue re-arms check and geo rows, capping the jitter for long intervals", async () => { + const armAt = NOW + 5_000_000; // well past the seeded nextScheduledAt so the jittered value is unambiguous + await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", intervalMs: 10_000, nextScheduledAt: NOW }); // shorter than the cap + await seedJob({ _id: "geo:mon-1", refId: "mon-1", type: "geo-check", intervalMs: 3_600_000, nextScheduledAt: NOW }); // far longer than the cap + + const modified = await repo.markMonitorsDue(["mon-1"], armAt); + + expect(modified).toBe(2); + const check = await readRow("check:mon-1"); + const geo = await readRow("geo:mon-1"); + // Short interval spreads over its own interval; long interval is capped so it still runs soon + expect(check!.nextScheduledAt).toBeGreaterThanOrEqual(armAt); + expect(check!.nextScheduledAt).toBeLessThan(armAt + 10_000); + expect(geo!.nextScheduledAt).toBeGreaterThanOrEqual(armAt); + expect(geo!.nextScheduledAt).toBeLessThan(armAt + 15_000); // REARM_JITTER_MS, not the hour-long interval + }); + + it("markMonitorsDue only touches the listed monitors and ignores evaluate/cleanup rows", async () => { + await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW }); + await seedJob({ _id: "check:mon-2", refId: "mon-2", type: "check", nextScheduledAt: NOW }); + await seedJob({ _id: "evaluate:mon-1", refId: "mon-1", type: "evaluate", nextScheduledAt: NOW }); + + const modified = await repo.markMonitorsDue(["mon-1"], NOW + 1_000); + + expect(modified).toBe(1); // only check:mon-1 + expect((await readRow("check:mon-2"))?.nextScheduledAt).toBe(NOW); // other monitor untouched + expect((await readRow("evaluate:mon-1"))?.nextScheduledAt).toBe(NOW); // evaluate row untouched + }); + + it("markMonitorsDue is a no-op for an empty monitor list", async () => { + await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW }); + + const modified = await repo.markMonitorsDue([], NOW + 1_000); + + expect(modified).toBe(0); + expect((await readRow("check:mon-1"))?.nextScheduledAt).toBe(NOW); + }); + it("deleteById drops every row for the monitor", async () => { await seedJob({ _id: "check:mon-1", type: "check" }); await seedJob({ _id: "geo:mon-1", type: "geo-check" }); diff --git a/server/test/unit/services/maintenanceWindowService.test.ts b/server/test/unit/services/maintenanceWindowService.test.ts index 9d381ae74..07d19e405 100644 --- a/server/test/unit/services/maintenanceWindowService.test.ts +++ b/server/test/unit/services/maintenanceWindowService.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals"; import { MaintenanceWindowService } from "../../../src/domain/maintenance-windows/maintenance-window.service.ts"; import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; +import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts"; import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -27,14 +28,21 @@ const createMonitorsRepo = () => updateByIds: jest.fn().mockResolvedValue(0), }) as unknown as jest.Mocked; +const createJobsRepo = () => + ({ + markMonitorsDue: jest.fn().mockResolvedValue(0), + }) as unknown as jest.Mocked; + const createService = (overrides?: { monitorsRepository?: ReturnType; maintenanceWindowsRepository?: ReturnType; + jobsRepository?: ReturnType; }) => { const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo(); const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo(); - const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository }); - return { service, monitorsRepository, maintenanceWindowsRepository }; + const jobsRepository = overrides?.jobsRepository ?? createJobsRepo(); + const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository }); + return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository }; }; const makeWindow = (overrides?: Partial): MaintenanceWindow => ({ @@ -267,12 +275,14 @@ describe("MaintenanceWindowService", () => { it("flips covered monitors to initializing when deleting an active window", async () => { const maintenanceWindowsRepository = createMaintenanceWindowsRepo(); (maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] })); - const { service, monitorsRepository } = createService({ maintenanceWindowsRepository }); + const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository }); await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" }); expect(monitorsRepository.updateByIds).toHaveBeenCalledTimes(1); expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]); + // Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval + expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number)); }); it("excludes the deleted window from the overlap check", async () => { From 9082cf08268f0bf6990441927be3de7944eca836 Mon Sep 17 00:00:00 2001 From: harsh-aghara Date: Thu, 18 Jun 2026 22:41:34 +0530 Subject: [PATCH 44/75] test(server): add comprehensive edge-case tests for customUpCodes validation --- .../unit/validation/monitorValidation.test.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/server/test/unit/validation/monitorValidation.test.ts b/server/test/unit/validation/monitorValidation.test.ts index c08b8be84..08f516f99 100644 --- a/server/test/unit/validation/monitorValidation.test.ts +++ b/server/test/unit/validation/monitorValidation.test.ts @@ -243,6 +243,14 @@ describe("monitorValidation — customUpCodes", () => { expect(parsed.customUpCodes).toEqual([200, 301, 404, 503]); }); + it("accepts non-standard HTTP status codes (Cloudflare, AWS ELB, etc.)", () => { + const parsed = createMonitorBodyValidation.parse({ + ...baseHttpBody, + customUpCodes: [419, 420, 440, 449, 460, 463, 497, 499, 509, 520, 521, 522, 523, 524, 525, 526, 527, 529, 530, 561], + }); + expect(parsed.customUpCodes).toEqual([419, 420, 440, 449, 460, 463, 497, 499, 509, 520, 521, 522, 523, 524, 525, 526, 527, 529, 530, 561]); + }); + it("rejects invalid HTTP status codes", () => { expect(() => createMonitorBodyValidation.parse({ @@ -259,6 +267,33 @@ describe("monitorValidation — customUpCodes", () => { ).toThrow(); }); + it("rejects fractional or floating-point status codes", () => { + expect(() => + createMonitorBodyValidation.parse({ + ...baseHttpBody, + customUpCodes: [200.5], + }) + ).toThrow(); + }); + + it("rejects status codes below 100", () => { + expect(() => + createMonitorBodyValidation.parse({ + ...baseHttpBody, + customUpCodes: [99], + }) + ).toThrow(); + }); + + it("rejects valid-looking but mathematically unsupported status codes (e.g. 599)", () => { + expect(() => + createMonitorBodyValidation.parse({ + ...baseHttpBody, + customUpCodes: [599], // Not in Node's default list nor our expanded list + }) + ).toThrow(); + }); + it("defaults to an empty array when not provided", () => { const parsed = createMonitorBodyValidation.parse(baseHttpBody); expect(parsed.customUpCodes).toEqual([]); @@ -273,6 +308,13 @@ describe("monitorValidation — customUpCodes", () => { expect(parsed.customUpCodes).toEqual([200, 201]); }); + it("accepts non-standard HTTP status codes on edits", () => { + const parsed = editMonitorBodyValidation.parse({ + customUpCodes: [419, 420, 440, 449, 460, 463, 497, 499, 509, 520, 521, 522, 523, 524, 525, 526, 527, 529, 530, 561], + }); + expect(parsed.customUpCodes).toEqual([419, 420, 440, 449, 460, 463, 497, 499, 509, 520, 521, 522, 523, 524, 525, 526, 527, 529, 530, 561]); + }); + it("rejects invalid HTTP status codes on edits", () => { expect(() => editMonitorBodyValidation.parse({ @@ -295,6 +337,20 @@ describe("monitorValidation — customUpCodes", () => { expect(parsed.monitors[0].customUpCodes).toEqual([404]); }); + it("accepts non-standard HTTP status codes on imported HTTP monitors", () => { + const parsed = importMonitorsBodyValidation.parse({ + monitors: [ + { + ...baseHttpBody, + customUpCodes: [419, 420, 440, 449, 460, 463, 497, 499, 509, 520, 521, 522, 523, 524, 525, 526, 527, 529, 530, 561], + }, + ], + }); + expect(parsed.monitors[0].customUpCodes).toEqual([ + 419, 420, 440, 449, 460, 463, 497, 499, 509, 520, 521, 522, 523, 524, 525, 526, 527, 529, 530, 561, + ]); + }); + it("rejects invalid HTTP status codes on imported HTTP monitors", () => { expect(() => importMonitorsBodyValidation.parse({ From 5efd64d44d56cb0c21fc70a1285709288da84363 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:05:17 -0700 Subject: [PATCH 45/75] thread prefetched monitor trough status service --- server/src/service/statusService.ts | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 47aa8d89a..f7d2b0b99 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -1,4 +1,3 @@ -import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import type { Check, CheckDiskInfo, CheckSnapshot } from "@/domain/checks/check.type.js"; @@ -17,7 +16,6 @@ import type { } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import { ILogger } from "@/utils/logger.js"; -import { IBufferService } from "./bufferService.js"; import type { HardwareStatusMetrics } from "@/types/network.js"; import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.types.js"; @@ -42,30 +40,21 @@ export interface IStatusService { | GrpcStatusPayload | undefined >, - check: Check + check: Check, + monitor: Monitor ): Promise; } export class StatusService implements IStatusService { static SERVICE_NAME = SERVICE_NAME; private logger: ILogger; - private buffer: IBufferService; private monitorsRepository: IMonitorsRepository; private monitorStatsRepository: IMonitorStatsRepository; - private checksRepository: IChecksRepository; - constructor( - logger: ILogger, - buffer: IBufferService, - monitorsRepository: IMonitorsRepository, - monitorStatsRepository: IMonitorStatsRepository, - checksRepository: IChecksRepository - ) { + constructor(logger: ILogger, monitorsRepository: IMonitorsRepository, monitorStatsRepository: IMonitorStatsRepository) { this.logger = logger; - this.buffer = buffer; this.monitorsRepository = monitorsRepository; this.monitorStatsRepository = monitorStatsRepository; - this.checksRepository = checksRepository; } get serviceName() { @@ -212,11 +201,11 @@ export class StatusService implements IStatusService { | GrpcStatusPayload | undefined >, - check: Check + check: Check, + monitor: Monitor ): Promise => { try { - const { monitorId, teamId, status, code } = statusResponse; - const monitor = await this.monitorsRepository.findById(monitorId, teamId); + const { status, code } = statusResponse; // Update running stats await this.tryUpdateRunningStats(monitor, statusResponse); @@ -262,7 +251,7 @@ export class StatusService implements IStatusService { }; } - // First evaluate reachability-based status changes, which apply to all monitor types + // First evaluate reachability status changes, which apply to all monitor types // and take precedence over hardware breaches. const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold); if (reachabilityResult.transitioned) { From f740eef0fad0c45676b103fc3068586475e2fb5e Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:05:57 -0700 Subject: [PATCH 46/75] thread prefetched worker, remove dead code --- server/src/worker/worker.check-evaluator.ts | 7 +-- server/src/worker/worker.check-pipeline.ts | 18 -------- server/src/worker/worker.db-queue.ts | 13 +++--- server/src/worker/worker.helper.ts | 49 +-------------------- 4 files changed, 13 insertions(+), 74 deletions(-) diff --git a/server/src/worker/worker.check-evaluator.ts b/server/src/worker/worker.check-evaluator.ts index bbae96d69..34c444271 100644 --- a/server/src/worker/worker.check-evaluator.ts +++ b/server/src/worker/worker.check-evaluator.ts @@ -1,11 +1,12 @@ import { MonitorStatusResponse } from "@/types/network.js"; import { MonitorEvaluation } from "@/worker/worker.interface.js"; import { Check } from "@/domain/checks/check.type.js"; +import { Monitor } from "@/domain/monitors/monitor.types.js"; import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; import { IStatusService } from "@/service/statusService.js"; export interface ICheckEvaluator { - evaluate(status: MonitorStatusResponse, check: Check): Promise; + evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise; } export class CheckEvaluator implements ICheckEvaluator { @@ -13,11 +14,11 @@ export class CheckEvaluator implements ICheckEvaluator { private statusService: IStatusService, private monitorStatusPolicy: IMonitorStatusPolicy ) {} - evaluate = async (status: MonitorStatusResponse, check: Check) => { + evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => { // **************************** // Step 3: Evaluate and return result to reactors // **************************** - const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); + const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor); // Step 5. Get decisions and create an evaluation obj const decision = this.monitorStatusPolicy.evaluate(statusChangeResult); diff --git a/server/src/worker/worker.check-pipeline.ts b/server/src/worker/worker.check-pipeline.ts index c472c4bfa..045cea1ad 100644 --- a/server/src/worker/worker.check-pipeline.ts +++ b/server/src/worker/worker.check-pipeline.ts @@ -7,8 +7,6 @@ import { IBufferService } from "@/service/bufferService.js"; import { AppError } from "@/utils/AppError.js"; import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; -import { ICheckProducer } from "@/worker/worker.check-producer.js"; -import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; const SERVICE_NAME = "CheckPipeline"; @@ -16,22 +14,6 @@ export interface ICheckPipeline { run(monitor: Monitor): Promise; // null = skipped } -export class CheckPipeline implements ICheckPipeline { - constructor( - private checkProducer: ICheckProducer, - private checkEvaluator: ICheckEvaluator - ) {} - - run = async (monitor: Monitor): Promise => { - const checkResult = await this.checkProducer.produce(monitor); - if (!checkResult) { - return null; // skipped due to maintenance / geo guards / other guards - } - const evaluation = await this.checkEvaluator.evaluate(checkResult.status, checkResult.check); - return evaluation; - }; -} - export class GeoChecksPipeline implements ICheckPipeline { constructor( private maintenanceWindowsRepository: IMaintenanceWindowsRepository, diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 9088dc3b3..bef7a838e 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -151,7 +151,7 @@ export class DBQueueWorker implements IWorker { private runCheck = async (job: Job) => { if (!job.refId) return; - const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId + const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId if (!monitor) return; await this.checkProducer.produce(monitor); }; @@ -162,19 +162,22 @@ export class DBQueueWorker implements IWorker { private runEvaluate = async (job: Job) => { if (!job.refId) return; - const [monitor] = await this.monitorsRepository.findByIds([job.refId]); // job row has no teamId + const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId if (!monitor) return; const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt); + + let current = monitor; for (const check of checks) { const status = this.checkService.toStatusResponse(check); - const evaluation = await this.checkEvaluator.evaluate(status, check); + const evaluation = await this.checkEvaluator.evaluate(status, check, current); await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications - await this.monitorsRepository.updateById(job.refId, monitor.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) }); + await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) }); + current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check } }; private runGeoCheck = async (job: Job) => { - const [monitor] = await this.monitorsRepository.findByIds([job.refId!]); + const monitor = await this.monitorsRepository.findByIdLean(job.refId!); if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff }; diff --git a/server/src/worker/worker.helper.ts b/server/src/worker/worker.helper.ts index 7b60a26ba..3d2d2e183 100644 --- a/server/src/worker/worker.helper.ts +++ b/server/src/worker/worker.helper.ts @@ -1,5 +1,4 @@ const SERVICE_NAME = "JobQueueHelper"; -import type { Monitor } from "@/domain/monitors/monitor.types.js"; import { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; import { ICheckService } from "@/domain/checks/check.service.js"; import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js"; @@ -11,13 +10,9 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; import { ILogger } from "@/utils/logger.js"; -import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; -import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; export interface IWorkerHelper { readonly serviceName: string; - getHeartbeatJob(): (monitor: Monitor) => Promise; - getHeartbeatGeoJob(): (monitor: Monitor) => Promise; getCleanupOrphanedJob(): () => Promise; getCleanupRetentionJob(): () => Promise; } @@ -49,9 +44,6 @@ export class WorkerHelper implements IWorkerHelper { private checksRepository: IChecksRepository; private incidentsRepository: IIncidentsRepository; private geoChecksRepository: IGeoChecksRepository; - private reactorDispatcher: IReactorDispatcher; - private checkPipeline: ICheckPipeline; - private geoCheckPipeline: ICheckPipeline; constructor( logger: ILogger, @@ -63,10 +55,7 @@ export class WorkerHelper implements IWorkerHelper { monitorStatsRepository: IMonitorStatsRepository, checksRepository: IChecksRepository, incidentsRepository: IIncidentsRepository, - geoChecksRepository: IGeoChecksRepository, - reactorDispatcher: IReactorDispatcher, - checkPipeline: ICheckPipeline, - geoCheckPipeline: ICheckPipeline + geoChecksRepository: IGeoChecksRepository ) { this.logger = logger; this.checkService = checkService; @@ -78,48 +67,12 @@ export class WorkerHelper implements IWorkerHelper { this.checksRepository = checksRepository; this.incidentsRepository = incidentsRepository; this.geoChecksRepository = geoChecksRepository; - this.reactorDispatcher = reactorDispatcher; - this.checkPipeline = checkPipeline; - this.geoCheckPipeline = geoCheckPipeline; } get serviceName() { return WorkerHelper.SERVICE_NAME; } - getHeartbeatJob = () => { - return async (monitor: Monitor) => { - try { - const evaluation = await this.checkPipeline.run(monitor); - if (evaluation) await this.reactorDispatcher.dispatch(evaluation); - } catch (error: unknown) { - this.logger.warn({ - message: error instanceof Error ? error.message : "Unknown error", - service: SERVICE_NAME, - method: "getMonitorJob", - stack: error instanceof Error ? error.stack : undefined, - }); - throw error; - } - }; - }; - - getHeartbeatGeoJob = () => { - return async (monitor: Monitor) => { - try { - await this.geoCheckPipeline.run(monitor); - } catch (error: unknown) { - this.logger.error({ - message: error instanceof Error ? error.message : "Unknown error", - service: SERVICE_NAME, - method: "getHeartbeatGeoJob", - stack: error instanceof Error ? error.stack : undefined, - }); - // Don't throw - geo check failures shouldn't crash the job scheduler - } - }; - }; - getCleanupOrphanedJob = () => { return async () => { try { From 190a3b47dad3cd69ff09305833c2a16d4100cd2b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:06:19 -0700 Subject: [PATCH 47/75] add findByIdLean for workers --- server/src/domain/monitors/monitor.repository.interface.ts | 2 ++ server/src/domain/monitors/monitor.repository.mongo.ts | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/server/src/domain/monitors/monitor.repository.interface.ts b/server/src/domain/monitors/monitor.repository.interface.ts index e5a0fa585..46f3df396 100644 --- a/server/src/domain/monitors/monitor.repository.interface.ts +++ b/server/src/domain/monitors/monitor.repository.interface.ts @@ -22,6 +22,8 @@ export interface IMonitorsRepository { createMonitors(monitors: Monitor[]): Promise; // single fetch findById(monitorId: string, teamId: string): Promise; + // for workers, don't need all details + findByIdLean(monitorId: string): Promise; // collection fetch findAll(): Promise; diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index e41c8a6f2..0ce0d48a6 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -38,6 +38,11 @@ class MongoMonitorsRepository implements IMonitorsRepository { return this.toEntity(monitor); }; + findByIdLean = async (monitorId: string): Promise => { + const monitor = await MonitorModel.findOne({ _id: monitorId }); + return monitor ? this.toEntity(monitor) : null; + }; + findAll = async (): Promise => { const monitors = await MonitorModel.find(); return this.mapDocuments(monitors); From e32590425b5379e73b14f01def85e30d4659d21e Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:06:43 -0700 Subject: [PATCH 48/75] remove unsed from constructor --- server/src/config/services.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 346f77209..9de13f27b 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -103,7 +103,7 @@ import { ILogger } from "@/utils/logger.js"; import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; -import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; +import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; import { CheckProducer } from "@/worker/worker.check-producer.js"; import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { DBQueueWorker } from "@/worker/worker.db-queue.js"; @@ -229,7 +229,7 @@ export const initializeServices = async ({ const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository); - const statusService = new StatusService(logger, bufferService, monitorsRepository, monitorStatsRepository, checksRepository); + const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); // Notification providers const webhookProvider = new WebhookProvider(logger); @@ -276,8 +276,6 @@ export const initializeServices = async ({ const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy); // pipelines - const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator); - const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger); const workerHelper = new WorkerHelper( @@ -290,10 +288,7 @@ export const initializeServices = async ({ monitorStatsRepository, checksRepository, incidentsRepository, - geoChecksRepository, - reactorDispatcher, - checkPipeline, - geoCheckPipeline + geoChecksRepository ); const worker = await DBQueueWorker.create( From 5ed9ab780ae3d5c697ef14c84a56bf39b4f99fe0 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 18 Jun 2026 11:06:57 -0700 Subject: [PATCH 49/75] update tests, add check producer tests --- .../helpers/InMemoryMonitorsRepository.ts | 5 + server/test/helpers/heartbeatTestHarness.ts | 54 ++---- .../test/unit/services/checkPipeline.test.ts | 113 +----------- .../test/unit/services/checkProducer.test.ts | 120 +++++++++++++ .../test/unit/services/dbQueueWorker.test.ts | 34 +++- .../test/unit/services/statusService.test.ts | 162 +++++++++--------- .../test/unit/services/workerHelper.test.ts | 93 +--------- 7 files changed, 253 insertions(+), 328 deletions(-) create mode 100644 server/test/unit/services/checkProducer.test.ts diff --git a/server/test/helpers/InMemoryMonitorsRepository.ts b/server/test/helpers/InMemoryMonitorsRepository.ts index c53f270a7..45db5b3f7 100644 --- a/server/test/helpers/InMemoryMonitorsRepository.ts +++ b/server/test/helpers/InMemoryMonitorsRepository.ts @@ -40,6 +40,11 @@ export class InMemoryMonitorsRepository implements IMonitorsRepository { return this.monitors.filter((m) => monitorIds.includes(m.id)).map((m) => ({ ...m })); } + async findByIdLean(monitorId: string): Promise { + const monitor = this.monitors.find((m) => m.id === monitorId); + return monitor ? { ...monitor } : null; + } + async findByIdsWithChecks(monitorIds: string[], _checksCount?: number): Promise { return this.findByIds(monitorIds); } diff --git a/server/test/helpers/heartbeatTestHarness.ts b/server/test/helpers/heartbeatTestHarness.ts index baf7eeee3..5711f5aa9 100644 --- a/server/test/helpers/heartbeatTestHarness.ts +++ b/server/test/helpers/heartbeatTestHarness.ts @@ -1,7 +1,5 @@ import { jest } from "@jest/globals"; -import { WorkerHelper } from "../../src/worker/worker.helper.ts"; import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts"; -import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts"; import { CheckProducer } from "../../src/worker/worker.check-producer.ts"; import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts"; import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts"; @@ -71,20 +69,6 @@ const createStubMonitorStatsRepo = () => ({ deleteByMonitorIdsNotIn: jest.fn(), }); -const createStubChecksRepo = () => ({ - create: jest.fn(), - createChecks: jest.fn(), - findByMonitorId: jest.fn(), - findByTeamId: jest.fn(), - findLatestByMonitorIds: jest.fn(), - findByDateRangeAndMonitorId: jest.fn(), - findSummaryByTeamId: jest.fn(), - deleteByMonitorId: jest.fn(), - deleteByTeamId: jest.fn(), - deleteByMonitorIdsNotIn: jest.fn(), - deleteOlderThan: jest.fn(), -}); - export interface HeartbeatTestHarness { monitorsRepo: InMemoryMonitorsRepository; incidentsRepo: InMemoryIncidentsRepository; @@ -108,13 +92,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { const logger = createMockLogger() as any; const bufferStub = { addToBuffer: jest.fn(), addGeoCheckToBuffer: jest.fn(), scheduleNextFlush: jest.fn() }; - const statusService = new StatusService( - logger, - bufferStub as any, - monitorsRepo as any, - createStubMonitorStatsRepo() as any, - createStubChecksRepo() as any - ); + const statusService = new StatusService(logger, monitorsRepo as any, createStubMonitorStatsRepo() as any); const messageBuilder = { extractThresholdBreaches: jest.fn().mockReturnValue([]) }; const incidentService = new IncidentService(logger, incidentsRepo, monitorsRepo as any, { findById: jest.fn() } as any, messageBuilder as any); @@ -149,7 +127,6 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { }; const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) }; - const geoChecksService = { buildGeoCheck: jest.fn() }; const checkProducer = new CheckProducer( monitorsRepo as any, @@ -160,28 +137,21 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { logger ); const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy()); - const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator); - const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger); const notificationReactor = new NotificationReactor(notificationsService as any); const incidentReactor = new IncidentReactor(incidentService as any); const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); - const helper = new WorkerHelper( - logger, - checkService as any, - { getDBSettings: jest.fn() } as any, - monitorsRepo as any, - { deleteByMonitorIdsNotIn: jest.fn() } as any, - { findAllTeamIds: jest.fn() } as any, - createStubMonitorStatsRepo() as any, - createStubChecksRepo() as any, - incidentsRepo as any, - { deleteByMonitorIdsNotIn: jest.fn() } as any, - reactorDispatcher, - checkPipeline, - geoCheckPipeline - ); + // Mirror the production check→evaluate flow: read the monitor fresh each cycle (so the + // accumulated statusWindow is visible), produce a check, then evaluate against that monitor. + const heartbeatJob = async (seedMonitor: Monitor) => { + const monitor = await monitorsRepo.findByIdLean(seedMonitor.id); + if (!monitor) return; + const result = await checkProducer.produce(monitor); + if (!result) return; // skipped (e.g. maintenance window) + const evaluation = await checkEvaluator.evaluate(result.status, result.check, monitor); + await reactorDispatcher.dispatch(evaluation); + }; return { monitorsRepo, @@ -193,7 +163,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness { bufferStub, maintenanceWindowsRepo, messageBuilder, - heartbeatJob: helper.getHeartbeatJob(), + heartbeatJob, setNextResponse, setNextResponseFull, }; diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/checkPipeline.test.ts index 9b1fffd8b..008e879fa 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/checkPipeline.test.ts @@ -1,8 +1,5 @@ import { describe, expect, it, jest } from "@jest/globals"; -import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts"; -import { CheckProducer } from "../../../src/worker/worker.check-producer.ts"; -import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts"; -import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; +import { GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; @@ -23,114 +20,6 @@ const activeWindow = () => { return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; }; -// ── CheckPipeline ──────────────────────────────────────────────────────────── - -const createCheckPipeline = (overrides?: Record) => { - const defaults = { - logger: createMockLogger(), - networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, - statusService: { - updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }), - }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - buffer: { addToBuffer: jest.fn() }, - monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, - monitorStatusPolicy: new MonitorStatusPolicy(), - ...overrides, - }; - const checkProducer = new CheckProducer( - defaults.monitorsRepository as any, - defaults.maintenanceWindowsRepository as any, - defaults.checkService as any, - defaults.networkService as any, - defaults.buffer as any, - defaults.logger as any - ); - const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); - const pipeline = new CheckPipeline(checkProducer, checkEvaluator); - return { pipeline, defaults }; -}; - -describe("CheckPipeline", () => { - it("throws when monitor id is missing", async () => { - const { pipeline } = createCheckPipeline(); - await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id"); - }); - - it("skips and sets maintenance status when in maintenance window", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - const result = await pipeline.run(makeMonitor({ status: "up" })); - - expect(result).toBeNull(); - expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" }); - }); - - it("does not re-write status when already in maintenance", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - await pipeline.run(makeMonitor({ status: "maintenance" })); - - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("throws when network response is null", async () => { - const { pipeline } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, - }); - await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response"); - }); - - it("returns null and warns when toCheck returns null", async () => { - const { pipeline, defaults } = createCheckPipeline({ - checkService: { toCheck: jest.fn().mockReturnValue(null) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(result).toBeNull(); - expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); - expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled(); - }); - - it("buffers the check, updates status, and returns an evaluation", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; - const statusChange = { monitor: { id: "m1", status: "down" }, statusChanged: true, prevStatus: "up", code: 500 }; - const { pipeline, defaults } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, - statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith({ id: "check-1" }); - expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalled(); - expect(result).toMatchObject({ - monitor: statusChange.monitor, - status, - check: { id: "check-1" }, - statusChange, - decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, incidentReason: "status_down" }), - }); - }); - - it("proceeds with the check when no maintenance window is active", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, - }); - - await pipeline.run(makeMonitor()); - - expect(defaults.networkService.requestStatus).toHaveBeenCalled(); - }); -}); - // ── GeoChecksPipeline ────────────────────────────────────────────────────────── const createGeoPipeline = (overrides?: Record) => { diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts new file mode 100644 index 000000000..bf93c8ea2 --- /dev/null +++ b/server/test/unit/services/checkProducer.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { CheckProducer } from "../../../src/worker/worker.check-producer.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "m1", + teamId: "team", + type: "http", + interval: 60000, + status: "up", + ...overrides, + }) as Monitor; + +// A maintenance window straddling "now" — active per isWindowActive. +const activeWindow = () => { + const now = Date.now(); + return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; +}; + +const createProducer = (overrides?: Record) => { + const defaults = { + logger: createMockLogger(), + monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, + checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, + networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, + buffer: { addToBuffer: jest.fn() }, + ...overrides, + }; + const producer = new CheckProducer( + defaults.monitorsRepository as any, + defaults.maintenanceWindowsRepository as any, + defaults.checkService as any, + defaults.networkService as any, + defaults.buffer as any, + defaults.logger as any + ); + return { producer, defaults }; +}; + +describe("CheckProducer", () => { + it("throws when monitor id is missing", async () => { + const { producer } = createProducer(); + await expect(producer.produce({} as Monitor)).rejects.toThrow("No monitor id"); + }); + + // ── maintenance gate ────────────────────────────────────────────────────── + + it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => { + const { producer, defaults } = createProducer({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, + }); + + const result = await producer.produce(makeMonitor({ status: "up" })); + + expect(result).toBeNull(); + expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" }); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); + }); + + it("does not re-write status when already in maintenance", async () => { + const { producer, defaults } = createProducer({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, + }); + + const result = await producer.produce(makeMonitor({ status: "maintenance" })); + + expect(result).toBeNull(); + expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("proceeds with the check when the maintenance window is inactive", async () => { + const { producer, defaults } = createProducer({ + maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, + }); + + await producer.produce(makeMonitor()); + + expect(defaults.networkService.requestStatus).toHaveBeenCalled(); + }); + + // ── acquire / record ────────────────────────────────────────────────────── + + it("throws when the network response is null", async () => { + const { producer } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, + }); + + await expect(producer.produce(makeMonitor())).rejects.toThrow("No network response"); + }); + + it("returns null, warns, and does not buffer when toCheck yields nothing", async () => { + const { producer, defaults } = createProducer({ + checkService: { toCheck: jest.fn().mockReturnValue(null) }, + }); + + const result = await producer.produce(makeMonitor()); + + expect(result).toBeNull(); + expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); + expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); + }); + + it("buffers the built check and returns the status and check on success", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; + const check = { id: "check-1" }; + const { producer, defaults } = createProducer({ + networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, + checkService: { toCheck: jest.fn().mockReturnValue(check) }, + }); + + const result = await producer.produce(makeMonitor()); + + expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check); + expect(result).toEqual({ status, check }); + }); +}); diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index e60f82a95..1769ed573 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -80,6 +80,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses }; const monitorsRepository = { findByIds: jest.fn().mockResolvedValue([makeMonitor()]), + findByIdLean: jest.fn().mockResolvedValue(makeMonitor()), findAll: jest.fn().mockResolvedValue([]), updateById: jest.fn().mockResolvedValue({}), }; @@ -94,7 +95,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses produce: jest.fn().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }), }; const checkEvaluator = { - evaluate: jest.fn().mockResolvedValue({ monitor: makeMonitor(), decision: {} }), + evaluate: jest.fn().mockResolvedValue({ monitor: makeMonitor(), statusChange: { monitor: makeMonitor() }, decision: {} }), }; const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) }; const dispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) }; @@ -268,7 +269,7 @@ describe("DBQueueWorker", () => { const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 }); const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); - expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]); + expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1"); expect(mocks.checkProducer.produce).toHaveBeenCalled(); // The check stage only produces; the buffer arms evaluate once the check is durably stored expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled(); @@ -307,6 +308,35 @@ describe("DBQueueWorker", () => { expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 }); }); + it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => { + const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null }); + const initialMonitor = makeMonitor(); + const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1 + const monitorsRepository = { + findByIdLean: jest.fn().mockResolvedValue(initialMonitor), + updateById: jest.fn().mockResolvedValue({}), + }; + const checksRepository = { findUnevaluatedByMonitorId: jest.fn().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) }; + const checkEvaluator = { + evaluate: jest.fn().mockResolvedValue({ monitor: postWriteMonitor, statusChange: { monitor: postWriteMonitor }, decision: {} }), + }; + const { mocks } = await start({ + mocks: { + jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, + monitorsRepository, + checksRepository, + checkEvaluator, + }, + }); + + // The monitor is read once for the whole backlog, not once per check + expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1); + expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2); + // First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first + expect(mocks.checkEvaluator.evaluate.mock.calls[0][2]).toBe(initialMonitor); + expect(mocks.checkEvaluator.evaluate.mock.calls[1][2]).toBe(postWriteMonitor); + }); + it("cleanup-orphaned job runs the helper's cleanup function", async () => { const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 }); const cleanupFn = jest.fn().mockResolvedValue(undefined); diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 2bd5fa74c..f011c183c 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -7,21 +7,15 @@ import type { Check } from "../../../src/domain/checks/check.type.ts"; import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; -import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts"; -import type { IBufferService } from "../../../src/service/bufferService.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── -const createBuffer = (): jest.Mocked> => ({ - addToBuffer: jest.fn(), -}); - const createMonitorsRepo = () => { const findById = jest.fn(); const updateStatusWindowAndChecks = jest .fn() .mockImplementation( - ( + async ( _id: unknown, _tid: unknown, status: boolean, @@ -30,27 +24,23 @@ const createMonitorsRepo = () => { maxRecentChecks: number, statusPatch?: Partial ) => { - // Grab the monitor that findById was configured to resolve with - const monitor = findById.mock.results.at(-1)?.value; - if (monitor && typeof monitor.then === "function") { - return monitor.then((m: any) => { - m.statusWindow = m.statusWindow || []; - m.statusWindow.push(status); - while (m.statusWindow.length > windowSize) { - m.statusWindow.shift(); - } - m.recentChecks = m.recentChecks || []; - m.recentChecks.push(checkSnapshot); - while (m.recentChecks.length > maxRecentChecks) { - m.recentChecks.shift(); - } - if (statusPatch) { - Object.assign(m, statusPatch); - } - return { ...m }; - }); + // updateMonitorStatus no longer reads the monitor itself — tests seed it via + // findById.mockResolvedValue, so the mock resolves it here as the post-write monitor. + const m: any = await findById(); + m.statusWindow = m.statusWindow || []; + m.statusWindow.push(status); + while (m.statusWindow.length > windowSize) { + m.statusWindow.shift(); } - return Promise.reject(new Error("findById not mocked")); + m.recentChecks = m.recentChecks || []; + m.recentChecks.push(checkSnapshot); + while (m.recentChecks.length > maxRecentChecks) { + m.recentChecks.shift(); + } + if (statusPatch) { + Object.assign(m, statusPatch); + } + return { ...m }; } ); return { @@ -67,23 +57,17 @@ const createMonitorStatsRepo = () => updateByMonitorId: jest.fn(), }) as unknown as jest.Mocked; -const createChecksRepo = () => ({}) as unknown as jest.Mocked; - const createService = (overrides?: { logger?: ReturnType; - buffer?: ReturnType; monitorsRepository?: ReturnType; monitorStatsRepository?: ReturnType; - checksRepository?: ReturnType; }) => { const logger = overrides?.logger ?? createMockLogger(); - const buffer = overrides?.buffer ?? createBuffer(); const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo(); const monitorStatsRepository = overrides?.monitorStatsRepository ?? createMonitorStatsRepo(); - const checksRepository = overrides?.checksRepository ?? createChecksRepo(); - const service = new StatusService(logger as any, buffer as any, monitorsRepository, monitorStatsRepository, checksRepository); - return { service, logger, buffer, monitorsRepository, monitorStatsRepository, checksRepository }; + const service = new StatusService(logger as any, monitorsRepository, monitorStatsRepository); + return { service, logger, monitorsRepository, monitorStatsRepository }; }; const makeMonitor = (overrides?: Partial): Monitor => @@ -264,19 +248,31 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); expect(result.statusChanged).toBe(false); expect(result.prevStatus).toBe("up"); }); + it("uses the prefetched monitor and skips findById when one is passed", async () => { + const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "up" }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockResolvedValue(monitor); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); + + expect(monitorsRepository.findById).not.toHaveBeenCalled(); + expect(result.prevStatus).toBe("up"); + expect(result.monitor).toBe(monitor); + }); + it("pushes to statusWindow and trims to statusWindowSize", async () => { const monitor = makeMonitor({ statusWindow: [true, true, true, true, true], statusWindowSize: 5 }); const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); // Atomic push should have been called with the correct status and window size expect(monitorsRepository.updateStatusWindowAndChecks).toHaveBeenCalledWith( @@ -297,7 +293,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" })); + await service.updateMonitorStatus(makeStatusResponse(), makeCheck({ id: "new-check" }), monitor); expect(monitor.recentChecks).toHaveLength(MAX_RECENT_CHECKS); expect(monitor.recentChecks[MAX_RECENT_CHECKS - 1]).toEqual(expect.objectContaining({ id: "new-check" })); @@ -315,7 +311,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -333,7 +329,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); @@ -350,7 +346,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -370,7 +366,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -387,7 +383,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -399,7 +395,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck()); + await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); expect(monitor.statusWindow).toEqual([true]); }); @@ -410,23 +406,25 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - await service.updateMonitorStatus(makeStatusResponse(), makeCheck()); + await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); expect(monitor.recentChecks).toHaveLength(1); }); it("throws AppError when repository throws", async () => { + const monitor = makeMonitor(); const { service, monitorsRepository } = createService(); - (monitorsRepository.findById as jest.Mock).mockRejectedValue(new Error("db error")); + (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue(new Error("db error")); - await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Failed to update monitor"); + await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Failed to update monitor"); }); it("throws AppError with 'Unknown error' for non-Error exceptions", async () => { + const monitor = makeMonitor(); const { service, monitorsRepository } = createService(); - (monitorsRepository.findById as jest.Mock).mockRejectedValue("string error"); + (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mockRejectedValue("string error"); - await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck())).rejects.toThrow("Unknown error"); + await expect(service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor)).rejects.toThrow("Unknown error"); }); it("keeps status 'down' on a single successful check that leaves failure rate at threshold (regression: #3438 mirror)", async () => { @@ -444,7 +442,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -462,7 +460,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -484,7 +482,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); @@ -502,7 +500,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); }); @@ -518,7 +516,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -535,7 +533,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -550,7 +548,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -567,7 +565,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("down"); @@ -580,7 +578,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); (monitorStatsRepository.updateByMonitorId as jest.Mock).mockRejectedValue(new Error("stats db down")); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck(), monitor); expect(result.monitor.status).toBe("up"); expect(logger.warn).toHaveBeenCalledWith( @@ -598,7 +596,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse({ code: 201 }), makeCheck(), monitor); expect(result.code).toBe(201); expect(result.timestamp).toBeGreaterThan(0); @@ -639,7 +637,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.5 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.cpu).toBe(true); expect(result.thresholdBreaches?.memory).toBe(false); @@ -656,7 +654,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.5 }, memory: { usage_percent: 0.9 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.memory).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -673,7 +671,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.95 }], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -690,7 +688,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [90] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.temp).toBe(true); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; @@ -712,7 +710,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} }, } as any); - await service.updateMonitorStatus(response, makeCheck()); + await service.updateMonitorStatus(response, makeCheck(), monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(5); @@ -731,7 +729,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.monitor.status).toBe("breached"); expect(result.statusChanged).toBe(false); @@ -746,7 +744,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [30] }, memory: { usage_percent: 0.1 }, disk: [{ usage_percent: 0.1 }], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("up"); @@ -769,7 +767,7 @@ describe("StatusService", () => { status: false, data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck({ status: false })); + const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor); expect(result.monitor.status).toBe("down"); }); @@ -781,7 +779,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: {}, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.cpu).toBe(false); }); @@ -793,7 +791,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: {}, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); @@ -807,7 +805,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1, temperature: [] }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); @@ -821,7 +819,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.temp).toBe(false); }); @@ -833,7 +831,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse(undefined as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); @@ -845,7 +843,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({} as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); @@ -860,7 +858,7 @@ describe("StatusService", () => { (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); - const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck()); + const result = await service.updateMonitorStatus(makeStatusResponse(), makeCheck(), monitor); expect(result.thresholdBreaches).toBeUndefined(); }); @@ -874,7 +872,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [null as any], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -888,7 +886,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [{ device: "/dev/sda" }], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -902,7 +900,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: undefined, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.cpu).toBe(false); expect(result.thresholdBreaches?.temp).toBe(false); @@ -917,7 +915,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: undefined, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.memory).toBe(false); }); @@ -931,7 +929,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: undefined, host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); expect(result.thresholdBreaches?.disk).toBe(false); }); @@ -945,7 +943,7 @@ describe("StatusService", () => { (monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m)); const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9 }, memory: { usage_percent: 0.1 }, disk: [], host: {} } } as any); - const result = await service.updateMonitorStatus(response, makeCheck()); + const result = await service.updateMonitorStatus(response, makeCheck(), monitor); const patch = (monitorsRepository.updateStatusWindowAndChecks as jest.Mock).mock.calls.at(-1)?.[6]; expect(patch.cpuAlertCounter).toBe(2); @@ -971,7 +969,7 @@ describe("StatusService", () => { status: false, data: { cpu: { usage_percent: 0.1 }, memory: { usage_percent: 0.1 }, disk: [], host: {} }, } as any); - const result = await service.updateMonitorStatus(response, makeCheck({ status: false })); + const result = await service.updateMonitorStatus(response, makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -991,7 +989,7 @@ describe("StatusService", () => { const response = makeHardwareResponse({ data: { cpu: { usage_percent: 0.9, temperature: [90] }, memory: { usage_percent: 0.9 }, disk: [{ usage_percent: 0.95 }], host: {} }, } as any); - await service.updateMonitorStatus(response, makeCheck()); + await service.updateMonitorStatus(response, makeCheck(), monitor); expect(monitor.cpuAlertCounter).toBe(0); expect(monitor.memoryAlertCounter).toBe(0); diff --git a/server/test/unit/services/workerHelper.test.ts b/server/test/unit/services/workerHelper.test.ts index 27f06ab6c..96b45d3b3 100644 --- a/server/test/unit/services/workerHelper.test.ts +++ b/server/test/unit/services/workerHelper.test.ts @@ -1,14 +1,11 @@ import { describe, expect, it, jest } from "@jest/globals"; import { WorkerHelper } from "../../../src/worker/worker.helper.ts"; -import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── // -// WorkerHelper is now thin glue: it wires the scheduler's job functions to the -// CheckPipeline / GeoChecksPipeline (acquire→record→decide) and the -// ReactorDispatcher (react), and owns the cleanup jobs. The pipeline and -// dispatcher internals are tested in checkPipeline.test.ts / reactorDispatcher.test.ts; +// WorkerHelper is now thin glue: it wires the geo job to the GeoChecksPipeline and +// owns the cleanup jobs. The pipeline internals are tested in checkPipeline.test.ts; // here we test only the glue + cleanup jobs. const createHelper = (overrides?: Record) => { @@ -42,10 +39,6 @@ const createHelper = (overrides?: Record) => { deleteOlderThan: jest.fn().mockResolvedValue(0), }; - const checkPipeline = { run: jest.fn().mockResolvedValue(null) }; - const geoCheckPipeline = { run: jest.fn().mockResolvedValue(null) }; - const reactorDispatcher = { dispatch: jest.fn().mockResolvedValue(undefined) }; - const defaults = { logger: createMockLogger(), checkService, @@ -57,9 +50,6 @@ const createHelper = (overrides?: Record) => { checksRepository, incidentsRepository, geoChecksRepository, - reactorDispatcher, - checkPipeline, - geoCheckPipeline, ...overrides, }; @@ -73,26 +63,11 @@ const createHelper = (overrides?: Record) => { defaults.monitorStatsRepository as any, defaults.checksRepository as any, defaults.incidentsRepository as any, - defaults.geoChecksRepository as any, - defaults.reactorDispatcher as any, - defaults.checkPipeline as any, - defaults.geoCheckPipeline as any + defaults.geoChecksRepository as any ); return { helper, defaults }; }; -const makeMonitor = (overrides?: Partial): Monitor => - ({ - id: "m1", - teamId: "team", - type: "http", - interval: 60000, - status: "up", - geoCheckEnabled: false, - geoCheckLocations: [], - ...overrides, - }) as Monitor; - // ── Tests ──────────────────────────────────────────────────────────────────── describe("WorkerHelper", () => { @@ -103,68 +78,6 @@ describe("WorkerHelper", () => { }); }); - // ── getHeartbeatJob (glue) ──────────────────────────────────────────────── - - describe("getHeartbeatJob", () => { - it("runs the pipeline and dispatches the evaluation to reactors", async () => { - const evaluation = { monitor: { id: "m1" }, decision: {} }; - const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(evaluation) } }); - - await helper.getHeartbeatJob()(makeMonitor()); - - expect(defaults.checkPipeline.run).toHaveBeenCalled(); - expect(defaults.reactorDispatcher.dispatch).toHaveBeenCalledWith(evaluation); - }); - - it("does not dispatch when the pipeline returns null (skipped)", async () => { - const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockResolvedValue(null) } }); - - await helper.getHeartbeatJob()(makeMonitor()); - - expect(defaults.reactorDispatcher.dispatch).not.toHaveBeenCalled(); - }); - - it("logs and rethrows when the pipeline throws", async () => { - const { helper, defaults } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue(new Error("boom")) } }); - - await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toThrow("boom"); - expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "boom" })); - }); - - it("rethrows non-Error failures", async () => { - const { helper } = createHelper({ checkPipeline: { run: jest.fn().mockRejectedValue("unexpected") } }); - - await expect(helper.getHeartbeatJob()(makeMonitor())).rejects.toBe("unexpected"); - }); - }); - - // ── getHeartbeatGeoJob (glue) ───────────────────────────────────────────── - - describe("getHeartbeatGeoJob", () => { - it("runs the geo pipeline", async () => { - const { helper, defaults } = createHelper(); - - await helper.getHeartbeatGeoJob()(makeMonitor()); - - expect(defaults.geoCheckPipeline.run).toHaveBeenCalled(); - }); - - it("catches and logs a pipeline error without rethrowing", async () => { - const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue(new Error("api timeout")) } }); - - await expect(helper.getHeartbeatGeoJob()(makeMonitor())).resolves.toBeUndefined(); - expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "api timeout" })); - }); - - it("logs 'Unknown error' for non-Error failures", async () => { - const { helper, defaults } = createHelper({ geoCheckPipeline: { run: jest.fn().mockRejectedValue("string error") } }); - - await helper.getHeartbeatGeoJob()(makeMonitor()); - - expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined })); - }); - }); - // ── getCleanupOrphanedJob ──────────────────────────────────────────────── describe("getCleanupOrphanedJob", () => { From a058c0518c9208e58c665322be1b3ed515b06d14 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 19 Jun 2026 09:37:24 -0700 Subject: [PATCH 50/75] change status on exit maintenance window --- server/src/service/statusService.ts | 4 ++-- server/src/worker/worker.check-producer.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/service/statusService.ts b/server/src/service/statusService.ts index 47aa8d89a..f9bbbb61c 100755 --- a/server/src/service/statusService.ts +++ b/server/src/service/statusService.ts @@ -232,10 +232,10 @@ export class StatusService implements IStatusService { // Build the status patch — computed against the projected window const patch: Partial = {}; - // Resolve "initializing" up-front, incidents should be created on initialization && down + // Resolve "initializing" and "maintenance" up front, incidents should be created on initialization && down let newStatus: MonitorStatus = monitor.status; let statusChanged = false; - if (monitor.status === "initializing") { + if (monitor.status === "initializing" || monitor.status === "maintenance") { newStatus = status === true ? "up" : "down"; patch.status = newStatus; statusChanged = newStatus === "down"; diff --git a/server/src/worker/worker.check-producer.ts b/server/src/worker/worker.check-producer.ts index ad3975909..3365ea93b 100644 --- a/server/src/worker/worker.check-producer.ts +++ b/server/src/worker/worker.check-producer.ts @@ -49,7 +49,8 @@ export class CheckProducer implements ICheckProducer { method: "produce", }); if (monitor.status !== "maintenance") { - await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" }); + // Clear the status window to avoid incidents being created on next check + await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] }); } return null; } From 7fe7b79103374138f97ceb27aabadf485a38165a Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 19 Jun 2026 09:51:25 -0700 Subject: [PATCH 51/75] update server tests --- .../test/unit/services/checkPipeline.test.ts | 2 +- .../test/unit/services/statusService.test.ts | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/checkPipeline.test.ts index 9b1fffd8b..3cbe71360 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/checkPipeline.test.ts @@ -67,7 +67,7 @@ describe("CheckPipeline", () => { expect(result).toBeNull(); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" }); + expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] }); }); it("does not re-write status when already in maintenance", async () => { diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 2bd5fa74c..d4d25364a 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -557,6 +557,55 @@ describe("StatusService", () => { expect(result.prevStatus).toBe("initializing"); }); + it("leaves 'maintenance' for 'up' on a passing check after the window closes (empty statusWindow)", async () => { + // On entry to a maintenance window the producer clears statusWindow, so the first + // check after the window closes sees an empty window and lands in the warmup + // early-return branch. A passing check must move the monitor out of 'maintenance' + // to 'up' without flagging a status change (no incident). + const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + + expect(result.statusChanged).toBe(false); + expect(result.monitor.status).toBe("up"); + expect(result.prevStatus).toBe("maintenance"); + }); + + it("leaves 'maintenance' for 'down' on a failing check after the window closes (empty statusWindow)", async () => { + // Mirror of the recovery case: a failing first check after the window closes must + // flip the monitor straight to 'down' and surface statusChanged so an incident opens, + // rather than leave it stuck in 'maintenance'. + const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + + expect(result.statusChanged).toBe(true); + expect(result.monitor.status).toBe("down"); + expect(result.prevStatus).toBe("maintenance"); + }); + + it("unsticks from 'maintenance' to 'up' on a passing check even with a full window (regression: stuck in maintenance)", async () => { + // Guards the up-front override directly: even if a full window survives (e.g. the + // entry-clear is bypassed), computeReachability only transitions out to 'up' from + // 'down', so without the override the monitor would stay stuck in 'maintenance'. + const monitor = makeMonitor({ + statusWindow: [true, true, true, true, true], + statusWindowSize: 5, + statusWindowThreshold: 80, + status: "maintenance", + }); + const { service, monitorsRepository } = createService(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); + + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + + expect(result.monitor.status).toBe("up"); + }); + it("does not flip status on subsequent checks during warmup once 'initializing' has been left (regression: down→up incident resolves once window fills)", async () => { // After a 'initializing'→'down' transition on check #1, monitor.status is now 'down' // and statusWindow has one false. Subsequent up checks during warmup must leave From 08e9c02a3a006a8cdc1de5c0885911418581b28b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 19 Jun 2026 10:02:06 -0700 Subject: [PATCH 52/75] update tests --- .../test/unit/services/checkEvaluator.test.ts | 87 ++++++++++++++ .../test/unit/services/checkPipeline.test.ts | 108 ------------------ .../test/unit/services/checkProducer.test.ts | 2 +- .../test/unit/services/statusService.test.ts | 6 +- 4 files changed, 91 insertions(+), 112 deletions(-) create mode 100644 server/test/unit/services/checkEvaluator.test.ts diff --git a/server/test/unit/services/checkEvaluator.test.ts b/server/test/unit/services/checkEvaluator.test.ts new file mode 100644 index 000000000..a22ceb1c7 --- /dev/null +++ b/server/test/unit/services/checkEvaluator.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts"; +import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts"; +import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts"; + +const makeMonitor = (overrides?: Partial): Monitor => + ({ + id: "m1", + teamId: "team", + type: "http", + interval: 60000, + status: "up", + ...overrides, + }) as Monitor; + +const createEvaluator = (overrides?: Record) => { + const defaults = { + statusService: { + updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }), + }, + monitorStatusPolicy: new MonitorStatusPolicy(), + ...overrides, + }; + const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); + return { evaluator, defaults }; +}; + +describe("CheckEvaluator", () => { + it("threads status, check, and monitor through to the status service", async () => { + const { evaluator, defaults } = createEvaluator(); + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const check = { id: "check-1" } as any; + const monitor = makeMonitor({ status: "up" }); + + await evaluator.evaluate(status, check, monitor); + + expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor); + }); + + it("returns an evaluation that opens an incident on a down transition", async () => { + const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any; + const check = { id: "check-1" } as any; + const statusChange = { monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 }; + const { evaluator } = createEvaluator({ + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, + }); + + const result = await evaluator.evaluate(status, check, makeMonitor()); + + expect(result).toMatchObject({ + monitor: statusChange.monitor, + status, + check, + statusChange, + decision: expect.objectContaining({ + shouldCreateIncident: true, + shouldSendNotification: true, + incidentReason: "status_down", + }), + }); + }); + + it("returns an evaluation that resolves an incident on recovery", async () => { + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const statusChange = { monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down", code: 200 }; + const { evaluator } = createEvaluator({ + statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, + }); + + const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor({ status: "down" })); + + expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true }); + }); + + it("produces a no-op decision when status did not change", async () => { + const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any; + const { evaluator } = createEvaluator(); + + const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor()); + + expect(result.decision).toMatchObject({ + shouldCreateIncident: false, + shouldResolveIncident: false, + shouldSendNotification: false, + }); + }); +}); diff --git a/server/test/unit/services/checkPipeline.test.ts b/server/test/unit/services/checkPipeline.test.ts index 645d58787..008e879fa 100644 --- a/server/test/unit/services/checkPipeline.test.ts +++ b/server/test/unit/services/checkPipeline.test.ts @@ -20,114 +20,6 @@ const activeWindow = () => { return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 }; }; -// ── CheckPipeline ──────────────────────────────────────────────────────────── - -const createCheckPipeline = (overrides?: Record) => { - const defaults = { - logger: createMockLogger(), - networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, - statusService: { - updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }), - }, - checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, - buffer: { addToBuffer: jest.fn() }, - monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, - monitorStatusPolicy: new MonitorStatusPolicy(), - ...overrides, - }; - const checkProducer = new CheckProducer( - defaults.monitorsRepository as any, - defaults.maintenanceWindowsRepository as any, - defaults.checkService as any, - defaults.networkService as any, - defaults.buffer as any, - defaults.logger as any - ); - const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any); - const pipeline = new CheckPipeline(checkProducer, checkEvaluator); - return { pipeline, defaults }; -}; - -describe("CheckPipeline", () => { - it("throws when monitor id is missing", async () => { - const { pipeline } = createCheckPipeline(); - await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id"); - }); - - it("skips and sets maintenance status when in maintenance window", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - const result = await pipeline.run(makeMonitor({ status: "up" })); - - expect(result).toBeNull(); - expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] }); - }); - - it("does not re-write status when already in maintenance", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) }, - }); - - await pipeline.run(makeMonitor({ status: "maintenance" })); - - expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled(); - }); - - it("throws when network response is null", async () => { - const { pipeline } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, - }); - await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response"); - }); - - it("returns null and warns when toCheck returns null", async () => { - const { pipeline, defaults } = createCheckPipeline({ - checkService: { toCheck: jest.fn().mockReturnValue(null) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(result).toBeNull(); - expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") })); - expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled(); - }); - - it("buffers the check, updates status, and returns an evaluation", async () => { - const status = { monitorId: "m1", status: false, code: 500, message: "Error" }; - const statusChange = { monitor: { id: "m1", status: "down" }, statusChanged: true, prevStatus: "up", code: 500 }; - const { pipeline, defaults } = createCheckPipeline({ - networkService: { requestStatus: jest.fn().mockResolvedValue(status) }, - statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) }, - }); - - const result = await pipeline.run(makeMonitor()); - - expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith({ id: "check-1" }); - expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalled(); - expect(result).toMatchObject({ - monitor: statusChange.monitor, - status, - check: { id: "check-1" }, - statusChange, - decision: expect.objectContaining({ shouldCreateIncident: true, shouldSendNotification: true, incidentReason: "status_down" }), - }); - }); - - it("proceeds with the check when no maintenance window is active", async () => { - const { pipeline, defaults } = createCheckPipeline({ - maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) }, - }); - - await pipeline.run(makeMonitor()); - - expect(defaults.networkService.requestStatus).toHaveBeenCalled(); - }); -}); - // ── GeoChecksPipeline ────────────────────────────────────────────────────────── const createGeoPipeline = (overrides?: Record) => { diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts index bf93c8ea2..4edd12bb5 100644 --- a/server/test/unit/services/checkProducer.test.ts +++ b/server/test/unit/services/checkProducer.test.ts @@ -56,7 +56,7 @@ describe("CheckProducer", () => { const result = await producer.produce(makeMonitor({ status: "up" })); expect(result).toBeNull(); - expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" }); + expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] }); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled(); }); diff --git a/server/test/unit/services/statusService.test.ts b/server/test/unit/services/statusService.test.ts index 99db9072a..5712de301 100644 --- a/server/test/unit/services/statusService.test.ts +++ b/server/test/unit/services/statusService.test.ts @@ -564,7 +564,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.statusChanged).toBe(false); expect(result.monitor.status).toBe("up"); @@ -579,7 +579,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }), monitor); expect(result.statusChanged).toBe(true); expect(result.monitor.status).toBe("down"); @@ -599,7 +599,7 @@ describe("StatusService", () => { const { service, monitorsRepository } = createService(); (monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor); - const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true })); + const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }), monitor); expect(result.monitor.status).toBe("up"); }); From 3b919044e6b7bf3616c4965cc1be0e9e2abb3e96 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 19 Jun 2026 10:25:59 -0700 Subject: [PATCH 53/75] wake worker on job change --- server/src/config/services.ts | 1 + .../maintenance-window.service.ts | 8 ++ server/src/worker/worker.db-queue.ts | 46 ++++++++++-- server/src/worker/worker.interface.ts | 2 + .../test/unit/services/dbQueueWorker.test.ts | 74 +++++++++++++++++++ .../services/maintenanceWindowService.test.ts | 27 ++++++- 6 files changed, 149 insertions(+), 9 deletions(-) diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 9de13f27b..2f1826241 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -334,6 +334,7 @@ export const initializeServices = async ({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, + worker, }); const monitorService = new MonitorService({ worker, diff --git a/server/src/domain/maintenance-windows/maintenance-window.service.ts b/server/src/domain/maintenance-windows/maintenance-window.service.ts index aef0f9fb4..b8601321f 100644 --- a/server/src/domain/maintenance-windows/maintenance-window.service.ts +++ b/server/src/domain/maintenance-windows/maintenance-window.service.ts @@ -1,6 +1,7 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; +import { IWorker } from "@/worker/worker.interface.js"; import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js"; import { AppError } from "@/utils/AppError.js"; import { isWindowActive } from "@/utils/maintenanceWindow.js"; @@ -42,19 +43,23 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { private monitorsRepository: IMonitorsRepository; private maintenanceWindowsRepository: IMaintenanceWindowsRepository; private jobsRepository: IJobsRepository; + private worker: IWorker; constructor({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, + worker, }: { monitorsRepository: IMonitorsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; jobsRepository: IJobsRepository; + worker: IWorker; }) { this.monitorsRepository = monitorsRepository; this.maintenanceWindowsRepository = maintenanceWindowsRepository; this.jobsRepository = jobsRepository; + this.worker = worker; } get serviceName() { @@ -78,6 +83,9 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]); // Rearm jobs to run soon (now + jitter to avoid herding) await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime()); + // Wake the (possibly idle, backed-off) loops so the rearmed jobs run promptly instead of waiting up to POLL_MAX_MS + this.worker.wake("check"); + this.worker.wake("geo-check"); } }; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index bef7a838e..24138695f 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -41,6 +41,17 @@ export class DBQueueWorker implements IWorker { "cleanup-retention": 0, }; + // Per job type poll intervals. These back off on idle and reset on work or wake() + private pollMs: Record = { + check: POLL_MS, + "geo-check": POLL_MS, + evaluate: POLL_MS, + "cleanup-orphaned": POLL_MS, + "cleanup-retention": POLL_MS, + }; + private ticking: Partial> = {}; // Whether a loop is mid tick + private tickFns = new Map void>(); // Store tick fns so we can run them immediately on wake + constructor( private logger: ILogger, private jobsRepository: IJobsRepository, @@ -239,9 +250,10 @@ export class DBQueueWorker implements IWorker { }; private startLoop = (type: JobType) => { - let pollMs = POLL_MS; + this.pollMs[type] = POLL_MS; const tick = async () => { if (this.stopped) return; + this.ticking[type] = true; try { // Claim a batch sized to the free capacity const capacity = CONCURRENCY[type] - this.inFlight[type]; @@ -252,8 +264,8 @@ export class DBQueueWorker implements IWorker { this.inFlight[type]--; // job done, free the slot }); } - // Back off if no jobs founds, reset to base if jobs found - pollMs = capacity > 0 && jobs.length === 0 ? Math.min(pollMs * 2, POLL_MAX_MS) : POLL_MS; + // Back off if no jobs found, reset to base if jobs found + this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS; } catch (error: unknown) { this.logger.error({ message: error instanceof Error ? error.message : String(error), @@ -261,11 +273,26 @@ export class DBQueueWorker implements IWorker { method: `loop:${type}`, }); } - if (!this.stopped) this.timers.set(type, setTimeout(tick, pollMs)); // re-arm unless shutting down + // Clearing the flag and re-arming run synchronously (no await between), so wake() can never interleave here. + this.ticking[type] = false; + if (!this.stopped) this.timers.set(type, setTimeout(tick, this.pollMs[type])); // re-arm unless shutting down }; + this.tickFns.set(type, tick); tick(); }; + // Wake a worker loop after adding a job that's due immediately. + // Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off + wake = (type: JobType) => { + this.pollMs[type] = POLL_MS; + if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing + const tick = this.tickFns.get(type); + if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done + const pending = this.timers.get(type); + if (pending) clearTimeout(pending); // Clear out scheduled tick if any + this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick + }; + // Seed queue private reconcile = async () => { const now = Date.now(); @@ -315,7 +342,11 @@ export class DBQueueWorker implements IWorker { addJob = async (_monitorId: string, monitor: Monitor) => { const now = Date.now(); await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true)); - if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); + this.wake("check"); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { + await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); + this.wake("geo-check"); + } }; deleteJob = async (monitor: Monitor) => { await this.jobsRepository.deleteById(monitor.id); @@ -324,12 +355,15 @@ export class DBQueueWorker implements IWorker { await this.jobsRepository.setActiveById(monitor.id, false); }; resumeJob = async (monitor: Monitor) => { - await this.jobsRepository.setActiveById(monitor.id, true); + await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately + this.wake("check"); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check"); }; updateJob = async (monitor: Monitor) => { await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval); if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away + this.wake("geo-check"); } else { await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check"); } diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index ccaa79d12..fd4160124 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -1,5 +1,6 @@ import { Monitor } from "@/domain/monitors/monitor.types.js"; import { Check } from "@/domain/checks/check.type.js"; +import { JobType } from "@/domain/jobs/job.type.js"; import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js"; import { MonitorActionDecision } from "@/worker/worker.helper.js"; import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js"; @@ -70,6 +71,7 @@ export interface IWorker { pauseJob(monitor: Monitor): Promise; resumeJob(monitor: Monitor): Promise; updateJob(monitor: Monitor): Promise; + wake(type: JobType): void; shutdown(): Promise; getMetrics(): Promise; getJobs(pagination: WorkerJobsPagination): Promise; diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index 1769ed573..cf15db8c0 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -479,6 +479,80 @@ describe("DBQueueWorker", () => { }); }); + // ── poll backoff + wake() ─────────────────────────────────────────────────── + // + // An idle loop grows its poll interval up to POLL_MAX_MS (5s). wake() fast-paths + // it back so a freshly-enqueued "run now" job doesn't wait out that backoff. + + describe("poll backoff and wake", () => { + const POLL_MAX_MS = 5000; + const claims = (mocks: any, type: string) => mocks.jobsRepository.claimDueBatch.mock.calls.filter((c: any[]) => c[0] === type).length; + + it("backs off the idle poll interval toward POLL_MAX_MS", async () => { + const { mocks } = await start({ queueMode: "worker" }); // claimDueBatch returns [] → always idle + await jest.advanceTimersByTimeAsync(15000); // let the check loop grow to the cap + const before = claims(mocks, "check"); + await jest.advanceTimersByTimeAsync(POLL_MAX_MS * 2); // ~2 polls at a 5s cadence + const delta = claims(mocks, "check") - before; + // At the cap that's ~2 polls; at the un-backed-off 250ms cadence it would be ~40. + expect(delta).toBeGreaterThanOrEqual(1); + expect(delta).toBeLessThanOrEqual(4); + }); + + it("wake() re-fires an idle, backed-off loop immediately", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // back off the check loop to the cap + const before = claims(mocks, "check"); + + worker.wake("check"); + await jest.advanceTimersByTimeAsync(1); + + expect(claims(mocks, "check")).toBe(before + 1); // fired now, not 5s later + }); + + it("wake() resets the cadence so the loop polls fast again, not at the 5s cap", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // backed off to the cap + worker.wake("check"); + await jest.advanceTimersByTimeAsync(1); // immediate tick (still idle → next poll at ~POLL_MS*2) + const before = claims(mocks, "check"); + + await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms + + // Polled again well within the old 5s cap, proving the interval was reset. + expect(claims(mocks, "check")).toBe(before + 1); + }); + + it("wake() is a no-op for a loop that isn't running in this process", async () => { + const { worker, mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false }); // no loops + expect(() => worker.wake("check")).not.toThrow(); + await jest.advanceTimersByTimeAsync(1); + expect(claims(mocks, "check")).toBe(0); + }); + + it("addJob wakes the check loop so a new monitor's first check runs promptly", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // backed off to the cap + const before = claims(mocks, "check"); + + await worker.addJob("m2", makeMonitor({ id: "m2" })); + await jest.advanceTimersByTimeAsync(1); + + expect(claims(mocks, "check")).toBe(before + 1); + }); + + it("resumeJob wakes the check loop so a resumed monitor runs promptly", async () => { + const { worker, mocks } = await start({ queueMode: "worker" }); + await jest.advanceTimersByTimeAsync(15000); // backed off to the cap + const before = claims(mocks, "check"); + + await worker.resumeJob(makeMonitor()); + await jest.advanceTimersByTimeAsync(1); + + expect(claims(mocks, "check")).toBe(before + 1); + }); + }); + // ── shutdown ──────────────────────────────────────────────────────────────── describe("shutdown", () => { diff --git a/server/test/unit/services/maintenanceWindowService.test.ts b/server/test/unit/services/maintenanceWindowService.test.ts index 07d19e405..5c7387da8 100644 --- a/server/test/unit/services/maintenanceWindowService.test.ts +++ b/server/test/unit/services/maintenanceWindowService.test.ts @@ -3,6 +3,7 @@ import { MaintenanceWindowService } from "../../../src/domain/maintenance-window import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts"; +import type { IWorker } from "../../../src/worker/worker.interface.ts"; import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -33,16 +34,23 @@ const createJobsRepo = () => markMonitorsDue: jest.fn().mockResolvedValue(0), }) as unknown as jest.Mocked; +const createWorker = () => + ({ + wake: jest.fn(), + }) as unknown as jest.Mocked; + const createService = (overrides?: { monitorsRepository?: ReturnType; maintenanceWindowsRepository?: ReturnType; jobsRepository?: ReturnType; + worker?: ReturnType; }) => { const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo(); const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo(); const jobsRepository = overrides?.jobsRepository ?? createJobsRepo(); - const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository }); - return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository }; + const worker = overrides?.worker ?? createWorker(); + const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker }); + return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker }; }; const makeWindow = (overrides?: Partial): MaintenanceWindow => ({ @@ -275,7 +283,7 @@ describe("MaintenanceWindowService", () => { it("flips covered monitors to initializing when deleting an active window", async () => { const maintenanceWindowsRepository = createMaintenanceWindowsRepo(); (maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] })); - const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository }); + const { service, monitorsRepository, jobsRepository, worker } = createService({ maintenanceWindowsRepository }); await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" }); @@ -283,6 +291,19 @@ describe("MaintenanceWindowService", () => { expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]); // Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number)); + // ...and the idle loops are woken so the re-armed jobs run promptly + expect(worker.wake).toHaveBeenCalledWith("check"); + expect(worker.wake).toHaveBeenCalledWith("geo-check"); + }); + + it("does not wake the loops when no monitor leaves maintenance", async () => { + const maintenanceWindowsRepository = createMaintenanceWindowsRepo(); + (maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeWindow({ active: false, monitorIds: ["mon-1"] })); + const { service, worker } = createService({ maintenanceWindowsRepository }); + + await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" }); + + expect(worker.wake).not.toHaveBeenCalled(); }); it("excludes the deleted window from the overlap check", async () => { From 755134ee7f9c525bb28ab20935be868cf5c3f667 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 09:15:39 -0700 Subject: [PATCH 54/75] add poll backoff tests --- .../test/unit/services/dbQueueWorker.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index cf15db8c0..21515c34f 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -499,6 +499,52 @@ describe("DBQueueWorker", () => { expect(delta).toBeLessThanOrEqual(4); }); + it("resets the poll interval to POLL_MS once a tick finds work after being idle", async () => { + let busy = false; + const claimDueBatch = jest.fn(async (type: string) => (busy && type === "check" ? [makeJob()] : [])); + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch } } }); + + await jest.advanceTimersByTimeAsync(15000); // idle → backed off to the 5s cap + busy = true; + await jest.advanceTimersByTimeAsync(POLL_MAX_MS); // the next (backed-off) tick fires, finds work → resets pollMs to POLL_MS + const before = claims(mocks, "check"); + + await jest.advanceTimersByTimeAsync(POLL_MS * 2); // 500ms + + // Polling fast again (~2 in 500ms), not once per 5s — proves the interval reset on finding work. + expect(claims(mocks, "check") - before).toBeGreaterThanOrEqual(1); + }); + + it("does not back off a loop that is at capacity (capacity === 0)", async () => { + // cleanup-orphaned has concurrency 1; hold its one job in-flight so capacity stays 0. + // The backoff condition is `capacity > 0 && jobs.length === 0`, so an at-capacity loop + // must keep polling at POLL_MS (a freed slot should be picked up promptly, not after 5s). + const gate = deferred(); + let handed = false; + const claimDueBatch = jest.fn(async (type: string) => { + if (type === "cleanup-orphaned" && !handed) { + handed = true; + return [makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 })]; + } + return []; + }); + const helper = { + getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn(() => gate.promise)), // blocks → keeps the slot full + getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + }; + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, helper } }); + + const before = claims(mocks, "cleanup-orphaned"); // the blocking job is now in-flight → capacity 0 + await jest.advanceTimersByTimeAsync(1000); + const delta = claims(mocks, "cleanup-orphaned") - before; + + // ~4 polls at POLL_MS; if an at-capacity loop wrongly backed off it would be ~1. + expect(delta).toBeGreaterThanOrEqual(3); + + gate.resolve(); + await jest.advanceTimersByTimeAsync(1); // let the held job finish for a clean shutdown + }); + it("wake() re-fires an idle, backed-off loop immediately", async () => { const { worker, mocks } = await start({ queueMode: "worker" }); await jest.advanceTimersByTimeAsync(15000); // back off the check loop to the cap From 89c86a7628fd6c5128a5e9a80ce4f42587584867 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 09:53:50 -0700 Subject: [PATCH 55/75] remove dead fields from WorkerSummary --- server/src/worker/worker.db-queue.ts | 9 +-------- server/src/worker/worker.interface.ts | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 24138695f..5e1a74bb4 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -137,23 +137,16 @@ export class DBQueueWorker implements IWorker { private toSummary = (job: Job): WorkerJobSummary => ({ monitorId: job.refId ?? job.id, - monitorUrl: null, // Jobs don't have URL monitorType: job.type, monitorInterval: job.intervalMs, - monitorGeoInterval: null, monitorActive: job.isActive, - active: job.isActive, lockedBy: job.lockedBy, lockedUntil: job.lockedUntil, - lockedAt: null, // no lockedAt + nextScheduledAt: job.nextScheduledAt, runCount: job.runCount, failCount: job.failCount, failReason: job.lastFailReason, - lastRunAt: null, // not tracked lastFinishedAt: job.lastFinishedAt, - lastRunTook: null, // not tracked - lastFailedAt: null, // not tracked - repeat: job.intervalMs, }); // ******************** diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index fd4160124..6dd360e50 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -34,23 +34,16 @@ export type WorkerMetrics = { export type WorkerJobSummary = { monitorId: string | number; - monitorUrl: string | null; monitorType: string | null; monitorInterval: number | null; - monitorGeoInterval: number | null; monitorActive: boolean | null; - active: boolean; lockedBy: string | null; lockedUntil: number | null; - lockedAt: number | null; + nextScheduledAt: number; runCount: number; failCount: number; failReason: string | null; - lastRunAt: number | null; lastFinishedAt: number | null; - lastRunTook: number | null; - lastFailedAt: number | null; - repeat: number | null; }; export type WorkerJobsPagination = { From dffa49ae9d037f83e03c4b01f364daf0b75ac737 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 09:54:17 -0700 Subject: [PATCH 56/75] remove dead fields --- client/src/Types/Queue.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/client/src/Types/Queue.ts b/client/src/Types/Queue.ts index 5a9ae7cdf..c1ec49ba4 100644 --- a/client/src/Types/Queue.ts +++ b/client/src/Types/Queue.ts @@ -30,23 +30,16 @@ export type QueueMetrics = { export type QueueJobSummary = { monitorId: string | number; - monitorUrl: string | null; monitorType: string | null; monitorInterval: number | null; - monitorGeoInterval: number | null; monitorActive: boolean | null; - active: boolean; lockedBy: string | null; lockedUntil: number | null; - lockedAt: number | null; + nextScheduledAt: number; runCount: number; failCount: number; failReason: string | null; - lastRunAt: number | null; lastFinishedAt: number | null; - lastRunTook: number | null; - lastFailedAt: number | null; - repeat: number | null; }; export interface QueueData { From 99ded83bd51759781a63bcfa52c99b3250ff672a Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 09:54:49 -0700 Subject: [PATCH 57/75] remove dead fields from table, add correct fields --- .../src/Pages/Logs/components/TableJobs.tsx | 59 ++++++------------- 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/client/src/Pages/Logs/components/TableJobs.tsx b/client/src/Pages/Logs/components/TableJobs.tsx index 241b6f016..26fe62bb3 100644 --- a/client/src/Pages/Logs/components/TableJobs.tsx +++ b/client/src/Pages/Logs/components/TableJobs.tsx @@ -42,12 +42,15 @@ export const TableJobs = ({ maxWidth: 220, }; + const isRunning = (row: QueueJobWithId) => + row.lockedBy !== null && row.lockedUntil !== null && row.lockedUntil > Date.now(); + const headers: Header[] = [ { id: "state", content: t("pages.logs.table.headers.state"), render: (row) => { - if (row.lockedAt) { + if (isRunning(row)) { return t("common.labels.running"); } if (row.monitorActive === true) { @@ -72,18 +75,6 @@ export const TableJobs = ({ ), }, - { - id: "url", - content: t("common.table.headers.url"), - render: (row) => ( - - {row.monitorUrl} - - ), - }, { id: "type", content: t("common.table.headers.type"), @@ -92,50 +83,36 @@ export const TableJobs = ({ { id: "interval", content: t("common.table.headers.interval"), - render: (row) => { - let interval; - if (row.monitorId.toString().includes("geo")) { - interval = row.monitorGeoInterval ?? 0; - } else { - interval = row.repeat ?? 0; - } - return {prettyMilliseconds(interval)}; - }, + render: (row) => ( + {prettyMilliseconds(row.monitorInterval ?? 0)} + ), }, { - id: "lastRun", - content: t("pages.logs.table.headers.lastRunAt"), + id: "lastFinishedAt", + content: t("pages.logs.table.headers.lastFinishedAt"), render: (row) => { - const v = formatTimestamp(row.lastRunAt) ?? "-"; + const lastFinishedAt = formatTimestamp(row.lastFinishedAt) ?? "-"; return ( - {v} + {lastFinishedAt} ); }, }, { - id: "lastRunTook", - content: t("pages.logs.table.headers.lastRunTook"), + id: "nextScheduledAt", + content: t("pages.logs.table.headers.nextScheduledAt"), render: (row) => { - const value = row.lastRunTook ? prettyMilliseconds(row.lastRunTook) : "-"; - return {value}; - }, - }, - { - id: "lockedAt", - content: t("pages.logs.table.headers.lockedAt"), - render: (row) => { - const v = formatTimestamp(row.lockedAt) ?? "-"; + const nextScheduledAt = formatTimestamp(row.nextScheduledAt) ?? "-"; return ( - {v} + {nextScheduledAt} ); }, @@ -169,7 +146,7 @@ export const TableJobs = ({ headers={headers} data={jobsWithId} getRowSx={(row) => ({ - ...(row.lockedAt && { + ...(isRunning(row) && { "& td": { backgroundColor: theme.palette.rowStatus.running }, }), ...(row.monitorActive === false && { From 10174da4f0fa21fb57c18c0653d388c26f133607 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 09:54:58 -0700 Subject: [PATCH 58/75] update locales --- client/src/locales/ar.json | 3 --- client/src/locales/ca.json | 3 --- client/src/locales/cs.json | 3 --- client/src/locales/de.json | 3 --- client/src/locales/en.json | 4 +--- client/src/locales/es.json | 3 --- client/src/locales/fi.json | 3 --- client/src/locales/fr.json | 3 --- client/src/locales/ja.json | 3 --- client/src/locales/pt-BR.json | 3 --- client/src/locales/ru.json | 3 --- client/src/locales/th.json | 3 --- client/src/locales/tr.json | 3 --- client/src/locales/uk.json | 3 --- client/src/locales/vi.json | 3 --- client/src/locales/zh-CN.json | 3 --- client/src/locales/zh-TW.json | 3 --- 17 files changed, 1 insertion(+), 51 deletions(-) diff --git a/client/src/locales/ar.json b/client/src/locales/ar.json index 504fa5b49..6e315bd9a 100644 --- a/client/src/locales/ar.json +++ b/client/src/locales/ar.json @@ -941,10 +941,7 @@ "monitorId": "معرّف المراقب", "runCount": "عدد التشغيلات", "failCount": "عدد حالات الفشل", - "lastRunAt": "آخر تشغيل في", - "lockedAt": "مقفل في", "lastFinishedAt": "آخر انتهاء في", - "lastRunTook": "مدة آخر تشغيل", "lastFailedAt": "آخر فشل في", "failReason": "سبب الفشل", "state": "الحالة" diff --git a/client/src/locales/ca.json b/client/src/locales/ca.json index 9760e7157..3c1a29613 100644 --- a/client/src/locales/ca.json +++ b/client/src/locales/ca.json @@ -945,10 +945,7 @@ "monitorId": "ID del monitor", "runCount": "Recompte d'execucions", "failCount": "Recompte", - "lastRunAt": "Darrera execució", - "lockedAt": "Bloquejat el", "lastFinishedAt": "Darrera finalització", - "lastRunTook": "Durada de la darrera execució", "lastFailedAt": "Darrera fallada", "failReason": "Motiu de la fallada", "state": "Estat" diff --git a/client/src/locales/cs.json b/client/src/locales/cs.json index 9f7ba63d5..ae683aefe 100644 --- a/client/src/locales/cs.json +++ b/client/src/locales/cs.json @@ -941,10 +941,7 @@ "monitorId": "ID monitoru", "runCount": "Počet spuštění", "failCount": "Počet selhání", - "lastRunAt": "Poslední spuštění", - "lockedAt": "Uzamčeno v", "lastFinishedAt": "Naposledy dokončeno", - "lastRunTook": "Poslední běh trval", "lastFailedAt": "Poslední selhání", "failReason": "Důvod selhání", "state": "Stav" diff --git a/client/src/locales/de.json b/client/src/locales/de.json index 2c6a33ae4..e042f9d55 100644 --- a/client/src/locales/de.json +++ b/client/src/locales/de.json @@ -941,10 +941,7 @@ "monitorId": "Monitor-ID", "runCount": "Ausführungen", "failCount": "Fehlerzahl", - "lastRunAt": "Letzter Lauf am", - "lockedAt": "Gesperrt am", "lastFinishedAt": "Zuletzt abgeschlossen am", - "lastRunTook": "Letzter Lauf dauerte", "lastFailedAt": "Letzter Fehler am", "failReason": "Fehlergrund", "state": "Status" diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 6e65aea62..ebe9fc546 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -1025,11 +1025,9 @@ "monitorId": "Monitor ID", "runCount": "Run count", "failCount": "Count", - "lastRunAt": "Last run at", - "lockedAt": "Locked at", + "nextScheduledAt": "Next run at", "lockedBy": "Locked by", "lastFinishedAt": "Last finished at", - "lastRunTook": "Last run took", "lastFailedAt": "Last failed", "failReason": "Fail reason" } diff --git a/client/src/locales/es.json b/client/src/locales/es.json index 70dd557d0..1c45ecad4 100644 --- a/client/src/locales/es.json +++ b/client/src/locales/es.json @@ -941,10 +941,7 @@ "monitorId": "ID de monitor", "runCount": "Conteo de ejecuciones", "failCount": "Conteo de fallas", - "lastRunAt": "Última ejecución", - "lockedAt": "Bloqueado a las", "lastFinishedAt": "Última finalización", - "lastRunTook": "Última ejecución tomó", "lastFailedAt": "Última falla", "failReason": "Razón de falla", "state": "Estado" diff --git a/client/src/locales/fi.json b/client/src/locales/fi.json index 3c34577a2..3ca1de82a 100644 --- a/client/src/locales/fi.json +++ b/client/src/locales/fi.json @@ -941,10 +941,7 @@ "monitorId": "Monitorin tunnus", "runCount": "Suorituskerrat", "failCount": "Epäonnistumiskerrat", - "lastRunAt": "Viimeisin suoritus", - "lockedAt": "Lukittu", "lastFinishedAt": "Viimeisin valmistuminen", - "lastRunTook": "Viimeisin suoritus kesti", "lastFailedAt": "Viimeisin epäonnistuminen", "failReason": "Epäonnistumisen syy", "state": "Tila" diff --git a/client/src/locales/fr.json b/client/src/locales/fr.json index 49e7e94bc..9b3e983a6 100644 --- a/client/src/locales/fr.json +++ b/client/src/locales/fr.json @@ -941,10 +941,7 @@ "monitorId": "ID du moniteur", "runCount": "Nombre d'exécutions", "failCount": "Nombre d'échecs", - "lastRunAt": "Dernière exécution", - "lockedAt": "Verrouillé le", "lastFinishedAt": "Dernière fin", - "lastRunTook": "Durée dernière exécution", "lastFailedAt": "Dernier échec", "failReason": "Raison de l'échec", "state": "État" diff --git a/client/src/locales/ja.json b/client/src/locales/ja.json index eb97dc105..031350b20 100644 --- a/client/src/locales/ja.json +++ b/client/src/locales/ja.json @@ -941,10 +941,7 @@ "monitorId": "モニターID", "runCount": "実行回数", "failCount": "失敗回数", - "lastRunAt": "最終実行日時", - "lockedAt": "ロック日時", "lastFinishedAt": "最終完了日時", - "lastRunTook": "最終実行時間", "lastFailedAt": "最終失敗日時", "failReason": "失敗理由", "state": "状態" diff --git a/client/src/locales/pt-BR.json b/client/src/locales/pt-BR.json index 39cbf4577..0286f2857 100644 --- a/client/src/locales/pt-BR.json +++ b/client/src/locales/pt-BR.json @@ -941,10 +941,7 @@ "monitorId": "ID do monitor", "runCount": "Contagem de execuções", "failCount": "Contagem de falhas", - "lastRunAt": "Última execução em", - "lockedAt": "Bloqueado em", "lastFinishedAt": "Última conclusão em", - "lastRunTook": "Última execução durou", "lastFailedAt": "Última falha em", "failReason": "Motivo da falha", "state": "Estado" diff --git a/client/src/locales/ru.json b/client/src/locales/ru.json index 9bfd20481..50226037a 100644 --- a/client/src/locales/ru.json +++ b/client/src/locales/ru.json @@ -941,10 +941,7 @@ "monitorId": "ID монитора", "runCount": "Кол-во запусков", "failCount": "Кол-во ошибок", - "lastRunAt": "Последний запуск", - "lockedAt": "Заблокировано в", "lastFinishedAt": "Последнее завершение", - "lastRunTook": "Последний запуск занял", "lastFailedAt": "Последняя ошибка", "failReason": "Причина ошибки", "state": "Состояние" diff --git a/client/src/locales/th.json b/client/src/locales/th.json index 5873b65d4..c0889a72f 100644 --- a/client/src/locales/th.json +++ b/client/src/locales/th.json @@ -941,10 +941,7 @@ "monitorId": "รหัสมอนิเตอร์", "runCount": "จำนวนรัน", "failCount": "จำนวนล้มเหลว", - "lastRunAt": "รันล่าสุดเมื่อ", - "lockedAt": "ล็อกเมื่อ", "lastFinishedAt": "เสร็จสิ้นล่าสุดเมื่อ", - "lastRunTook": "รันล่าสุดใช้เวลา", "lastFailedAt": "ล้มเหลวล่าสุดเมื่อ", "failReason": "สาเหตุที่ล้มเหลว", "state": "สถานะ" diff --git a/client/src/locales/tr.json b/client/src/locales/tr.json index 2d6deaa97..c82091a83 100644 --- a/client/src/locales/tr.json +++ b/client/src/locales/tr.json @@ -941,10 +941,7 @@ "monitorId": "Monitör ID", "runCount": "Çalışma sayısı", "failCount": "Başarısızlık sayısı", - "lastRunAt": "Son çalışma zamanı", - "lockedAt": "Kilitlenme zamanı", "lastFinishedAt": "Son tamamlanma zamanı", - "lastRunTook": "Son çalışma süresi", "lastFailedAt": "Son başarısızlık zamanı", "failReason": "Başarısızlık nedeni", "state": "Durum" diff --git a/client/src/locales/uk.json b/client/src/locales/uk.json index de40bbf05..6c3b47b66 100644 --- a/client/src/locales/uk.json +++ b/client/src/locales/uk.json @@ -941,10 +941,7 @@ "monitorId": "ID монітора", "runCount": "Кількість запусків", "failCount": "Кількість помилок", - "lastRunAt": "Останній запуск", - "lockedAt": "Заблоковано о", "lastFinishedAt": "Останнє завершення", - "lastRunTook": "Останній запуск тривав", "lastFailedAt": "Остання помилка", "failReason": "Причина помилки", "state": "Стан" diff --git a/client/src/locales/vi.json b/client/src/locales/vi.json index 13907f21b..72c933601 100644 --- a/client/src/locales/vi.json +++ b/client/src/locales/vi.json @@ -941,10 +941,7 @@ "monitorId": "ID trình giám sát", "runCount": "Số lần chạy", "failCount": "Số lần lỗi", - "lastRunAt": "Chạy lần cuối lúc", - "lockedAt": "Khóa lúc", "lastFinishedAt": "Hoàn thành lần cuối lúc", - "lastRunTook": "Lần chạy cuối mất", "lastFailedAt": "Lỗi lần cuối lúc", "failReason": "Lý do lỗi", "state": "Trạng thái" diff --git a/client/src/locales/zh-CN.json b/client/src/locales/zh-CN.json index a7166e6c2..4499dd5fb 100644 --- a/client/src/locales/zh-CN.json +++ b/client/src/locales/zh-CN.json @@ -941,10 +941,7 @@ "monitorId": "监控 ID", "runCount": "运行次数", "failCount": "失败次数", - "lastRunAt": "上次运行于", - "lockedAt": "锁定于", "lastFinishedAt": "上次完成于", - "lastRunTook": "上次运行耗时", "lastFailedAt": "上次失败于", "failReason": "失败原因", "state": "状态" diff --git a/client/src/locales/zh-TW.json b/client/src/locales/zh-TW.json index ba599bb4f..7b560ae7d 100644 --- a/client/src/locales/zh-TW.json +++ b/client/src/locales/zh-TW.json @@ -941,10 +941,7 @@ "monitorId": "監控 ID", "runCount": "執行次數", "failCount": "失敗次數", - "lastRunAt": "最後執行於", - "lockedAt": "鎖定於", "lastFinishedAt": "最後完成於", - "lastRunTook": "最後執行耗時", "lastFailedAt": "最後失敗於", "failReason": "失敗原因", "state": "狀態" From 9db4e557b359c46b94a75927d703eeb67b57b0d0 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 09:58:11 -0700 Subject: [PATCH 59/75] record last finished timestamp for failures --- server/src/domain/jobs/job.repository.mongo.ts | 1 + server/test/integration/jobsRepository.test.ts | 1 + server/test/unit/services/dbQueueWorker.test.ts | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index 1dd912d10..89d8dfd04 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -127,6 +127,7 @@ class MongoJobsRepository implements IJobsRepository { nextScheduledAt: now + BACKOFF_MS, lockedBy: null, // Remove lock lockedUntil: null, + lastFinishedAt: now, // a failed run still finished used in getMetrics for failedAt timestamp lastFailReason: error instanceof Error ? error.message : String(error), }, $inc: { diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index 41781114a..02b0ecc5d 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -258,6 +258,7 @@ describe("MongoJobsRepository", () => { const row = await readRow("check:mon-1"); expect(row?.nextScheduledAt).toBe(NOW + BACKOFF_MS); expect(row?.lastFailReason).toBe("connection refused"); + expect(row?.lastFinishedAt).toBe(NOW); expect(row?.failCount).toBe(3); expect(row?.lockedBy).toBeNull(); expect(row?.lockedUntil).toBeNull(); diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index 21515c34f..66ca6753d 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -680,7 +680,7 @@ describe("DBQueueWorker", () => { expect(page.count).toBe(1); expect(page.jobs).toHaveLength(1); - expect(page.jobs[0]).toEqual(expect.objectContaining({ monitorId: "m1", monitorType: "check", monitorInterval: 60000, repeat: 60000 })); + expect(page.jobs[0]).toEqual(expect.objectContaining({ monitorId: "m1", monitorType: "check", monitorInterval: 60000 })); }); it("falls back to the job id for monitorId when refId is null (global jobs)", async () => { From 8e5757beb14e2f7f9ea1f7887f949a2f29d6c7f5 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 10:15:14 -0700 Subject: [PATCH 60/75] add a method field to model, add enums and validation for http methods --- server/src/api/validation/monitorValidation.ts | 5 +++++ server/src/domain/monitors/monitor.model.ts | 8 +++++++- server/src/domain/monitors/monitor.repository.mongo.ts | 2 ++ server/src/domain/monitors/monitor.types.ts | 4 ++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index 2dfee1239..e9e40b83e 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -3,6 +3,7 @@ import { booleanCoercion, dnsHostnameRegex, dnsServerValidation } from "./shared import { GeoContinents } from "@/domain/geo-checks/geo-check.type.js"; import { DnsRecordTypes, + HttpMethods, HttpStatusCodeSet, MonitorMatchMethods, MonitorStatuses, @@ -95,6 +96,7 @@ export const createMonitorBodyValidation = z jsonPath: z.union([z.string(), z.literal("")]).optional(), expectedValue: z.union([z.string(), z.literal("")]).optional(), matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(), + method: z.enum(HttpMethods).optional(), gameId: z.union([z.string(), z.literal("")]).optional(), grpcServiceName: z.union([z.string(), z.literal("")]).default(""), strategy: z.enum(PageSpeedStrategies).optional(), @@ -127,6 +129,7 @@ export const editMonitorBodyValidation = z jsonPath: z.union([z.string(), z.literal("")]).optional(), expectedValue: z.union([z.string(), z.literal("")]).optional(), matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(), + method: z.enum(HttpMethods).optional(), port: z.number().min(1).max(65535).optional(), cpuAlertThreshold: z.number().optional(), memoryAlertThreshold: z.number().optional(), @@ -184,6 +187,7 @@ const importedMonitorSchema = z jsonPath: z.union([z.string(), z.literal("")]).optional(), expectedValue: z.union([z.string(), z.literal("")]).optional(), matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(), + method: z.enum(HttpMethods).optional(), url: z.string().min(1, "URL is required"), port: z.number().optional(), isActive: z.boolean().default(true), @@ -251,6 +255,7 @@ export const monitorResponseSchema = z jsonPath: z.string().optional(), expectedValue: z.string().optional(), matchMethod: z.enum(MonitorMatchMethods).optional(), + method: z.enum(HttpMethods), notifications: z.array(z.string()), tags: z.array(z.string()), customUpCodes: z.array(httpStatusCode).optional(), diff --git a/server/src/domain/monitors/monitor.model.ts b/server/src/domain/monitors/monitor.model.ts index 417599d2c..8a095658c 100644 --- a/server/src/domain/monitors/monitor.model.ts +++ b/server/src/domain/monitors/monitor.model.ts @@ -1,6 +1,6 @@ import { Schema, model, Types } from "mongoose"; import type { Monitor, MonitorMatchMethod, CheckSnapshot } from "@/domain/monitors/monitor.types.js"; -import { DnsRecordTypes, MonitorTypes, MonitorStatuses, PageSpeedStrategies } from "@/domain/monitors/monitor.types.js"; +import { DnsRecordTypes, MonitorTypes, MonitorStatuses, PageSpeedStrategies, HttpMethods } from "@/domain/monitors/monitor.types.js"; import type { CheckAudits, CheckCaptureInfo, @@ -221,6 +221,12 @@ const MonitorSchema = new Schema( description: { type: String, }, + method: { + type: String, + enum: HttpMethods, + default: "GET", + required: true, + }, status: { type: String, enum: MonitorStatuses, diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index 0ce0d48a6..d55fdd62d 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -536,6 +536,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { teamId: toStringId(doc.teamId), name: doc.name, description: doc.description ?? undefined, + method: doc.method, status: doc.status ?? "initializing", statusWindow: doc.statusWindow ?? [], statusWindowSize: doc.statusWindowSize, @@ -602,6 +603,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { teamId: toStringId(doc.teamId), name: doc.name, description: doc.description ?? undefined, + method: doc.method, status: doc.status ?? "initializing", statusWindow: doc.statusWindow ?? [], statusWindowSize: doc.statusWindowSize, diff --git a/server/src/domain/monitors/monitor.types.ts b/server/src/domain/monitors/monitor.types.ts index b614d3191..22090f68c 100644 --- a/server/src/domain/monitors/monitor.types.ts +++ b/server/src/domain/monitors/monitor.types.ts @@ -62,6 +62,9 @@ export type MonitorMatchMethod = (typeof MonitorMatchMethods)[number] | ""; export const DnsRecordTypes = ["A", "AAAA", "CNAME", "MX", "TXT", "NS"] as const; export type DnsRecordType = (typeof DnsRecordTypes)[number]; +export const HttpMethods = ["GET", "HEAD"] as const; +export type HttpMethod = (typeof HttpMethods)[number]; + export const MAX_RECENT_CHECKS = 50; export interface Monitor { @@ -70,6 +73,7 @@ export interface Monitor { teamId: string; name: string; description?: string; + method: HttpMethod; status: MonitorStatus; statusWindow: boolean[]; statusWindowSize: number; From 32ec909bf7b131f32dc12d226cd35311c6f4277f Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 10:25:27 -0700 Subject: [PATCH 61/75] add default GET for imported monitors --- server/src/api/validation/monitorValidation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index e9e40b83e..e1fa13685 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -187,7 +187,7 @@ const importedMonitorSchema = z jsonPath: z.union([z.string(), z.literal("")]).optional(), expectedValue: z.union([z.string(), z.literal("")]).optional(), matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(), - method: z.enum(HttpMethods).optional(), + method: z.enum(HttpMethods).optional().default("GET"), url: z.string().min(1, "URL is required"), port: z.number().optional(), isActive: z.boolean().default(true), From 93b9c5033adb3e3dded2cbbe9c3e2c4027b5f507 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 10:26:10 -0700 Subject: [PATCH 62/75] add http metod option, add monitor.method guard to buildResponse --- server/src/service/network/HttpProvider.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/server/src/service/network/HttpProvider.ts b/server/src/service/network/HttpProvider.ts index 9f638dbd6..2f08dab00 100644 --- a/server/src/service/network/HttpProvider.ts +++ b/server/src/service/network/HttpProvider.ts @@ -53,6 +53,22 @@ export class HttpProvider implements IStatusProvider { } ): MonitorStatusResponse { const { body, contentType, statusCode, statusUp, message, responseTime, timings } = opts; + + // Return early for HEAD requests, no body to parse + if (monitor.method === "HEAD") { + return { + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: statusUp, + code: statusCode, + message, + responseTime, + timings, + payload: body as T | string, + }; + } + const isJson = contentType.includes("application/json"); if (monitor.jsonPath && !isJson) { @@ -155,6 +171,8 @@ export class HttpProvider implements IStatusProvider { https: ignoreTlsErrors ? this.httpsAgentInsecure : this.httpsAgent, }; + options.method = monitor.method; + try { const response = await this.got(url, options); const statusUp = isStatusUp(response.statusCode, monitor.customUpCodes); From 3489ee000dc946efbfba54e1340ad4e34eb11ec0 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 10:31:13 -0700 Subject: [PATCH 63/75] add super refine to reject method HEAD and advanced matching/json path --- .../src/api/validation/monitorValidation.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index e1fa13685..6dea5b856 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -71,6 +71,16 @@ const refineStrategyType = (body: { type?: string; strategy?: string }, ctx: z.R } }; +const refineHeadMatching = (body: { method?: string; useAdvancedMatching?: boolean; jsonPath?: string }, ctx: z.RefinementCtx) => { + if (body.method === "HEAD" && (body.useAdvancedMatching === true || (body.jsonPath ?? "") !== "")) { + ctx.addIssue({ + code: "custom", + path: ["method"], + message: "HEAD requests have no response body, so they cannot use advanced matching or a JSON path", + }); + } +}; + export const createMonitorBodyValidation = z .object({ _id: z.string().optional(), @@ -109,7 +119,8 @@ export const createMonitorBodyValidation = z dnsRecordType: z.enum(DnsRecordTypes).optional(), }) .superRefine(refineDnsHostname) - .superRefine(refineStrategyType); + .superRefine(refineStrategyType) + .superRefine(refineHeadMatching); export const editMonitorBodyValidation = z .object({ @@ -147,7 +158,8 @@ export const editMonitorBodyValidation = z dnsRecordType: z.enum(DnsRecordTypes).optional(), }) .superRefine(refineDnsHostname) - .superRefine(refineStrategyType); + .superRefine(refineStrategyType) + .superRefine(refineHeadMatching); export const pauseMonitorParamValidation = z.object({ monitorId: z.string().min(1, "Monitor ID is required"), @@ -219,7 +231,8 @@ const importedMonitorSchema = z updatedAt: z.string().optional(), }) .superRefine(refineDnsHostname) - .superRefine(refineStrategyType); + .superRefine(refineStrategyType) + .superRefine(refineHeadMatching); export const importMonitorsBodyValidation = z.object({ monitors: z.array(importedMonitorSchema).min(1, "At least one monitor is required"), From 71a4fddffdce3d334874d1ebd6cbfe18da2d9786 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 10:38:15 -0700 Subject: [PATCH 64/75] add tests --- .../providers/network/httpProvider.test.ts | 61 +++++++++++++++++++ .../unit/validation/monitorValidation.test.ts | 51 ++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/server/test/unit/providers/network/httpProvider.test.ts b/server/test/unit/providers/network/httpProvider.test.ts index 10b66a93a..7556f7595 100644 --- a/server/test/unit/providers/network/httpProvider.test.ts +++ b/server/test/unit/providers/network/httpProvider.test.ts @@ -433,4 +433,65 @@ describe("HttpProvider", () => { expect(result.extracted).toBe("down"); }); }); + + // ── HTTP method (GET / HEAD) ───────────────────────────────────────── + + describe("request method", () => { + it("passes GET to got when method is GET", async () => { + mockGot.mockResolvedValue(makeGotResponse()); + const { provider } = createProvider(); + + await provider.handle(makeMonitor({ method: "GET" })); + + expect(mockGot).toHaveBeenCalledWith("https://example.com", expect.objectContaining({ method: "GET" })); + }); + + it("passes HEAD to got when method is HEAD", async () => { + mockGot.mockResolvedValue(makeGotResponse({ body: "" })); + const { provider } = createProvider(); + + await provider.handle(makeMonitor({ method: "HEAD" })); + + expect(mockGot).toHaveBeenCalledWith("https://example.com", expect.objectContaining({ method: "HEAD" })); + }); + + it("does not mark a HEAD monitor down on an empty body even with advanced matching on", async () => { + mockGot.mockResolvedValue(makeGotResponse({ body: "", headers: {} })); + // A matcher that would fail — proving HEAD never reaches it + const matcher = createMockMatcher({ ok: false, message: "Extracted value is falsy" }); + const { provider, advancedMatcher } = createProvider(matcher); + + const result = await provider.handle(makeMonitor({ method: "HEAD", useAdvancedMatching: true, jsonPath: "status" })); + + expect(result.status).toBe(true); + expect(advancedMatcher.validate).not.toHaveBeenCalled(); + }); + + it("bases HEAD status on the status code (down when not an up code)", async () => { + const err = new HTTPError("Internal Server Error"); + (err as any).response = { statusCode: 500, body: "", headers: {} }; + (err as any).timings = { phases: { total: 40 } }; + mockGot.mockRejectedValue(err); + const { provider } = createProvider(); + + const result = await provider.handle(makeMonitor({ method: "HEAD" })); + + expect(result.status).toBe(false); + expect(result.code).toBe(500); + }); + + it("respects customUpCodes for HEAD requests without consulting the matcher", async () => { + const err = new HTTPError("Unauthorized"); + (err as any).response = { statusCode: 401, body: "", headers: {} }; + (err as any).timings = { phases: { total: 30 } }; + mockGot.mockRejectedValue(err); + const { provider, advancedMatcher } = createProvider(); + + const result = await provider.handle(makeMonitor({ method: "HEAD", customUpCodes: [401], useAdvancedMatching: true })); + + expect(result.status).toBe(true); + expect(result.code).toBe(401); + expect(advancedMatcher.validate).not.toHaveBeenCalled(); + }); + }); }); diff --git a/server/test/unit/validation/monitorValidation.test.ts b/server/test/unit/validation/monitorValidation.test.ts index 08f516f99..d7a68b00b 100644 --- a/server/test/unit/validation/monitorValidation.test.ts +++ b/server/test/unit/validation/monitorValidation.test.ts @@ -227,6 +227,57 @@ describe("monitorValidation — strategy gating", () => { }); }); +describe("monitorValidation — HEAD method gating", () => { + const baseHead = { + name: "HEAD check", + type: "http" as const, + url: "https://example.com", + method: "HEAD" as const, + }; + + describe("createMonitorBodyValidation", () => { + it("accepts a HEAD monitor with no body matching", () => { + const parsed = createMonitorBodyValidation.parse(baseHead); + expect(parsed.method).toBe("HEAD"); + }); + + it("accepts GET together with advanced matching", () => { + const parsed = createMonitorBodyValidation.parse({ + ...baseHead, + method: "GET", + useAdvancedMatching: true, + jsonPath: "status", + }); + expect(parsed.method).toBe("GET"); + }); + + it("rejects HEAD combined with advanced matching", () => { + expect(() => createMonitorBodyValidation.parse({ ...baseHead, useAdvancedMatching: true })).toThrow(); + }); + + it("rejects HEAD combined with a non-empty jsonPath", () => { + expect(() => createMonitorBodyValidation.parse({ ...baseHead, jsonPath: "status" })).toThrow(); + }); + + it("allows HEAD with an empty-string jsonPath", () => { + const parsed = createMonitorBodyValidation.parse({ ...baseHead, jsonPath: "" }); + expect(parsed.method).toBe("HEAD"); + }); + }); + + describe("editMonitorBodyValidation", () => { + it("rejects HEAD combined with advanced matching", () => { + expect(() => editMonitorBodyValidation.parse({ method: "HEAD", useAdvancedMatching: true })).toThrow(); + }); + }); + + describe("importMonitorsBodyValidation", () => { + it("rejects HEAD combined with a jsonPath on imported monitors", () => { + expect(() => importMonitorsBodyValidation.parse({ monitors: [{ ...baseHead, jsonPath: "status" }] })).toThrow(); + }); + }); +}); + describe("monitorValidation — customUpCodes", () => { const baseHttpBody = { name: "HTTP check", From 8afd67a06d1e95b8a0c84dfb8ad171d6f4e159d3 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 10:43:30 -0700 Subject: [PATCH 65/75] add method to monitor type --- client/src/Types/Monitor.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index b2d90cb0b..fd9b521d5 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -48,6 +48,10 @@ export const PageSpeedStrategies = ["desktop", "mobile"] as const; export type PageSpeedStrategy = (typeof PageSpeedStrategies)[number]; export const DefaultPageSpeedStrategy: PageSpeedStrategy = "desktop"; +export const HttpMethods = ["GET", "HEAD"] as const; +export type HttpMethod = (typeof HttpMethods)[number]; +export const DefaultHttpMethod: HttpMethod = "GET"; + export interface Monitor { id: string; userId: string; @@ -64,6 +68,7 @@ export interface Monitor { jsonPath?: string; expectedValue?: string; matchMethod?: MonitorMatchMethod; + method?: HttpMethod; url: string; port?: number; isActive: boolean; From d84fc7e299cd68d667feb7157dc79ac9a20c2945 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 10:48:39 -0700 Subject: [PATCH 66/75] add method to validation --- client/src/Validation/monitor.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index e0c4eae07..a31cdb919 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -1,7 +1,12 @@ import { z } from "zod"; import type { FieldPath } from "react-hook-form"; import { GeoContinents } from "@/Types/GeoCheck"; -import { DnsRecordTypes, PageSpeedStrategies, type MonitorType } from "@/Types/Monitor"; +import { + DnsRecordTypes, + HttpMethods, + PageSpeedStrategies, + type MonitorType, +} from "@/Types/Monitor"; import { ALL_HTTP_STATUS_CODES } from "@/Utils/statusCode"; // Wizard step a field is validated on. Attached inline to each field below so @@ -62,6 +67,7 @@ const httpStatusCode = z.number().refine((code) => httpStatusCodeSet.has(code), const httpSchema = baseSchema.extend({ type: z.literal("http"), url: urlSchema, + method: z.enum(HttpMethods).optional().register(monitorStepRegistry, { step: 2 }), ignoreTlsErrors: z.boolean().register(monitorStepRegistry, { step: 2 }), useAdvancedMatching: z.boolean().register(monitorStepRegistry, { step: 2 }), matchMethod: z From a31dd2f19ea5610493d2caaaadad78a729c324f6 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 11:07:56 -0700 Subject: [PATCH 67/75] update montior form hook to load method --- client/src/Hooks/useMonitorForm.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 0ae463911..9c6200d22 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -1,6 +1,6 @@ import { useMemo } from "react"; import { monitorSchema, type MonitorFormData } from "@/Validation/monitor"; -import { DefaultPageSpeedStrategy } from "@/Types/Monitor"; +import { DefaultHttpMethod, DefaultPageSpeedStrategy } from "@/Types/Monitor"; import type { Monitor, MonitorType } from "@/Types/Monitor"; interface UseMonitorFormOptions { @@ -40,6 +40,7 @@ export const getMonitorDefaults = ( ...getGeoCheckDefaults(data), type: "http", url: data?.url || "", + method: data?.method ?? DefaultHttpMethod, ignoreTlsErrors: data?.ignoreTlsErrors || false, useAdvancedMatching: data?.useAdvancedMatching || false, matchMethod: data?.matchMethod || "", @@ -134,6 +135,7 @@ export const getMonitorDefaults = ( ...getGeoCheckDefaults(data), type: "http", url: "", + method: DefaultHttpMethod, ignoreTlsErrors: false, useAdvancedMatching: false, matchMethod: "", From dead97169fccc0374da723e15f0f6910d75b4c54 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 11:24:03 -0700 Subject: [PATCH 68/75] add default GET --- server/package-lock.json | 40 +++++++++++++++++-- .../monitors/monitor.repository.mongo.ts | 4 +- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/server/package-lock.json b/server/package-lock.json index 0fbe45e08..61dd35875 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -1554,9 +1554,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", "optional": true, @@ -5200,6 +5200,40 @@ "node": ">=14.0.0" } }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz", diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index d55fdd62d..cba542c96 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -536,7 +536,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { teamId: toStringId(doc.teamId), name: doc.name, description: doc.description ?? undefined, - method: doc.method, + method: doc.method ?? "GET", status: doc.status ?? "initializing", statusWindow: doc.statusWindow ?? [], statusWindowSize: doc.statusWindowSize, @@ -603,7 +603,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { teamId: toStringId(doc.teamId), name: doc.name, description: doc.description ?? undefined, - method: doc.method, + method: doc.method ?? "GET", status: doc.status ?? "initializing", statusWindow: doc.statusWindow ?? [], statusWindowSize: doc.statusWindowSize, From ba943e90c22bc05e71e6b869df07671d762b0c6a Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 22 Jun 2026 11:24:35 -0700 Subject: [PATCH 69/75] add form UI --- client/src/Pages/CreateMonitor/index.tsx | 96 ++++++++++++++----- .../Components/UptimeMonitorsTable.tsx | 4 +- client/src/locales/en.json | 7 ++ 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 465b9d15d..96b05be71 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -42,8 +42,11 @@ import { type Monitor, type MonitorType, type GamesMap, + type HttpMethod, supportsGeoCheck, DefaultPageSpeedStrategy, + DefaultHttpMethod, + HttpMethods, } from "@/Types/Monitor"; import type { Notification } from "@/Types/Notification"; import type { Tag } from "@/Types/Tag"; @@ -259,7 +262,7 @@ const CreateMonitorPage = () => { resolver: zodResolver(schema), defaultValues: defaults, }); - const { control, watch, handleSubmit, clearErrors, trigger, reset } = form; + const { control, watch, handleSubmit, clearErrors, trigger, reset, setValue } = form; useEffect(() => { reset(defaults); @@ -271,6 +274,7 @@ const CreateMonitorPage = () => { const showStep = (step: number) => isEditMode || currentStep === step; const watchedType = watch("type") as MonitorType; + const watchedMethod = watch("method") as HttpMethod | undefined; const watchedUseAdvancedMatching = watch("useAdvancedMatching") as boolean; const watchGeoCheckEnabled = watch("geoCheckEnabled") as boolean; @@ -1091,6 +1095,50 @@ const CreateMonitorPage = () => { subtitle={t("pages.createMonitor.form.advanced.description")} rightContent={ + ( + + + + {t( + `pages.createMonitor.form.advanced.option.method.description.${field.value ?? DefaultHttpMethod}` + )} + + + )} + /> { ); }} /> - ( - - field.onChange(e.target.checked)} - /> - - {t( - "pages.createMonitor.form.advanced.option.advancedMatching.label" - )} - - - )} - /> - {watchedUseAdvancedMatching && ( + {watchedMethod !== "HEAD" && ( + ( + + field.onChange(e.target.checked)} + /> + + {t( + "pages.createMonitor.form.advanced.option.advancedMatching.label" + )} + + + )} + /> + )} + {watchedUseAdvancedMatching && watchedMethod !== "HEAD" && ( ), render: (row) => { - return row.type; + return row.type === "http" && row.method + ? `${row.type} (${row.method})` + : row.type; }, }, { diff --git a/client/src/locales/en.json b/client/src/locales/en.json index 6e65aea62..2e6c7db2c 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -490,6 +490,13 @@ "advanced": { "description": "Optional settings for advanced use cases", "option": { + "method": { + "label": "Request method", + "description": { + "GET": "GET fetches the full response including the body. Advanced matching fully supported", + "HEAD": "HEAD fetches only response headers for reducing load on the target. Supports status code matching but not advanced matching as there is no body." + } + }, "advancedMatching": { "label": "Use advanced matching" }, From c9b8f434cd5c880c3351db76a16dd91a8750b8e7 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 23 Jun 2026 11:22:37 -0700 Subject: [PATCH 70/75] split worker into a scheduler and worker --- server/src/api/controllers/queueController.ts | 17 +- server/src/config/services.ts | 10 +- .../maintenance-window.service.ts | 14 +- server/src/domain/monitors/monitor.service.ts | 32 ++-- server/src/domain/users/user.service.ts | 12 +- server/src/worker/worker.db-queue.ts | 152 ++-------------- server/src/worker/worker.interface.ts | 10 +- server/src/worker/worker.job-scheduler.ts | 167 ++++++++++++++++++ 8 files changed, 227 insertions(+), 187 deletions(-) create mode 100644 server/src/worker/worker.job-scheduler.ts diff --git a/server/src/api/controllers/queueController.ts b/server/src/api/controllers/queueController.ts index 0d3e2d1d0..0a21761ab 100755 --- a/server/src/api/controllers/queueController.ts +++ b/server/src/api/controllers/queueController.ts @@ -1,4 +1,4 @@ -import { IWorker } from "@/worker/worker.interface.js"; +import { IJobScheduler } from "@/worker/worker.interface.js"; import { getQueueJobsQueryValidation } from "@/api/validation/queueValidation.js"; import { Request, Response, NextFunction } from "express"; @@ -13,10 +13,7 @@ export interface IJobQueueController { class JobQueueController implements IJobQueueController { static SERVICE_NAME = SERVICE_NAME; - private worker: IWorker; - constructor(worker: IWorker) { - this.worker = worker; - } + constructor(private scheduler: IJobScheduler) {} get serviceName() { return JobQueueController.SERVICE_NAME; @@ -24,7 +21,7 @@ class JobQueueController implements IJobQueueController { getMetrics = async (req: Request, res: Response, next: NextFunction) => { try { - const metrics = await this.worker.getMetrics(); + const metrics = await this.scheduler.getMetrics(); res.status(200).json({ success: true, msg: "Queue metrics fetched successfully", @@ -38,7 +35,7 @@ class JobQueueController implements IJobQueueController { getJobs = async (req: Request, res: Response, next: NextFunction) => { try { const pagination = getQueueJobsQueryValidation.parse(req.query); - const { jobs, count } = await this.worker.getJobs(pagination); + const { jobs, count } = await this.scheduler.getJobs(pagination); return res.status(200).json({ success: true, msg: "Queue jobs fetched successfully", @@ -52,8 +49,8 @@ class JobQueueController implements IJobQueueController { getAllMetrics = async (req: Request, res: Response, next: NextFunction) => { try { const pagination = getQueueJobsQueryValidation.parse(req.query); - const { jobs, count } = await this.worker.getJobs(pagination); - const metrics = await this.worker.getMetrics(); + const { jobs, count } = await this.scheduler.getJobs(pagination); + const metrics = await this.scheduler.getMetrics(); return res.status(200).json({ success: true, msg: "Queue metrics fetched successfully", @@ -66,7 +63,7 @@ class JobQueueController implements IJobQueueController { flushQueue = async (req: Request, res: Response, next: NextFunction) => { try { - const result = await this.worker.flushQueues(); + const result = await this.scheduler.flushQueues(); return res.status(200).json({ success: true, msg: "Queue flushed successfully", diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 2f1826241..7bc59f881 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -31,7 +31,7 @@ import { BufferService, IBufferService } from "@/service/bufferService.js"; import { EmailService, IEmailService } from "@/service/emailService.js"; import { GlobalPingService } from "@/service/globalPingService.js"; import { WorkerHelper } from "@/worker/worker.helper.js"; -import { IWorker } from "@/worker/worker.interface.js"; +import { IQueueWorker } from "@/worker/worker.interface.js"; import { INetworkService, NetworkService } from "@/service/networkService.js"; import { IStatusService, StatusService } from "@/service/statusService.js"; import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; @@ -117,7 +117,7 @@ export type InitializedServices = { emailService: IEmailService; bufferService: IBufferService; statusService: IStatusService; - worker: IWorker; + worker: IQueueWorker; userService: IUserService; checkService: ICheckService; geoChecksService: IGeoChecksService; @@ -315,7 +315,7 @@ export const initializeServices = async ({ settingsService, logger, jwt, - worker, + scheduler: worker, monitorsRepository, usersRepository, invitesRepository, @@ -334,10 +334,10 @@ export const initializeServices = async ({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, - worker, + scheduler: worker, }); const monitorService = new MonitorService({ - worker, + scheduler: worker, logger, games, monitorsRepository, diff --git a/server/src/domain/maintenance-windows/maintenance-window.service.ts b/server/src/domain/maintenance-windows/maintenance-window.service.ts index b8601321f..5551aef62 100644 --- a/server/src/domain/maintenance-windows/maintenance-window.service.ts +++ b/server/src/domain/maintenance-windows/maintenance-window.service.ts @@ -1,10 +1,10 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; -import { IWorker } from "@/worker/worker.interface.js"; import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js"; import { AppError } from "@/utils/AppError.js"; import { isWindowActive } from "@/utils/maintenanceWindow.js"; +import { IJobScheduler } from "@/worker/worker.interface.js"; const SERVICE_NAME = "maintenanceWindowService"; @@ -43,23 +43,23 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { private monitorsRepository: IMonitorsRepository; private maintenanceWindowsRepository: IMaintenanceWindowsRepository; private jobsRepository: IJobsRepository; - private worker: IWorker; + private scheduler: IJobScheduler; constructor({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, - worker, + scheduler, }: { monitorsRepository: IMonitorsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; jobsRepository: IJobsRepository; - worker: IWorker; + scheduler: IJobScheduler; }) { this.monitorsRepository = monitorsRepository; this.maintenanceWindowsRepository = maintenanceWindowsRepository; this.jobsRepository = jobsRepository; - this.worker = worker; + this.scheduler = scheduler; } get serviceName() { @@ -84,8 +84,8 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { // Rearm jobs to run soon (now + jitter to avoid herding) await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime()); // Wake the (possibly idle, backed-off) loops so the rearmed jobs run promptly instead of waiting up to POLL_MAX_MS - this.worker.wake("check"); - this.worker.wake("geo-check"); + this.scheduler.wake("check"); + this.scheduler.wake("geo-check"); } }; diff --git a/server/src/domain/monitors/monitor.service.ts b/server/src/domain/monitors/monitor.service.ts index 8fce77723..455a1daff 100644 --- a/server/src/domain/monitors/monitor.service.ts +++ b/server/src/domain/monitors/monitor.service.ts @@ -21,8 +21,8 @@ import type { IStatusPagesRepository } from "@/domain/status-pages/status-page-r import demoMonitorsData from "@/utils/demoMonitors.json" with { type: "json" }; import { AppError } from "@/utils/AppError.js"; import type { ImportedMonitor } from "@/api/validation/monitorValidation.js"; -import { IWorker } from "@/worker/worker.interface.js"; import { ILogger } from "@/utils/logger.js"; +import { IJobScheduler } from "@/worker/worker.interface.js"; const SERVICE_NAME = "MonitorService"; type DateRangeKey = "recent" | "day" | "week" | "month" | "all"; @@ -94,7 +94,7 @@ export interface IMonitorService { export class MonitorService implements IMonitorService { static SERVICE_NAME = SERVICE_NAME; - private worker: IWorker; + private scheduler: IJobScheduler; private logger: ILogger; private games: GamesMap; private monitorsRepository: IMonitorsRepository; @@ -105,7 +105,7 @@ export class MonitorService implements IMonitorService { private incidentsRepository: IIncidentsRepository; constructor({ - worker, + scheduler, logger, games, monitorsRepository, @@ -115,7 +115,7 @@ export class MonitorService implements IMonitorService { statusPagesRepository, incidentsRepository, }: { - worker: IWorker; + scheduler: IJobScheduler; logger: ILogger; games: GamesMap; monitorsRepository: IMonitorsRepository; @@ -125,7 +125,7 @@ export class MonitorService implements IMonitorService { statusPagesRepository: IStatusPagesRepository; incidentsRepository: IIncidentsRepository; }) { - this.worker = worker; + this.scheduler = scheduler; this.logger = logger; this.games = games; this.monitorsRepository = monitorsRepository; @@ -171,7 +171,7 @@ export class MonitorService implements IMonitorService { throw new AppError({ message: "Failed to create monitor", status: 500, service: SERVICE_NAME, method: "createMonitor" }); } - this.worker.addJob(monitor.id, monitor); + this.scheduler.addJob(monitor.id, monitor); }; createMonitors = async (monitors: Array): Promise => { @@ -180,7 +180,7 @@ export class MonitorService implements IMonitorService { throw new AppError({ message: "Failed to create monitors", status: 500, service: SERVICE_NAME, method: "createMonitors" }); } - await Promise.all(createdMonitors.map((monitor) => this.worker.addJob(monitor.id, monitor))); + await Promise.all(createdMonitors.map((monitor) => this.scheduler.addJob(monitor.id, monitor))); return createdMonitors; }; @@ -196,7 +196,7 @@ export class MonitorService implements IMonitorService { })); const demoMonitors = await this.monitorsRepository.createMonitors(monitors as unknown as Monitor[]); - await Promise.all(demoMonitors.map((monitor) => this.worker.addJob(monitor.id, monitor))); + await Promise.all(demoMonitors.map((monitor) => this.scheduler.addJob(monitor.id, monitor))); return demoMonitors; }; @@ -428,7 +428,7 @@ export class MonitorService implements IMonitorService { editMonitor = async ({ teamId, monitorId, body }: { teamId: string; monitorId: string; body: Partial }) => { const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body); - await this.worker.updateJob(editedMonitor); + await this.scheduler.updateJob(editedMonitor); return editedMonitor; }; @@ -448,7 +448,7 @@ export class MonitorService implements IMonitorService { // If notifications were updated, we should update the jobs in the queue if (modifiedCount > 0) { const monitors = await this.monitorsRepository.findByIds(monitorIds); - await Promise.all(monitors.map((monitor) => this.worker.updateJob(monitor))); + await Promise.all(monitors.map((monitor) => this.scheduler.updateJob(monitor))); } return modifiedCount; @@ -457,9 +457,9 @@ export class MonitorService implements IMonitorService { pauseMonitor = async ({ teamId, monitorId }: { teamId: string; monitorId: string }): Promise => { const monitor = await this.monitorsRepository.togglePauseById(monitorId, teamId); if (monitor.isActive) { - await this.worker.resumeJob(monitor); + await this.scheduler.resumeJob(monitor); } else { - await this.worker.pauseJob(monitor); + await this.scheduler.pauseJob(monitor); } return monitor; }; @@ -478,9 +478,9 @@ export class MonitorService implements IMonitorService { const results = await Promise.allSettled( monitors.map(async (monitor) => { if (monitor.isActive) { - await this.worker.resumeJob(monitor); + await this.scheduler.resumeJob(monitor); } else { - await this.worker.pauseJob(monitor); + await this.scheduler.pauseJob(monitor); } }) ); @@ -541,7 +541,7 @@ export class MonitorService implements IMonitorService { }); }); - await this.worker.deleteJob(monitor); + await this.scheduler.deleteJob(monitor); return monitor; }; @@ -550,7 +550,7 @@ export class MonitorService implements IMonitorService { await Promise.all( monitors.map(async (monitor) => { try { - await this.worker.deleteJob(monitor); + await this.scheduler.deleteJob(monitor); await this.checksRepository.deleteByMonitorId(monitor.id); await this.geoChecksRepository.deleteByMonitorId(monitor.id); await this.statusPagesRepository.removeMonitorFromStatusPages(monitor.id); diff --git a/server/src/domain/users/user.service.ts b/server/src/domain/users/user.service.ts index 33d95f166..7bdb9677f 100644 --- a/server/src/domain/users/user.service.ts +++ b/server/src/domain/users/user.service.ts @@ -11,9 +11,9 @@ import { AppError } from "@/utils/AppError.js"; import { IEmailService } from "@/service/emailService.js"; import { EnvConfig, ISettingsService } from "@/domain/app-settings/app-settings.service.js"; import { ILogger } from "@/utils/logger.js"; +import { IJobScheduler } from "@/worker/worker.interface.js"; import jwt from "jsonwebtoken"; import crypto from "crypto"; -import { IWorker } from "@/worker/worker.interface.js"; type CryptoType = typeof crypto; type JwtType = typeof jwt; const SERVICE_NAME = "userService"; @@ -53,7 +53,7 @@ export class UserService implements IUserService { private settingsService: ISettingsService; private logger: ILogger; private jwt: JwtType; - private worker: IWorker; + private scheduler: IJobScheduler; private crypto: CryptoType; private monitorsRepository: IMonitorsRepository; private usersRepository: IUsersRepository; @@ -68,7 +68,7 @@ export class UserService implements IUserService { settingsService, logger, jwt, - worker, + scheduler, monitorsRepository, usersRepository, invitesRepository, @@ -81,7 +81,7 @@ export class UserService implements IUserService { settingsService: ISettingsService; logger: ILogger; jwt: JwtType; - worker: IWorker; + scheduler: IJobScheduler; monitorsRepository: IMonitorsRepository; usersRepository: IUsersRepository; invitesRepository: IInvitesRepository; @@ -93,7 +93,7 @@ export class UserService implements IUserService { this.settingsService = settingsService; this.logger = logger; this.jwt = jwt; - this.worker = worker; + this.scheduler = scheduler; this.crypto = crypto; this.monitorsRepository = monitorsRepository; this.usersRepository = usersRepository; @@ -330,7 +330,7 @@ export class UserService implements IUserService { if (roles.includes("superadmin")) { const monitors = await this.monitorsRepository.findByTeamId(teamId, {}); if (monitors) { - await Promise.all(monitors.map((monitor) => this.worker.deleteJob(monitor))); + await Promise.all(monitors.map((monitor) => this.scheduler.deleteJob(monitor))); } } // 6. Delete the user by id diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 5e1a74bb4..ec7d1cf94 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -1,5 +1,5 @@ -import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; -import { IWorker, WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js"; +import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; +import { IQueueWorker } from "@/worker/worker.interface.js"; import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js"; import { ILogger } from "@/utils/logger.js"; import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; @@ -14,6 +14,7 @@ import { IWorkerHelper } from "@/worker/worker.helper.js"; import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; +import { JobScheduler } from "@/worker/worker.job-scheduler.js"; const SERVICE_NAME = "JobQueue"; const POLL_MS = 250; // base poll interval while a loop is actively claiming work const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs @@ -29,10 +30,8 @@ const CONCURRENCY: Record = { "cleanup-retention": 1, }; -export class DBQueueWorker implements IWorker { +export class DBQueueWorker extends JobScheduler implements IQueueWorker { static SERVICE_NAME = SERVICE_NAME; - private stopped = false; - private timers = new Map(); private inFlight: Record = { check: 0, "geo-check": 0, @@ -41,33 +40,24 @@ export class DBQueueWorker implements IWorker { "cleanup-retention": 0, }; - // Per job type poll intervals. These back off on idle and reset on work or wake() - private pollMs: Record = { - check: POLL_MS, - "geo-check": POLL_MS, - evaluate: POLL_MS, - "cleanup-orphaned": POLL_MS, - "cleanup-retention": POLL_MS, - }; - private ticking: Partial> = {}; // Whether a loop is mid tick - private tickFns = new Map void>(); // Store tick fns so we can run them immediately on wake - constructor( private logger: ILogger, - private jobsRepository: IJobsRepository, + jobsRepository: IJobsRepository, // ← no modifier: forwarded to super private monitorsRepository: IMonitorsRepository, private checksRepository: IChecksRepository, - private checkService: ICheckService, // toStatusResponse + toLastEvaluatedAt + private checkService: ICheckService, private checkProducer: ICheckProducer, private checkEvaluator: ICheckEvaluator, private geoCheckPipeline: ICheckPipeline, private dispatcher: IReactorDispatcher, private helper: IWorkerHelper, - private queueWorkersRepository: IQueueWorkersRepository, + queueWorkersRepository: IQueueWorkersRepository, // ← no modifier: forwarded private queueMode: QueueMode, private queuePrimaryProcesses: boolean, - private readonly workerId: string // same id MongoJobsRepository stamps into lockedBy - ) {} + workerId: string + ) { + super(jobsRepository, queueWorkersRepository, workerId); + } get serviceName() { return DBQueueWorker.SERVICE_NAME; @@ -109,23 +99,6 @@ export class DBQueueWorker implements IWorker { return instance; } - // Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd. - private toCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ - id: jobId("check", monitor.id), - type: "check", - refId: monitor.id, - isActive: monitor.isActive, - nextScheduledAt: immediate ? now : now + Math.floor(Math.random() * monitor.interval), // Avoid herd - intervalMs: monitor.interval, - }); - - private toGeoCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ - ...this.toCheckJob(monitor, now, immediate), - id: jobId("geo-check", monitor.id), - type: "geo-check", - intervalMs: monitor.geoCheckInterval!, - }); - private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({ id: jobId(type, null), type, @@ -135,20 +108,6 @@ export class DBQueueWorker implements IWorker { intervalMs: CLEANUP_INTERVAL_MS, }); - private toSummary = (job: Job): WorkerJobSummary => ({ - monitorId: job.refId ?? job.id, - monitorType: job.type, - monitorInterval: job.intervalMs, - monitorActive: job.isActive, - lockedBy: job.lockedBy, - lockedUntil: job.lockedUntil, - nextScheduledAt: job.nextScheduledAt, - runCount: job.runCount, - failCount: job.failCount, - failReason: job.lastFailReason, - lastFinishedAt: job.lastFinishedAt, - }); - // ******************** // Stage 1: This can eventually be offloaded to a separate service // ******************** @@ -276,15 +235,6 @@ export class DBQueueWorker implements IWorker { // Wake a worker loop after adding a job that's due immediately. // Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off - wake = (type: JobType) => { - this.pollMs[type] = POLL_MS; - if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing - const tick = this.tickFns.get(type); - if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done - const pending = this.timers.get(type); - if (pending) clearTimeout(pending); // Clear out scheduled tick if any - this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick - }; // Seed queue private reconcile = async () => { @@ -332,85 +282,7 @@ export class DBQueueWorker implements IWorker { return true; }; - addJob = async (_monitorId: string, monitor: Monitor) => { - const now = Date.now(); - await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true)); - this.wake("check"); - if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { - await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); - this.wake("geo-check"); - } - }; - deleteJob = async (monitor: Monitor) => { - await this.jobsRepository.deleteById(monitor.id); - }; - pauseJob = async (monitor: Monitor) => { - await this.jobsRepository.setActiveById(monitor.id, false); - }; - resumeJob = async (monitor: Monitor) => { - await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately - this.wake("check"); - if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check"); - }; - updateJob = async (monitor: Monitor) => { - await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval); - if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { - await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away - this.wake("geo-check"); - } else { - await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check"); - } - }; - shutdown = async () => { - this.stopped = true; - // Stop all timers - for (const timer of this.timers.values()) { - clearTimeout(timer); - } - this.timers.clear(); // Clear the map out - // Remove workers from registry - await this.queueWorkersRepository.deleteById(this.workerId); - }; - getMetrics = async (): Promise => { - const rows = await this.jobsRepository.findAll(); - const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS); - const now = Date.now(); - return rows.reduce( - (acc, job) => { - acc.jobs++; - acc.totalRuns += job.runCount; - acc.totalFailures += job.failCount; - if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++; - if (job.failCount > 0) { - acc.failingJobs++; - acc.jobsWithFailures.push({ - monitorId: job.refId ?? job.id, - monitorUrl: null, - monitorType: job.type, - failedAt: job.lastFinishedAt, - failCount: job.failCount, - failReason: job.lastFailReason, - }); - } - return acc; - }, - { - jobs: 0, - activeJobs: 0, - failingJobs: 0, - jobsWithFailures: [], - totalRuns: 0, - totalFailures: 0, - workers, - } - ); - }; - - getJobs = async (pagination: WorkerJobsPagination) => { - const { jobs, count } = await this.jobsRepository.findPage(pagination); - return { jobs: jobs.map(this.toSummary), count }; - }; - flushQueues = async () => { + override flushQueues = async () => { await this.shutdown(); const ok = await this.init(); return { success: ok }; diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 6dd360e50..522ed3204 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -56,17 +56,21 @@ export type WorkerJobsPage = { count: number; }; -export interface IWorker { +export interface IJobScheduler { readonly serviceName: string; - init(): Promise; addJob(monitorId: string, monitor: Monitor): Promise; deleteJob(monitor: Monitor): Promise; pauseJob(monitor: Monitor): Promise; resumeJob(monitor: Monitor): Promise; updateJob(monitor: Monitor): Promise; wake(type: JobType): void; - shutdown(): Promise; getMetrics(): Promise; getJobs(pagination: WorkerJobsPagination): Promise; flushQueues(): Promise<{ success: boolean }>; + shutdown(): Promise; +} + +export interface IQueueWorker extends IJobScheduler { + readonly serviceName: string; + init(): Promise; } diff --git a/server/src/worker/worker.job-scheduler.ts b/server/src/worker/worker.job-scheduler.ts new file mode 100644 index 000000000..4e05e4c38 --- /dev/null +++ b/server/src/worker/worker.job-scheduler.ts @@ -0,0 +1,167 @@ +import { IJobScheduler } from "@/worker/worker.interface.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; +import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js"; +import { type Job, type JobSeed, type JobType, jobId } from "@/domain/jobs/job.type.js"; +import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; +import { WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js"; +import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js"; + +const POLL_MS = 250; // base poll interval while a loop is actively claiming work +const SERVICE_NAME = "JobScheduler"; +const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window + +export class JobScheduler implements IJobScheduler { + static SERVICE_NAME = SERVICE_NAME; + constructor( + protected jobsRepository: IJobsRepository, + protected queueWorkersRepository: IQueueWorkersRepository, + protected readonly workerId: string // same id MongoJobsRepository stamps into lockedBy + ) {} + + get serviceName() { + return JobScheduler.SERVICE_NAME; + } + + protected stopped = false; + protected ticking: Partial> = {}; // Whether a loop is mid tick + protected tickFns = new Map void>(); // Store tick fns so we can run them immediately on wake + protected timers = new Map(); + + // Per job type poll intervals. These back off on idle and reset on work or wake() + protected pollMs: Record = { + check: POLL_MS, + "geo-check": POLL_MS, + evaluate: POLL_MS, + "cleanup-orphaned": POLL_MS, + "cleanup-retention": POLL_MS, + }; + // Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd. + protected toCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ + id: jobId("check", monitor.id), + type: "check", + refId: monitor.id, + isActive: monitor.isActive, + nextScheduledAt: immediate ? now : now + Math.floor(Math.random() * monitor.interval), // Avoid herd + intervalMs: monitor.interval, + }); + + protected toGeoCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({ + ...this.toCheckJob(monitor, now, immediate), + id: jobId("geo-check", monitor.id), + type: "geo-check", + intervalMs: monitor.geoCheckInterval!, + }); + + private toSummary = (job: Job): WorkerJobSummary => ({ + monitorId: job.refId ?? job.id, + monitorType: job.type, + monitorInterval: job.intervalMs, + monitorActive: job.isActive, + lockedBy: job.lockedBy, + lockedUntil: job.lockedUntil, + nextScheduledAt: job.nextScheduledAt, + runCount: job.runCount, + failCount: job.failCount, + failReason: job.lastFailReason, + lastFinishedAt: job.lastFinishedAt, + }); + wake = (type: JobType) => { + this.pollMs[type] = POLL_MS; + if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing + const tick = this.tickFns.get(type); + if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done + const pending = this.timers.get(type); + if (pending) clearTimeout(pending); // Clear out scheduled tick if any + this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick + }; + + addJob = async (_monitorId: string, monitor: Monitor) => { + const now = Date.now(); + await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true)); + this.wake("check"); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { + await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true)); + this.wake("geo-check"); + } + }; + + deleteJob = async (monitor: Monitor) => { + await this.jobsRepository.deleteById(monitor.id); + }; + + pauseJob = async (monitor: Monitor) => { + await this.jobsRepository.setActiveById(monitor.id, false); + }; + + resumeJob = async (monitor: Monitor) => { + await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately + this.wake("check"); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check"); + }; + + updateJob = async (monitor: Monitor) => { + await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval); + if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) { + await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away + this.wake("geo-check"); + } else { + await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check"); + } + }; + + getMetrics = async (): Promise => { + const rows = await this.jobsRepository.findAll(); + const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS); + const now = Date.now(); + return rows.reduce( + (acc, job) => { + acc.jobs++; + acc.totalRuns += job.runCount; + acc.totalFailures += job.failCount; + if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++; + if (job.failCount > 0) { + acc.failingJobs++; + acc.jobsWithFailures.push({ + monitorId: job.refId ?? job.id, + monitorUrl: null, + monitorType: job.type, + failedAt: job.lastFinishedAt, + failCount: job.failCount, + failReason: job.lastFailReason, + }); + } + return acc; + }, + { + jobs: 0, + activeJobs: 0, + failingJobs: 0, + jobsWithFailures: [], + totalRuns: 0, + totalFailures: 0, + workers, + } + ); + }; + + getJobs = async (pagination: WorkerJobsPagination) => { + const { jobs, count } = await this.jobsRepository.findPage(pagination); + return { jobs: jobs.map(this.toSummary), count }; + }; + + flushQueues = async () => { + await this.shutdown(); + return { success: true }; + }; + + shutdown = async () => { + this.stopped = true; + // Stop all timers + for (const timer of this.timers.values()) { + clearTimeout(timer); + } + this.timers.clear(); // Clear the map out + // Remove workers from registry + await this.queueWorkersRepository.deleteById(this.workerId); + }; +} From f0a5974eaf787474919872301ae32cdc326d6c1d Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 23 Jun 2026 11:31:35 -0700 Subject: [PATCH 71/75] update tests --- .../unit/services/maintenanceWindowService.test.ts | 6 +++--- server/test/unit/services/monitorService.test.ts | 4 ++-- server/test/unit/services/userService.test.ts | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/test/unit/services/maintenanceWindowService.test.ts b/server/test/unit/services/maintenanceWindowService.test.ts index 5c7387da8..9547eaac8 100644 --- a/server/test/unit/services/maintenanceWindowService.test.ts +++ b/server/test/unit/services/maintenanceWindowService.test.ts @@ -3,7 +3,7 @@ import { MaintenanceWindowService } from "../../../src/domain/maintenance-window import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts"; -import type { IWorker } from "../../../src/worker/worker.interface.ts"; +import type { IJobScheduler } from "../../../src/worker/worker.interface.ts"; import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -37,7 +37,7 @@ const createJobsRepo = () => const createWorker = () => ({ wake: jest.fn(), - }) as unknown as jest.Mocked; + }) as unknown as jest.Mocked; const createService = (overrides?: { monitorsRepository?: ReturnType; @@ -49,7 +49,7 @@ const createService = (overrides?: { const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo(); const jobsRepository = overrides?.jobsRepository ?? createJobsRepo(); const worker = overrides?.worker ?? createWorker(); - const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker }); + const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository, scheduler: worker }); return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository, worker }; }; diff --git a/server/test/unit/services/monitorService.test.ts b/server/test/unit/services/monitorService.test.ts index d64007ebe..65d6923a6 100644 --- a/server/test/unit/services/monitorService.test.ts +++ b/server/test/unit/services/monitorService.test.ts @@ -98,7 +98,7 @@ const createService = ( const jobQueue = overrides.jobQueue ?? createJobQueueMock(); const logger = overrides.logger ?? createMockLogger(); const service = new MonitorService({ - worker: jobQueue as any, + scheduler: jobQueue as any, logger: logger as any, games: (overrides.games ?? { cs2: { name: "Counter-Strike 2" } }) as any, monitorsRepository: overrides.monitorsRepository ?? createMonitorsRepositoryMock(), @@ -1535,7 +1535,7 @@ describe("MonitorService", () => { const logger = createMockLogger(); const service = new MonitorService({ - worker: jobQueue as any, + scheduler: jobQueue as any, logger: logger as any, games: {} as any, monitorsRepository, diff --git a/server/test/unit/services/userService.test.ts b/server/test/unit/services/userService.test.ts index 7b2fb380b..061003e7f 100644 --- a/server/test/unit/services/userService.test.ts +++ b/server/test/unit/services/userService.test.ts @@ -65,7 +65,7 @@ const createService = (overrides?: Record) => { const settingsService = { getSettings: jest.fn().mockReturnValue(makeAppSettings()), }; - const worker = { + const scheduler = { deleteJob: jest.fn().mockResolvedValue(undefined), }; const jwtMock = { @@ -85,7 +85,7 @@ const createService = (overrides?: Record) => { monitorsRepository, emailService, settingsService, - worker, + scheduler, jwt: jwtMock, crypto: cryptoMock, ...overrides, @@ -475,23 +475,23 @@ describe("UserService", () => { describe("deleteUser", () => { it("deletes user and cleans up monitor jobs for superadmin", async () => { - const { service, usersRepository, worker, monitorsRepository } = createService(); + const { service, usersRepository, scheduler, monitorsRepository } = createService(); await service.deleteUser({ userId: "user-1", teamId: "team-1", roles: ["superadmin"] }); expect(monitorsRepository.findByTeamId).toHaveBeenCalledWith("team-1", {}); - expect(worker.deleteJob).toHaveBeenCalledTimes(2); + expect(scheduler.deleteJob).toHaveBeenCalledTimes(2); expect(usersRepository.deleteById).toHaveBeenCalledWith("user-1"); }); it("handles null monitors for superadmin gracefully", async () => { - const { service, usersRepository, worker } = createService({ + const { service, usersRepository, scheduler } = createService({ monitorsRepository: { findByTeamId: jest.fn().mockResolvedValue(null) }, }); await service.deleteUser({ userId: "user-1", teamId: "team-1", roles: ["superadmin"] }); - expect(worker.deleteJob).not.toHaveBeenCalled(); + expect(scheduler.deleteJob).not.toHaveBeenCalled(); expect(usersRepository.deleteById).toHaveBeenCalledWith("user-1"); }); From f59a7c31132f889186afcba76a1961f1464de499 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 23 Jun 2026 12:26:42 -0700 Subject: [PATCH 72/75] extract shared services --- server/src/config/services.api.ts | 0 server/src/config/services.shared.ts | 218 +++++++++++++++++++++++++++ server/src/config/services.ts | 154 +++++-------------- server/src/config/services.worker.ts | 0 4 files changed, 253 insertions(+), 119 deletions(-) create mode 100644 server/src/config/services.api.ts create mode 100644 server/src/config/services.shared.ts create mode 100644 server/src/config/services.worker.ts diff --git a/server/src/config/services.api.ts b/server/src/config/services.api.ts new file mode 100644 index 000000000..e69de29bb diff --git a/server/src/config/services.shared.ts b/server/src/config/services.shared.ts new file mode 100644 index 000000000..49b73f52e --- /dev/null +++ b/server/src/config/services.shared.ts @@ -0,0 +1,218 @@ +import { Mongoose } from "mongoose"; +import { hostname } from "node:os"; +import { randomUUID } from "node:crypto"; +import fs from "fs"; +import path from "path"; +import nodemailer from "nodemailer"; +import pkg from "handlebars"; +const { compile } = pkg; +import mjml2html from "mjml"; + +import MongoDB from "@/db/db.mongo.js"; +import { IDb } from "@/db/db.interface.js"; +import { ISettingsService, EnvConfig, SettingsService } from "@/domain/app-settings/app-settings.service.js"; +import { ICheckService, CheckService } from "@/domain/checks/check.service.js"; +import { IGeoChecksService, GeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; +import { IIncidentService, IncidentService } from "@/domain/incidents/incident.service.js"; +import { INotificationMessageBuilder, NotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; +import { INotificationsService, NotificationsService } from "@/domain/notifications/notification.service.js"; +import { IEmailService, EmailService } from "@/service/emailService.js"; +import { GlobalPingService } from "@/service/globalPingService.js"; +import { ILogger } from "@/utils/logger.js"; + +// Notification providers +import { WebhookProvider } from "@/domain/notifications/providers/webhook.js"; +import { SlackProvider } from "@/domain/notifications/providers/slack.js"; +import { EmailProvider } from "@/domain/notifications/providers/email.js"; +import { DiscordProvider } from "@/domain/notifications/providers/discord.js"; +import { PagerDutyProvider } from "@/domain/notifications/providers/pagerduty.js"; +import { MatrixProvider } from "@/domain/notifications/providers/matrix.js"; +import { TeamsProvider } from "@/domain/notifications/providers/teams.js"; +import { TelegramProvider } from "@/domain/notifications/providers/telegram.js"; +import { PushoverProvider } from "@/domain/notifications/providers/pushover.js"; +import { TwilioProvider } from "@/domain/notifications/providers/twilio.js"; +import { NtfyProvider } from "@/domain/notifications/providers/ntfy.js"; + +// Repository interfaces +import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js"; +import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; +import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.interface.js"; +import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js"; +import { IInvitesRepository } from "@/domain/invites/invite.repository.interface.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; +import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; +import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; +import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js"; +import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; +import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js"; +import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js"; +import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js"; +import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; +import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; + +// Mongo repository implementations +import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js"; +import MongoChecksRepository from "@/domain/checks/check.repository.mongo.js"; +import MongoGeoChecksRepository from "@/domain/geo-checks/geo-check.repository.mongo.js"; +import MongoIncidentsRepository from "@/domain/incidents/incident.repository.mongo.js"; +import MongoInvitesRepository from "@/domain/invites/invite.repository.mongo.js"; +import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js"; +import MongoMaintenanceWindowsRepository from "@/domain/maintenance-windows/maintenance-window.repository.mongo.js"; +import MongoMonitorStatsRepository from "@/domain/monitor-stats/monitor-stats.repository.mongo.js"; +import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js"; +import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js"; +import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js"; +import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js"; +import MongoStatusPagesRepository from "@/domain/status-pages/status-page-repository.mongo.js"; +import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js"; +import MongoTeamsRepository from "@/domain/teams/team.repository.model.js"; +import MongoUsersRepository from "@/domain/users/user.repository.mongo.js"; + +// Shared infrastructure + business services that both the API and the worker process construct. +export interface SharedServices { + logger: ILogger; + db: IDb; + settingsService: ISettingsService; + emailService: IEmailService; + notificationMessageBuilder: INotificationMessageBuilder; + incidentService: IIncidentService; + checkService: ICheckService; + geoChecksService: IGeoChecksService; + notificationsService: INotificationsService; + + // Queue identity (one per process; the worker reuses these) + workerId: string; + jobsRepository: IJobsRepository; + queueWorkersRepository: IQueueWorkersRepository; + + // Repositories + monitorsRepository: IMonitorsRepository; + checksRepository: IChecksRepository; + geoChecksRepository: IGeoChecksRepository; + monitorStatsRepository: IMonitorStatsRepository; + statusPagesRepository: IStatusPagesRepository; + usersRepository: IUsersRepository; + invitesRepository: IInvitesRepository; + recoveryTokensRepository: IRecoveryTokensRepository; + settingsRepository: ISettingsRepository; + notificationsRepository: INotificationsRepository; + tagsRepository: ITagsRepository; + incidentsRepository: IIncidentsRepository; + teamsRepository: ITeamsRepository; + maintenanceWindowsRepository: IMaintenanceWindowsRepository; +} + +export const buildShared = async ({ + logger, + envSettings, + settingsService, +}: { + logger: ILogger; + envSettings: EnvConfig; + settingsService: ISettingsService; +}): Promise => { + // Shared worker ID + const workerId = `${hostname()}:${process.pid}:${randomUUID()}`; + + // Create DB + let db: IDb | null = null; + db = new MongoDB(logger, envSettings); + await db.connect(); + + // Repositories + const monitorsRepository = new MongoMonitorsRepository(); + const checksRepository = new MongoChecksRepository(logger); + const incidentsRepository = new MongoIncidentsRepository(); + const usersRepository = new MongoUsersRepository(); + const geoChecksRepository = new MongoGeoChecksRepository(logger); + const notificationsRepository = new MongoNotificationsRepository(); + const jobsRepository = new MongoJobsRepository(workerId); + const queueWorkersRepository = new MongoQueueWorkersRepository(); + const monitorStatsRepository = new MongoMonitorStatsRepository(); + const statusPagesRepository = new MongoStatusPagesRepository(); + const invitesRepository = new MongoInvitesRepository(); + const recoveryTokensRepository = new MongoRecoveryTokensRepository(); + const settingsRepository = new MongoSettingsRepository(); + const tagsRepository = new MongoTagsRepository(); + const teamsRepository = new MongoTeamsRepository(); + const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); + + // Inject settings repository into settings service (now that DB is connected) + (settingsService as SettingsService).setRepository(settingsRepository); + + // Services + const notificationMessageBuilder = new NotificationMessageBuilder(); + const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger); + const incidentService = new IncidentService(logger, incidentsRepository, monitorsRepository, usersRepository, notificationMessageBuilder); + const checkService = new CheckService(monitorsRepository, logger, checksRepository); + + const globalPingService = new GlobalPingService(logger); + const geoChecksService = new GeoChecksService({ + logger, + geoChecksRepository, + globalPingService, + monitorsRepository, + }); + + const webhookProvider = new WebhookProvider(logger); + const slackProvider = new SlackProvider(logger); + const emailProvider = new EmailProvider(emailService, logger); + const discordProvider = new DiscordProvider(logger); + const pagerDutyProvider = new PagerDutyProvider(logger); + const matrixProvider = new MatrixProvider(logger); + const teamsProvider = new TeamsProvider(logger); + const telegramProvider = new TelegramProvider(logger); + const pushoverProvider = new PushoverProvider(logger); + const twilioProvider = new TwilioProvider(logger); + const ntfyProvider = new NtfyProvider(logger); + + const notificationsService = new NotificationsService( + notificationsRepository, + monitorsRepository, + webhookProvider, + emailProvider, + slackProvider, + discordProvider, + pagerDutyProvider, + matrixProvider, + teamsProvider, + telegramProvider, + pushoverProvider, + twilioProvider, + ntfyProvider, + settingsService, + logger, + notificationMessageBuilder + ); + + const sharedServices: SharedServices = { + logger, + db, + settingsService, + emailService, + notificationMessageBuilder, + incidentService, + checkService, + geoChecksService, + notificationsService, + workerId, + jobsRepository, + monitorsRepository, + queueWorkersRepository, + checksRepository, + geoChecksRepository, + monitorStatsRepository, + statusPagesRepository, + usersRepository, + invitesRepository, + recoveryTokensRepository, + settingsRepository, + notificationsRepository, + tagsRepository, + incidentsRepository, + teamsRepository, + maintenanceWindowsRepository, + }; + return sharedServices; +}; diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 7bc59f881..78dabadf8 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -1,35 +1,21 @@ import { Mongoose } from "mongoose"; -import { hostname } from "node:os"; -import { randomUUID } from "node:crypto"; -import MongoDB from "../db/db.mongo.js"; +import { buildShared } from "@/config/services.shared.js"; import { IDb } from "@/db/db.interface.js"; -import { EnvConfig, ISettingsService, SettingsService } from "@/domain/app-settings/app-settings.service.js"; -import { CheckService, ICheckService } from "@/domain/checks/check.service.js"; +import { EnvConfig, ISettingsService } from "@/domain/app-settings/app-settings.service.js"; +import { ICheckService } from "@/domain/checks/check.service.js"; import { DiagnosticService, IDiagnosticService } from "@/domain/diagnostics/diagnostic.service.js"; -import { GeoChecksService, IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; -import { IIncidentService, IncidentService } from "@/domain/incidents/incident.service.js"; +import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; +import { IIncidentService } from "@/domain/incidents/incident.service.js"; import { IInviteService, InviteService } from "@/domain/invites/invite.service.js"; import { IMaintenanceWindowService, MaintenanceWindowService } from "@/domain/maintenance-windows/maintenance-window.service.js"; import { IMonitorService, MonitorService } from "@/domain/monitors/monitor.service.js"; -import { INotificationMessageBuilder, NotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; -import { INotificationsService, NotificationsService } from "@/domain/notifications/notification.service.js"; -import { DiscordProvider } from "@/domain/notifications/providers/discord.js"; -import { EmailProvider } from "@/domain/notifications/providers/email.js"; -import { MatrixProvider } from "@/domain/notifications/providers/matrix.js"; -import { NtfyProvider } from "@/domain/notifications/providers/ntfy.js"; -import { PagerDutyProvider } from "@/domain/notifications/providers/pagerduty.js"; -import { PushoverProvider } from "@/domain/notifications/providers/pushover.js"; -import { SlackProvider } from "@/domain/notifications/providers/slack.js"; -import { TeamsProvider } from "@/domain/notifications/providers/teams.js"; -import { TelegramProvider } from "@/domain/notifications/providers/telegram.js"; -import { TwilioProvider } from "@/domain/notifications/providers/twilio.js"; -import { WebhookProvider } from "@/domain/notifications/providers/webhook.js"; +import { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; +import { INotificationsService } from "@/domain/notifications/notification.service.js"; import { IStatusPageService, StatusPageService } from "@/domain/status-pages/status-page.service.js"; import { ITagsService, TagsService } from "@/domain/tags/tag.service.js"; import { IUserService, UserService } from "@/domain/users/user.service.js"; import { BufferService, IBufferService } from "@/service/bufferService.js"; -import { EmailService, IEmailService } from "@/service/emailService.js"; -import { GlobalPingService } from "@/service/globalPingService.js"; +import { IEmailService } from "@/service/emailService.js"; import { WorkerHelper } from "@/worker/worker.helper.js"; import { IQueueWorker } from "@/worker/worker.interface.js"; import { INetworkService, NetworkService } from "@/service/networkService.js"; @@ -56,12 +42,6 @@ import got from "got"; import ping from "ping"; import Docker from "dockerode"; import net from "net"; -import fs from "fs"; -import path from "path"; -import nodemailer from "nodemailer"; -import pkg from "handlebars"; -const { compile } = pkg; -import mjml2html from "mjml"; import jwt from "jsonwebtoken"; import crypto from "crypto"; import { games, GameDig } from "gamedig"; @@ -72,33 +52,19 @@ import WebSocket from "ws"; // Repositories import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js"; -import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js"; import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; -import MongoChecksRepository from "@/domain/checks/check.repository.mongo.js"; import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.interface.js"; -import MongoGeoChecksRepository from "@/domain/geo-checks/geo-check.repository.mongo.js"; import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js"; -import MongoIncidentsRepository from "@/domain/incidents/incident.repository.mongo.js"; import { IInvitesRepository } from "@/domain/invites/invite.repository.interface.js"; -import MongoInvitesRepository from "@/domain/invites/invite.repository.mongo.js"; import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; -import MongoMaintenanceWindowsRepository from "@/domain/maintenance-windows/maintenance-window.repository.mongo.js"; import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; -import MongoMonitorStatsRepository from "@/domain/monitor-stats/monitor-stats.repository.mongo.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; -import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js"; import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js"; -import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js"; import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js"; -import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js"; import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js"; -import MongoStatusPagesRepository from "@/domain/status-pages/status-page-repository.mongo.js"; import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js"; -import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; -import MongoTeamsRepository from "@/domain/teams/team.repository.model.js"; import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; -import MongoUsersRepository from "@/domain/users/user.repository.mongo.js"; import { ILogger } from "@/utils/logger.js"; import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; @@ -107,8 +73,6 @@ import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; import { CheckProducer } from "@/worker/worker.check-producer.js"; import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { DBQueueWorker } from "@/worker/worker.db-queue.js"; -import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js"; -import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js"; export type InitializedServices = { settingsService: ISettingsService; @@ -158,33 +122,33 @@ export const initializeServices = async ({ envSettings: EnvConfig; settingsService: ISettingsService; }): Promise => { - // Create DB - - let db: IDb | null = null; - db = new MongoDB(logger, envSettings); - await db.connect(); - - const monitorsRepository = new MongoMonitorsRepository(); - const checksRepository = new MongoChecksRepository(logger); - const geoChecksRepository = new MongoGeoChecksRepository(logger); - const monitorStatsRepository = new MongoMonitorStatsRepository(); - const statusPagesRepository = new MongoStatusPagesRepository(); - const usersRepository = new MongoUsersRepository(); - const invitesRepository = new MongoInvitesRepository(); - const recoveryTokensRepository = new MongoRecoveryTokensRepository(); - const settingsRepository = new MongoSettingsRepository(); - const notificationsRepository = new MongoNotificationsRepository(); - const tagsRepository = new MongoTagsRepository(); - const incidentsRepository = new MongoIncidentsRepository(); - const teamsRepository = new MongoTeamsRepository(); - const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); - // One identity per process - const workerId = `${hostname()}:${process.pid}:${randomUUID()}`; - const jobsRepository = new MongoJobsRepository(workerId); - const queueWorkersRepository = new MongoQueueWorkersRepository(); - - // Inject settings repository into settings service (now that DB is connected) - (settingsService as SettingsService).setRepository(settingsRepository); + const shared = await buildShared({ logger, envSettings, settingsService }); + const { + db, + emailService, + notificationMessageBuilder, + incidentService, + checkService, + geoChecksService, + notificationsService, + workerId, + jobsRepository, + monitorsRepository, + queueWorkersRepository, + checksRepository, + geoChecksRepository, + monitorStatsRepository, + statusPagesRepository, + usersRepository, + invitesRepository, + recoveryTokensRepository, + settingsRepository, + notificationsRepository, + tagsRepository, + incidentsRepository, + teamsRepository, + maintenanceWindowsRepository, + } = shared; // Network providers const pingProvider = new PingProvider(ping); @@ -210,59 +174,11 @@ export const initializeServices = async ({ webSocketProvider, dnsProvider, ]); - const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger); - - const notificationMessageBuilder = new NotificationMessageBuilder(); - - const incidentService = new IncidentService(logger, incidentsRepository, monitorsRepository, usersRepository, notificationMessageBuilder); - - const checkService = new CheckService(monitorsRepository, logger, checksRepository); - - const globalPingService = new GlobalPingService(logger); - - const geoChecksService = new GeoChecksService({ - logger, - geoChecksRepository, - globalPingService, - monitorsRepository, - }); const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository); const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); - // Notification providers - const webhookProvider = new WebhookProvider(logger); - const slackProvider = new SlackProvider(logger); - const emailProvider = new EmailProvider(emailService, logger); - const discordProvider = new DiscordProvider(logger); - const pagerDutyProvider = new PagerDutyProvider(logger); - const matrixProvider = new MatrixProvider(logger); - const teamsProvider = new TeamsProvider(logger); - const telegramProvider = new TelegramProvider(logger); - const pushoverProvider = new PushoverProvider(logger); - const twilioProvider = new TwilioProvider(logger); - const ntfyProvider = new NtfyProvider(logger); - - const notificationsService = new NotificationsService( - notificationsRepository, - monitorsRepository, - webhookProvider, - emailProvider, - slackProvider, - discordProvider, - pagerDutyProvider, - matrixProvider, - teamsProvider, - telegramProvider, - pushoverProvider, - twilioProvider, - ntfyProvider, - settingsService, - logger, - notificationMessageBuilder - ); - const tagsService = new TagsService(tagsRepository, monitorsRepository); const monitorStatusPolicy = new MonitorStatusPolicy(); diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts new file mode 100644 index 000000000..e69de29bb From 71d3e789ff3e06237e1eff7b512b8c3d60cf5bac Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 24 Jun 2026 10:30:21 -0700 Subject: [PATCH 73/75] split services to api, worker, and shared --- server/openapi.json | 24 +- server/src/app.ts | 6 +- server/src/config/controllers.ts | 32 +-- server/src/config/routes.ts | 8 +- server/src/config/services.api.ts | 109 +++++++++ server/src/config/services.ts | 309 ------------------------ server/src/config/services.worker.ts | 151 ++++++++++++ server/src/domain/checks/check.model.ts | 1 + server/src/index.ts | 78 ++++-- server/src/shutdown.ts | 10 +- 10 files changed, 375 insertions(+), 353 deletions(-) delete mode 100644 server/src/config/services.ts diff --git a/server/openapi.json b/server/openapi.json index f3a6a2505..d0bbe45dc 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -386,6 +386,10 @@ "type": "string", "enum": ["equal", "include", "regex"] }, + "method": { + "type": "string", + "enum": ["GET", "HEAD"] + }, "notifications": { "type": "array", "items": { @@ -466,6 +470,9 @@ }, "updatedAt": { "type": "string" + }, + "lastEvaluatedAt": { + "type": "number" } }, "required": [ @@ -480,6 +487,7 @@ "statusWindowThreshold", "ignoreTlsErrors", "useAdvancedMatching", + "method", "notifications", "tags", "cpuAlertThreshold", @@ -493,7 +501,8 @@ "teamId", "userId", "createdAt", - "updatedAt" + "updatedAt", + "lastEvaluatedAt" ], "additionalProperties": { "nullable": true @@ -7066,6 +7075,10 @@ } ] }, + "method": { + "type": "string", + "enum": ["GET", "HEAD"] + }, "gameId": { "anyOf": [ { @@ -7666,6 +7679,11 @@ } ] }, + "method": { + "type": "string", + "enum": ["GET", "HEAD"], + "default": "GET" + }, "url": { "type": "string", "minLength": 1 @@ -8278,6 +8296,10 @@ } ] }, + "method": { + "type": "string", + "enum": ["GET", "HEAD"] + }, "port": { "type": "number", "minimum": 1, diff --git a/server/src/app.ts b/server/src/app.ts index 907c0f77d..d559edb1f 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -9,20 +9,20 @@ import { handleErrors } from "@/api/middleware/handleErrors.js"; import { generalApiLimiter } from "@/api/middleware/rateLimiter.js"; import { sanitizeBody, sanitizeQuery } from "@/api/middleware/sanitization.js"; import { setupRoutes } from "@/config/routes.js"; -import { InitializedServices } from "@/config/services.js"; import { InitializedControllers } from "@/config/controllers.js"; import { EnvConfig } from "@/domain/app-settings/app-settings.service.js"; import { createStatusPageCorsOrigin } from "@/api/middleware/statusPageCorsOrigin.js"; import { isPublicStatusPageApiPath } from "@/api/middleware/statusPagePublicApiPath.js"; +import { ApiServices } from "@/config/services.api.js"; export const createApp = ({ - services, + apiServices: services, controllers, envSettings, frontendPath, openApiSpec, }: { - services: InitializedServices; + apiServices: ApiServices; controllers: InitializedControllers; envSettings: EnvConfig; frontendPath: string; diff --git a/server/src/config/controllers.ts b/server/src/config/controllers.ts index cdbb876b8..7646b41db 100644 --- a/server/src/config/controllers.ts +++ b/server/src/config/controllers.ts @@ -12,7 +12,7 @@ import NotificationController from "../api/controllers/notificationController.js import TagsController from "../api/controllers/tagController.js"; import DiagnosticController from "../api/controllers/diagnosticController.js"; import IncidentController from "../api/controllers/incidentController.js"; -import type { InitializedServices } from "@/config/services.js"; +import { ApiServices } from "@/config/services.api.js"; export interface InitializedControllers { authController: AuthController; @@ -30,21 +30,21 @@ export interface InitializedControllers { diagnosticController: DiagnosticController; incidentController: IncidentController; } -export const initializeControllers = (services: InitializedServices): InitializedControllers => { +export const initializeControllers = (apiServices: ApiServices): InitializedControllers => { return { - authController: new AuthController(services.userService), - monitorController: new MonitorController(services.monitorService, services.notificationsService), - settingsController: new SettingsController(services.settingsService, services.emailService), - checkController: new CheckController(services.checkService), - geoCheckController: new GeoCheckController(services.geoChecksService), - inviteController: new InviteController(services.inviteService), - maintenanceWindowController: new MaintenanceWindowController(services.maintenanceWindowService), - queueController: new QueueController(services.worker), - logController: new LogController(services.logger), - statusPageController: new StatusPageController(services.statusPageService, services.monitorsRepository, services.settingsService), - notificationController: new NotificationController(services.notificationsService, services.monitorsRepository), - tagController: new TagsController(services.tagsService), - diagnosticController: new DiagnosticController(services.diagnosticService), - incidentController: new IncidentController(services.incidentService), + authController: new AuthController(apiServices.userService), + monitorController: new MonitorController(apiServices.monitorService, apiServices.notificationsService), + settingsController: new SettingsController(apiServices.settingsService, apiServices.emailService), + checkController: new CheckController(apiServices.checkService), + geoCheckController: new GeoCheckController(apiServices.geoChecksService), + inviteController: new InviteController(apiServices.inviteService), + maintenanceWindowController: new MaintenanceWindowController(apiServices.maintenanceWindowService), + queueController: new QueueController(apiServices.worker), + logController: new LogController(apiServices.logger), + statusPageController: new StatusPageController(apiServices.statusPageService, apiServices.monitorsRepository, apiServices.settingsService), + notificationController: new NotificationController(apiServices.notificationsService, apiServices.monitorsRepository), + tagController: new TagsController(apiServices.tagsService), + diagnosticController: new DiagnosticController(apiServices.diagnosticService), + incidentController: new IncidentController(apiServices.incidentService), }; }; diff --git a/server/src/config/routes.ts b/server/src/config/routes.ts index d3fbab2c7..855c18abb 100644 --- a/server/src/config/routes.ts +++ b/server/src/config/routes.ts @@ -3,7 +3,7 @@ import { createVerifyJWT } from "../api/middleware/verifyJWT.js"; import { createVerifyStatusPageAccess } from "../api/middleware/verifyStatusPageAccess.js"; import { authApiLimiter } from "../api/middleware/rateLimiter.js"; import type { InitializedControllers } from "./controllers.js"; -import type { InitializedServices } from "./services.js"; +import type { ApiServices } from "@/config/services.api.js"; import AuthRoutes from "../api/routes/authRoute.js"; import InviteRoutes from "../api/routes/inviteRoute.js"; @@ -21,8 +21,8 @@ import TagRoutes from "../api/routes/tagRoutes.js"; import IncidentRoutes from "../api/routes/incidentRoute.js"; -export const setupRoutes = (app: Application, controllers: InitializedControllers, services: InitializedServices) => { - const verifyJWT = createVerifyJWT(services.settingsService); +export const setupRoutes = (app: Application, controllers: InitializedControllers, apiServices: ApiServices) => { + const verifyJWT = createVerifyJWT(apiServices.settingsService); const authRoutes = new AuthRoutes(controllers.authController, verifyJWT); const monitorRoutes = new MonitorRoutes(controllers.monitorController); const settingsRoutes = new SettingsRoutes(controllers.settingsController); @@ -32,7 +32,7 @@ export const setupRoutes = (app: Application, controllers: InitializedController const maintenanceWindowRoutes = new MaintenanceWindowRoutes(controllers.maintenanceWindowController); const queueRoutes = new QueueRoutes(controllers.queueController); const logRoutes = new LogRoutes(controllers.logController); - const verifyStatusPageAccess = createVerifyStatusPageAccess(services.statusPagesRepository, verifyJWT); + const verifyStatusPageAccess = createVerifyStatusPageAccess(apiServices.statusPagesRepository, verifyJWT); const statusPageRoutes = new StatusPageRoutes(controllers.statusPageController, verifyJWT, verifyStatusPageAccess); const notificationRoutes = new NotificationRoutes(controllers.notificationController); const tagRoutes = new TagRoutes(controllers.tagController); diff --git a/server/src/config/services.api.ts b/server/src/config/services.api.ts index e69de29bb..0b9361e5d 100644 --- a/server/src/config/services.api.ts +++ b/server/src/config/services.api.ts @@ -0,0 +1,109 @@ +import { DiagnosticService, IDiagnosticService } from "@/domain/diagnostics/diagnostic.service.js"; +import { IInviteService, InviteService } from "@/domain/invites/invite.service.js"; +import { IMaintenanceWindowService, MaintenanceWindowService } from "@/domain/maintenance-windows/maintenance-window.service.js"; +import { IMonitorService, MonitorService } from "@/domain/monitors/monitor.service.js"; +import { IStatusPageService, StatusPageService } from "@/domain/status-pages/status-page.service.js"; +import { ITagsService, TagsService } from "@/domain/tags/tag.service.js"; +import { IUserService, UserService } from "@/domain/users/user.service.js"; +import { IJobScheduler } from "@/worker/worker.interface.js"; + +// Third-party +import jwt from "jsonwebtoken"; +import crypto from "crypto"; +import { games } from "gamedig"; + +import { SharedServices } from "@/config/services.shared.js"; + +export interface ApiServices extends SharedServices { + worker: IJobScheduler; // control-plane handle only (DBQueueWorker in all-in-one, bare JobScheduler in API-only) + userService: IUserService; + monitorService: IMonitorService; + maintenanceWindowService: IMaintenanceWindowService; + inviteService: IInviteService; + statusPageService: IStatusPageService; + tagsService: ITagsService; + diagnosticService: IDiagnosticService; +} + +export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => { + const { + logger, + db, + settingsService, + emailService, + monitorsRepository, + checksRepository, + geoChecksRepository, + monitorStatsRepository, + statusPagesRepository, + usersRepository, + invitesRepository, + recoveryTokensRepository, + settingsRepository, + tagsRepository, + incidentsRepository, + teamsRepository, + maintenanceWindowsRepository, + jobsRepository, + } = shared; + + const userService = new UserService({ + crypto, + emailService, + settingsService, + logger, + jwt, + scheduler: jobScheduler, + monitorsRepository, + usersRepository, + invitesRepository, + recoveryTokensRepository, + settingsRepository, + teamsRepository, + }); + + // *********************** + // Business services + // *********************** + + const monitorService = new MonitorService({ + scheduler: jobScheduler, + logger, + games, + monitorsRepository, + checksRepository, + geoChecksRepository, + monitorStatsRepository, + statusPagesRepository, + incidentsRepository, + }); + + const maintenanceWindowService = new MaintenanceWindowService({ + monitorsRepository, + maintenanceWindowsRepository, + jobsRepository, + scheduler: jobScheduler, + }); + + const inviteService = new InviteService({ + invitesRepository, + settingsService, + emailService, + }); + + const statusPageService = new StatusPageService(statusPagesRepository, settingsService); + const tagsService = new TagsService(tagsRepository, monitorsRepository); + const diagnosticService = new DiagnosticService(db); + + return { + ...shared, + worker: jobScheduler, + userService, + monitorService, + maintenanceWindowService, + inviteService, + statusPageService, + tagsService, + diagnosticService, + }; +}; diff --git a/server/src/config/services.ts b/server/src/config/services.ts deleted file mode 100644 index 78dabadf8..000000000 --- a/server/src/config/services.ts +++ /dev/null @@ -1,309 +0,0 @@ -import { Mongoose } from "mongoose"; -import { buildShared } from "@/config/services.shared.js"; -import { IDb } from "@/db/db.interface.js"; -import { EnvConfig, ISettingsService } from "@/domain/app-settings/app-settings.service.js"; -import { ICheckService } from "@/domain/checks/check.service.js"; -import { DiagnosticService, IDiagnosticService } from "@/domain/diagnostics/diagnostic.service.js"; -import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js"; -import { IIncidentService } from "@/domain/incidents/incident.service.js"; -import { IInviteService, InviteService } from "@/domain/invites/invite.service.js"; -import { IMaintenanceWindowService, MaintenanceWindowService } from "@/domain/maintenance-windows/maintenance-window.service.js"; -import { IMonitorService, MonitorService } from "@/domain/monitors/monitor.service.js"; -import { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js"; -import { INotificationsService } from "@/domain/notifications/notification.service.js"; -import { IStatusPageService, StatusPageService } from "@/domain/status-pages/status-page.service.js"; -import { ITagsService, TagsService } from "@/domain/tags/tag.service.js"; -import { IUserService, UserService } from "@/domain/users/user.service.js"; -import { BufferService, IBufferService } from "@/service/bufferService.js"; -import { IEmailService } from "@/service/emailService.js"; -import { WorkerHelper } from "@/worker/worker.helper.js"; -import { IQueueWorker } from "@/worker/worker.interface.js"; -import { INetworkService, NetworkService } from "@/service/networkService.js"; -import { IStatusService, StatusService } from "@/service/statusService.js"; -import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; - -// Network providers -import { PingProvider } from "@/service/network/PingProvider.js"; -import { HttpProvider } from "@/service/network/HttpProvider.js"; -import { AdvancedMatcher } from "@/service/network/AdvancedMatcher.js"; -import { PageSpeedProvider } from "@/service/network/PageSpeedProvider.js"; -import { HardwareProvider } from "@/service/network/HardwareProvider.js"; -import { DockerProvider } from "@/service/network/DockerProvider.js"; -import { PortProvider } from "@/service/network/PortProvider.js"; -import { GameProvider } from "@/service/network/GameProvider.js"; -import { GrpcProvider } from "@/service/network/GrpcProvider.js"; -import { WebSocketProvider } from "@/service/network/WebSocketProvider.js"; -import { DNSProvider } from "@/service/network/DNSProvider.js"; - -// Third-party -import { Resolver } from "dns/promises"; -import axios from "axios"; -import got from "got"; -import ping from "ping"; -import Docker from "dockerode"; -import net from "net"; -import jwt from "jsonwebtoken"; -import crypto from "crypto"; -import { games, GameDig } from "gamedig"; -import jmespath from "jmespath"; -import * as grpc from "@grpc/grpc-js"; -import * as protoLoader from "@grpc/proto-loader"; -import WebSocket from "ws"; - -// Repositories -import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js"; -import { IChecksRepository } from "@/domain/checks/check.repository.interface.js"; -import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.interface.js"; -import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js"; -import { IInvitesRepository } from "@/domain/invites/invite.repository.interface.js"; -import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; -import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js"; -import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; -import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js"; -import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js"; -import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js"; -import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js"; -import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; -import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; -import { ILogger } from "@/utils/logger.js"; -import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; -import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; -import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; -import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; -import { CheckProducer } from "@/worker/worker.check-producer.js"; -import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; -import { DBQueueWorker } from "@/worker/worker.db-queue.js"; - -export type InitializedServices = { - settingsService: ISettingsService; - db: IDb; - networkService: INetworkService; - emailService: IEmailService; - bufferService: IBufferService; - statusService: IStatusService; - worker: IQueueWorker; - userService: IUserService; - checkService: ICheckService; - geoChecksService: IGeoChecksService; - diagnosticService: IDiagnosticService; - inviteService: IInviteService; - maintenanceWindowService: IMaintenanceWindowService; - monitorService: IMonitorService; - incidentService: IIncidentService; - logger: ILogger; - notificationsService: INotificationsService; - tagsService: ITagsService; - statusPageService: IStatusPageService; - notificationMessageBuilder: INotificationMessageBuilder; - - // Repositories - monitorsRepository: IMonitorsRepository; - checksRepository: IChecksRepository; - geoChecksRepository: IGeoChecksRepository; - monitorStatsRepository: IMonitorStatsRepository; - statusPagesRepository: IStatusPagesRepository; - usersRepository: IUsersRepository; - invitesRepository: IInvitesRepository; - recoveryTokensRepository: IRecoveryTokensRepository; - settingsRepository: ISettingsRepository; - notificationsRepository: INotificationsRepository; - tagsRepository: ITagsRepository; - incidentsRepository: IIncidentsRepository; - teamsRepository: ITeamsRepository; - maintenanceWindowsRepository: IMaintenanceWindowsRepository; -}; - -export const initializeServices = async ({ - logger, - envSettings, - settingsService, -}: { - logger: ILogger; - envSettings: EnvConfig; - settingsService: ISettingsService; -}): Promise => { - const shared = await buildShared({ logger, envSettings, settingsService }); - const { - db, - emailService, - notificationMessageBuilder, - incidentService, - checkService, - geoChecksService, - notificationsService, - workerId, - jobsRepository, - monitorsRepository, - queueWorkersRepository, - checksRepository, - geoChecksRepository, - monitorStatsRepository, - statusPagesRepository, - usersRepository, - invitesRepository, - recoveryTokensRepository, - settingsRepository, - notificationsRepository, - tagsRepository, - incidentsRepository, - teamsRepository, - maintenanceWindowsRepository, - } = shared; - - // Network providers - const pingProvider = new PingProvider(ping); - const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath)); - const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger); - const hardwareProvider = new HardwareProvider(httpProvider); - const dockerProvider = new DockerProvider(logger, Docker); - const portProvider = new PortProvider(net); - const gameProvider = new GameProvider(logger, GameDig); - const grpcProvider = new GrpcProvider(grpc, protoLoader); - const webSocketProvider = new WebSocketProvider(WebSocket); - const dnsProvider = new DNSProvider(() => new Resolver()); - - const networkService = new NetworkService(axios, logger, [ - pingProvider, - httpProvider, - pageSpeedProvider, - hardwareProvider, - dockerProvider, - portProvider, - gameProvider, - grpcProvider, - webSocketProvider, - dnsProvider, - ]); - - const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository); - - const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); - - const tagsService = new TagsService(tagsRepository, monitorsRepository); - - const monitorStatusPolicy = new MonitorStatusPolicy(); - // reactors and dispatcher - const notificationReactor = new NotificationReactor(notificationsService); - const incidentReactor = new IncidentReactor(incidentService); - const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); - - // Check producer/evaluator - const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger); - const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy); - - // pipelines - const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger); - - const workerHelper = new WorkerHelper( - logger, - checkService, - settingsService, - monitorsRepository, - jobsRepository, - teamsRepository, - monitorStatsRepository, - checksRepository, - incidentsRepository, - geoChecksRepository - ); - - const worker = await DBQueueWorker.create( - logger, - jobsRepository, - monitorsRepository, - checksRepository, - checkService, - checkProducer, - checkEvaluator, - geoCheckPipeline, - reactorDispatcher, - workerHelper, - queueWorkersRepository, - envSettings.queueMode, - envSettings.queuePrimaryProcesses, - workerId - ); - - // Business services - const userService = new UserService({ - crypto, - emailService, - settingsService, - logger, - jwt, - scheduler: worker, - monitorsRepository, - usersRepository, - invitesRepository, - recoveryTokensRepository, - settingsRepository, - teamsRepository, - }); - - const diagnosticService = new DiagnosticService(db); - const inviteService = new InviteService({ - invitesRepository, - settingsService, - emailService, - }); - const maintenanceWindowService = new MaintenanceWindowService({ - monitorsRepository, - maintenanceWindowsRepository, - jobsRepository, - scheduler: worker, - }); - const monitorService = new MonitorService({ - scheduler: worker, - logger, - games, - monitorsRepository, - checksRepository, - geoChecksRepository, - monitorStatsRepository, - statusPagesRepository, - incidentsRepository, - }); - - const statusPageService = new StatusPageService(statusPagesRepository, settingsService); - - const services = { - settingsService, - db, - networkService, - emailService, - bufferService, - statusService, - worker, - userService, - checkService, - geoChecksService, - diagnosticService, - inviteService, - maintenanceWindowService, - monitorService, - incidentService, - logger, - notificationsService, - tagsService, - statusPageService, - notificationMessageBuilder, - - // Repositories - monitorsRepository, - checksRepository, - geoChecksRepository, - monitorStatsRepository, - statusPagesRepository, - usersRepository, - invitesRepository, - recoveryTokensRepository, - settingsRepository, - notificationsRepository, - tagsRepository, - incidentsRepository, - teamsRepository, - maintenanceWindowsRepository, - }; - - return services; -}; diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index e69de29bb..93233cd37 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -0,0 +1,151 @@ +import { Resolver } from "dns/promises"; +import axios from "axios"; +import got from "got"; +import ping from "ping"; +import Docker from "dockerode"; +import net from "net"; +import { GameDig } from "gamedig"; +import jmespath from "jmespath"; +import * as grpc from "@grpc/grpc-js"; +import * as protoLoader from "@grpc/proto-loader"; +import WebSocket from "ws"; + +import { EnvConfig } from "@/domain/app-settings/app-settings.service.js"; +import { SharedServices } from "@/config/services.shared.js"; +import { INetworkService, NetworkService } from "@/service/networkService.js"; +import { IBufferService, BufferService } from "@/service/bufferService.js"; +import { IStatusService, StatusService } from "@/service/statusService.js"; +import { IQueueWorker } from "@/worker/worker.interface.js"; +import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; +import { WorkerHelper } from "@/worker/worker.helper.js"; +import { CheckProducer } from "@/worker/worker.check-producer.js"; +import { CheckEvaluator } from "@/worker/worker.check-evaluator.js"; +import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js"; +import { NotificationReactor } from "@/worker/reactors/reactor.notification.js"; +import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; +import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; +import { DBQueueWorker } from "@/worker/worker.db-queue.js"; + +// Network providers +import { PingProvider } from "@/service/network/PingProvider.js"; +import { HttpProvider } from "@/service/network/HttpProvider.js"; +import { AdvancedMatcher } from "@/service/network/AdvancedMatcher.js"; +import { PageSpeedProvider } from "@/service/network/PageSpeedProvider.js"; +import { HardwareProvider } from "@/service/network/HardwareProvider.js"; +import { DockerProvider } from "@/service/network/DockerProvider.js"; +import { PortProvider } from "@/service/network/PortProvider.js"; +import { GameProvider } from "@/service/network/GameProvider.js"; +import { GrpcProvider } from "@/service/network/GrpcProvider.js"; +import { WebSocketProvider } from "@/service/network/WebSocketProvider.js"; +import { DNSProvider } from "@/service/network/DNSProvider.js"; +export interface WorkerServices { + worker: IQueueWorker; + networkService: INetworkService; + bufferService: IBufferService; + statusService: IStatusService; +} + +export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise => { + const { + logger, + settingsService, + checkService, + geoChecksService, + notificationsService, + incidentService, + workerId, + jobsRepository, + queueWorkersRepository, + monitorsRepository, + checksRepository, + geoChecksRepository, + monitorStatsRepository, + incidentsRepository, + teamsRepository, + maintenanceWindowsRepository, + } = shared; + + // *********************** + // Network providers + // *********************** + const pingProvider = new PingProvider(ping); + const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath)); + const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger); + const hardwareProvider = new HardwareProvider(httpProvider); + const dockerProvider = new DockerProvider(logger, Docker); + const portProvider = new PortProvider(net); + const gameProvider = new GameProvider(logger, GameDig); + const grpcProvider = new GrpcProvider(grpc, protoLoader); + const webSocketProvider = new WebSocketProvider(WebSocket); + const dnsProvider = new DNSProvider(() => new Resolver()); + + const networkService = new NetworkService(axios, logger, [ + pingProvider, + httpProvider, + pageSpeedProvider, + hardwareProvider, + dockerProvider, + portProvider, + gameProvider, + grpcProvider, + webSocketProvider, + dnsProvider, + ]); + + const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository); + const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); + const monitorStatusPolicy = new MonitorStatusPolicy(); + + // *********************** + // Reactors and dispatcher + // Handles notifications and incidents + // *********************** + + const notificationReactor = new NotificationReactor(notificationsService); + const incidentReactor = new IncidentReactor(incidentService); + const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]); + + // *********************** + // Check producer/evaluator + // Handles creating and evaluatiog checks + // *********************** + const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger); + const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy); + const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger); + + // *********************** + // Worker + // *********************** + + const workerHelper = new WorkerHelper( + logger, + checkService, + settingsService, + monitorsRepository, + jobsRepository, + teamsRepository, + monitorStatsRepository, + checksRepository, + incidentsRepository, + geoChecksRepository + ); + + const worker = await DBQueueWorker.create( + logger, + jobsRepository, + monitorsRepository, + checksRepository, + checkService, + checkProducer, + checkEvaluator, + geoCheckPipeline, + reactorDispatcher, + workerHelper, + queueWorkersRepository, + envSettings.queueMode, + envSettings.queuePrimaryProcesses, + workerId + ); + + return { worker, networkService, bufferService, statusService }; +}; diff --git a/server/src/domain/checks/check.model.ts b/server/src/domain/checks/check.model.ts index 82be6937c..c3fbc23ff 100644 --- a/server/src/domain/checks/check.model.ts +++ b/server/src/domain/checks/check.model.ts @@ -241,6 +241,7 @@ const CheckSchema = new Schema( }, errors: { type: [errorSchema], + suppressReservedKeysWarning: true, default: () => [], }, capture: { diff --git a/server/src/index.ts b/server/src/index.ts index 37d4e59d8..d121d7b25 100755 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,4 +1,7 @@ -import { initializeServices } from "./config/services.js"; +import { buildShared } from "@/config/services.shared.js"; +import { buildApi } from "@/config/services.api.js"; +import { buildWorker } from "@/config/services.worker.js"; + import { initializeControllers } from "./config/controllers.js"; import { createApp } from "./app.js"; import { initShutdownListener } from "@/shutdown.js"; @@ -9,44 +12,83 @@ import fs from "fs"; import Logger, { ILogger } from "@/utils/logger.js"; import { SettingsService } from "@/domain/app-settings/app-settings.service.js"; - +import { JobScheduler } from "@/worker/worker.job-scheduler.js"; +import { IJobScheduler } from "@/worker/worker.interface.js"; const SERVICE_NAME = "Server"; let logger: ILogger; const startApp = async () => { - // Validate environment variables first - const env = validateEnv(); + // *********************** + // Basic setup + // *********************** - // Create settings service (env only — DB repository injected after connect) + const env = validateEnv(); const settingsService = new SettingsService(env); const envSettings = settingsService.loadSettings(); - // Create logger logger = new Logger({ envSettings }); + const { queueMode, queuePrimaryProcesses } = envSettings; - // Initialize services (connects DB, creates repositories, injects settingsRepository) - const services = await initializeServices({ logger, envSettings, settingsService }); + logger.info({ + message: `Queue mode: ${queueMode}`, + service: SERVICE_NAME, + method: "startApp", + }); - // If this is a worker instance, we're done. No need for express - if (envSettings.queueMode === "worker") { - logger.info({ - message: "Worker instance started. API will not be started", - service: SERVICE_NAME, - }); - initShutdownListener(null, services); + logger.info({ + message: `Process: ${queuePrimaryProcesses}`, + service: SERVICE_NAME, + method: "startApp", + }); + + // *********************** + // Build shared services + // *********************** + const shared = await buildShared({ logger, envSettings, settingsService }); + + // *********************** + // Worker node path, don't need API + // *********************** + if (queueMode === "worker") { + const { worker } = await buildWorker(shared, envSettings); + logger.info({ message: "Worker instance started. API will not be started", service: SERVICE_NAME }); + initShutdownListener(null, { worker, db: shared.db }); return; } - // FE path + // *********************** + // Primary node path + // *********************** + let scheduler: IJobScheduler; + + // *********************** + //Primary node processes jobs, need full worker + // *********************** + if (queuePrimaryProcesses === true) { + const { worker } = await buildWorker(shared, envSettings); + scheduler = worker; + } + // *********************** + // Primary node does not process jobs, only need scheduler + // *********************** + else { + scheduler = new JobScheduler(shared.jobsRepository, shared.queueWorkersRepository, shared.workerId); + } + + const services = buildApi(shared, scheduler); + + // *********************** + // FE assets always needed for primary node + // *********************** + const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const openApiSpec = JSON.parse(fs.readFileSync(path.join(__dirname, "../openapi.json"), "utf8")); const frontendPath = path.join(__dirname, "..", "public"); - // Initialize controllers const controllers = initializeControllers(services); const app = createApp({ - services, + apiServices: services, controllers, envSettings, frontendPath, diff --git a/server/src/shutdown.ts b/server/src/shutdown.ts index 38d1d5114..584cfea39 100644 --- a/server/src/shutdown.ts +++ b/server/src/shutdown.ts @@ -1,8 +1,14 @@ -import { InitializedServices } from "./config/services.js"; +import { IDb } from "@/db/db.interface.js"; import { logger } from "./utils/logger.js"; import type { Server } from "http"; +import { Mongoose } from "mongoose"; +import { IJobScheduler } from "@/worker/worker.interface.js"; -export const initShutdownListener = (server: Server | null, services: InitializedServices) => { +type ShutdownTargets = { + worker: IJobScheduler; + db: IDb; +}; +export const initShutdownListener = (server: Server | null, services: ShutdownTargets) => { const SERVICE_NAME = "Server"; let isShuttingDown = false; From 74783fd7381d594869f9918f7e57ca3820beac52 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 24 Jun 2026 10:53:03 -0700 Subject: [PATCH 74/75] clean up logger --- server/src/api/middleware/handleErrors.ts | 4 +-- server/src/app.ts | 2 +- server/src/config/envValidation.ts | 28 +++++++++++++++---- server/src/db/db.mongo.ts | 2 +- .../0003_cleanupDuplicateMonitorStats.ts | 4 +-- .../0004_fixInfrastructureThresholds.ts | 4 +-- .../0005_migrateStatusPageTypeToArray.ts | 4 +-- ...anupDuplicateMonitorStatsForUniqueIndex.ts | 4 +-- ...igrateMaintenanceWindowMonitorIdToArray.ts | 4 +-- .../0008_backfillMonitorLastEvaluatedAt.ts | 4 +-- server/src/db/migration/index.ts | 16 +++++------ server/src/domain/monitors/monitor.model.ts | 2 +- server/src/index.ts | 7 +++-- server/src/shutdown.ts | 4 ++- server/src/utils/logger.ts | 19 +++++-------- 15 files changed, 61 insertions(+), 47 deletions(-) diff --git a/server/src/api/middleware/handleErrors.ts b/server/src/api/middleware/handleErrors.ts index 239349b90..77817f12e 100755 --- a/server/src/api/middleware/handleErrors.ts +++ b/server/src/api/middleware/handleErrors.ts @@ -1,12 +1,12 @@ import type { NextFunction, Request, Response } from "express"; import { MulterError } from "multer"; -import { logger } from "@/utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; import { AppError } from "@/utils/AppError.js"; // Multer throws MulterError, send 413 Payload Too Large / 400 const multerErrorStatus = (error: MulterError): number => (error.code === "LIMIT_FILE_SIZE" ? 413 : 400); -const handleErrors = (error: unknown, req: Request, res: Response, _next: NextFunction) => { +const handleErrors = (logger: ILogger) => (error: unknown, req: Request, res: Response, _next: NextFunction) => { const status = error instanceof MulterError ? multerErrorStatus(error) : error instanceof AppError ? error.status || 500 : 500; const message = error instanceof AppError ? error.message : error instanceof Error ? error.message : "Server error"; const service = error instanceof AppError ? error.service : error instanceof MulterError ? "uploadMiddleware" : "unknownService"; diff --git a/server/src/app.ts b/server/src/app.ts index d559edb1f..94a122137 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -112,6 +112,6 @@ export const createApp = ({ app.get("*", (req, res) => { res.sendFile(path.join(frontendPath, "index.html")); }); - app.use(handleErrors); + app.use(handleErrors(services.logger)); return app; }; diff --git a/server/src/config/envValidation.ts b/server/src/config/envValidation.ts index b65ef5165..bdfacf906 100644 --- a/server/src/config/envValidation.ts +++ b/server/src/config/envValidation.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js"; import { booleanCoercion } from "@/api/validation/shared.js"; +import { ILogger } from "@/utils/logger.js"; const envSchema = z.object({ // Server Configuration @@ -31,24 +32,39 @@ const envSchema = z.object({ export type ValidatedEnv = z.infer; -export const validateEnv = (): ValidatedEnv => { +export const validateEnv = (logger: ILogger): ValidatedEnv => { const result = envSchema.safeParse(process.env); if (!result.success) { - console.error("Environment validation failed:"); - console.error(""); + logger.error({ + message: "Env validation failed", + method: "validateEnv", + service: "Server", + }); const errors = result.error.format(); for (const [key, value] of Object.entries(errors)) { if (key !== "_errors" && value && typeof value === "object" && "_errors" in value) { - console.error(` ${key}: ${value._errors.join(", ")}`); + logger.error({ + message: `${key}: ${value._errors.join(", ")}`, + method: "validateEnv", + service: "Server", + }); } } - console.error("Please check your .env file and ensure all required variables are set."); + logger.error({ + message: "Please check your .env file and ensure all required variables are set.", + method: "validateEnv", + service: "Server", + }); process.exit(1); } - console.log("Environment variables validated successfully"); + logger.info({ + message: "Environment variables validated successfully", + method: "validateEnv", + service: "Server", + }); return result.data; }; diff --git a/server/src/db/db.mongo.ts b/server/src/db/db.mongo.ts index 2585eb58b..5aca5550a 100755 --- a/server/src/db/db.mongo.ts +++ b/server/src/db/db.mongo.ts @@ -42,7 +42,7 @@ class MongoDB implements IDb { // the unique index on MonitorStats.monitorId is enforced, otherwise databases // with stray duplicates (caused by the pre-fix race in updateRunningStats) // will crash at startup when the unique index fails to build. - await runMigrations(); + await runMigrations(this.logger); // Sync indexes const models = mongoose.modelNames(); diff --git a/server/src/db/migration/0003_cleanupDuplicateMonitorStats.ts b/server/src/db/migration/0003_cleanupDuplicateMonitorStats.ts index f772b5229..39a67d91e 100644 --- a/server/src/db/migration/0003_cleanupDuplicateMonitorStats.ts +++ b/server/src/db/migration/0003_cleanupDuplicateMonitorStats.ts @@ -1,12 +1,12 @@ import { MonitorStatsModel } from "../../domain/monitor-stats/monitor-stats.model.js"; -import { logger } from "@/utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; /** * Cleanup duplicate MonitorStats documents * Keeps the most recent document (by updatedAt) for each monitorId * and deletes all older duplicates */ -export async function cleanupDuplicateMonitorStats(): Promise { +export async function cleanupDuplicateMonitorStats(logger: ILogger): Promise { const SERVICE_NAME = "Migration:CleanupDuplicateMonitorStats"; try { diff --git a/server/src/db/migration/0004_fixInfrastructureThresholds.ts b/server/src/db/migration/0004_fixInfrastructureThresholds.ts index dfc45c4fe..c21d552a6 100644 --- a/server/src/db/migration/0004_fixInfrastructureThresholds.ts +++ b/server/src/db/migration/0004_fixInfrastructureThresholds.ts @@ -1,11 +1,11 @@ import { MonitorModel } from "../../domain/monitors/monitor.model.js"; -import { logger } from "@/utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; /** * Fix infrastructure monitors that had thresholds incorrectly set to 0 or 5 * due to the old default value. Updates them to 100 (disabled). */ -export async function fixInfrastructureThresholds(): Promise { +export async function fixInfrastructureThresholds(logger: ILogger): Promise { const SERVICE_NAME = "Migration:FixInfrastructureThresholds"; try { diff --git a/server/src/db/migration/0005_migrateStatusPageTypeToArray.ts b/server/src/db/migration/0005_migrateStatusPageTypeToArray.ts index d5dcafbd1..d75b069e0 100644 --- a/server/src/db/migration/0005_migrateStatusPageTypeToArray.ts +++ b/server/src/db/migration/0005_migrateStatusPageTypeToArray.ts @@ -1,8 +1,8 @@ import mongoose from "mongoose"; -import { logger } from "@/utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; import StatusPageModel from "../../domain/status-pages/status-page.model.js"; -export async function migrateStatusPageTypeToArray(): Promise { +export async function migrateStatusPageTypeToArray(logger: ILogger): Promise { const SERVICE_NAME = "Migration:MigrateStatusPageTypeToArray"; try { diff --git a/server/src/db/migration/0006_cleanupDuplicateMonitorStatsForUniqueIndex.ts b/server/src/db/migration/0006_cleanupDuplicateMonitorStatsForUniqueIndex.ts index 4a34e9926..2479f83c6 100644 --- a/server/src/db/migration/0006_cleanupDuplicateMonitorStatsForUniqueIndex.ts +++ b/server/src/db/migration/0006_cleanupDuplicateMonitorStatsForUniqueIndex.ts @@ -1,5 +1,5 @@ import { MonitorStatsModel } from "../../domain/monitor-stats/monitor-stats.model.js"; -import { logger } from "@/utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; /** * Second-pass dedupe of MonitorStats before the unique index on monitorId is enforced. @@ -11,7 +11,7 @@ import { logger } from "@/utils/logger.js"; * Keeps the document with the highest totalChecks for each monitorId (most data) and * deletes the rest. Falls back to most recent updatedAt when totalChecks tie. */ -export async function cleanupDuplicateMonitorStatsForUniqueIndex(): Promise { +export async function cleanupDuplicateMonitorStatsForUniqueIndex(logger: ILogger): Promise { const SERVICE_NAME = "Migration:CleanupDuplicateMonitorStatsForUniqueIndex"; try { diff --git a/server/src/db/migration/0007_migrateMaintenanceWindowMonitorIdToArray.ts b/server/src/db/migration/0007_migrateMaintenanceWindowMonitorIdToArray.ts index bf5766000..437c66704 100644 --- a/server/src/db/migration/0007_migrateMaintenanceWindowMonitorIdToArray.ts +++ b/server/src/db/migration/0007_migrateMaintenanceWindowMonitorIdToArray.ts @@ -1,6 +1,6 @@ import mongoose from "mongoose"; import type { AnyBulkWriteOperation, Document } from "mongodb"; -import { logger } from "@/utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; import MaintenanceWindowModel from "../../domain/maintenance-windows/maintenance-window.model.js"; type GroupResult = { @@ -18,7 +18,7 @@ type GroupResult = { monitorIds: mongoose.Types.ObjectId[]; }; -export async function migrateMaintenanceWindowMonitorIdToArray(): Promise { +export async function migrateMaintenanceWindowMonitorIdToArray(logger: ILogger): Promise { const SERVICE_NAME = "Migration:MigrateMaintenanceWindowMonitorIdToArray"; try { diff --git a/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts index deb7cb02e..148c4b74d 100644 --- a/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts +++ b/server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts @@ -1,6 +1,6 @@ import mongoose from "mongoose"; import { MonitorModel } from "../../domain/monitors/monitor.model.js"; -import { logger } from "@/utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; const JOBS_COLLECTION = "jobs"; @@ -9,7 +9,7 @@ const JOBS_COLLECTION = "jobs"; * * Stamp the upgrade moment, not 0. If set to 0, all checks will be replayed */ -export async function backfillMonitorLastEvaluatedAt(): Promise { +export async function backfillMonitorLastEvaluatedAt(logger: ILogger): Promise { const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt"; try { diff --git a/server/src/db/migration/index.ts b/server/src/db/migration/index.ts index d68a2468e..0b0745d19 100644 --- a/server/src/db/migration/index.ts +++ b/server/src/db/migration/index.ts @@ -11,7 +11,7 @@ import type { ILogger } from "@/utils/logger.js"; type MigrationEntry = { name: string; - execute: () => Promise; + execute: (logger: ILogger) => Promise; }; const migrations: MigrationEntry[] = [ @@ -25,34 +25,34 @@ const migrations: MigrationEntry[] = [ { name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt }, ]; -const runMigrations = async (logger?: ILogger) => { +const runMigrations = async (logger: ILogger) => { try { - logger?.info({ message: "Running migrations", service: "Migrations" }); + logger.info({ message: "Running migrations", service: "Migrations" }); for (const migration of migrations) { const exists = await MigrationModel.findOne({ name: migration.name, status: "completed" }); if (exists) { - logger?.info({ message: `Skipping ${migration.name}`, service: "Migrations" }); + logger.info({ message: `Skipping ${migration.name}`, service: "Migrations" }); continue; } try { - await migration.execute(); + await migration.execute(logger); await MigrationModel.findOneAndUpdate( { name: migration.name }, { status: "completed", completedAt: new Date(), error: undefined }, { upsert: true } ); - logger?.info({ message: `Completed ${migration.name}`, service: "Migrations" }); + logger.info({ message: `Completed ${migration.name}`, service: "Migrations" }); } catch (error) { const err = error as Error; await MigrationModel.findOneAndUpdate({ name: migration.name }, { status: "failed", error: err?.message }, { upsert: true }); throw error; } } - logger?.info({ message: "Migrations completed", service: "Migrations" }); + logger.info({ message: "Migrations completed", service: "Migrations" }); } catch (error) { const err = error as Error; - logger?.error({ message: "Migration failed", service: "Migrations", details: { error: err?.message }, stack: err?.stack }); + logger.error({ message: "Migration failed", service: "Migrations", details: { error: err?.message }, stack: err?.stack }); throw error; } }; diff --git a/server/src/domain/monitors/monitor.model.ts b/server/src/domain/monitors/monitor.model.ts index 8a095658c..47aa70b5e 100644 --- a/server/src/domain/monitors/monitor.model.ts +++ b/server/src/domain/monitors/monitor.model.ts @@ -197,7 +197,7 @@ const checkSnapshotSchema = new Schema( audits: { type: snapshotAuditsSchema }, createdAt: { type: Date, required: true }, }, - { _id: false } + { _id: false, suppressReservedKeysWarning: true } ); const MonitorSchema = new Schema( diff --git a/server/src/index.ts b/server/src/index.ts index d121d7b25..3b0ab6ba4 100755 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -22,10 +22,11 @@ const startApp = async () => { // Basic setup // *********************** - const env = validateEnv(); + logger = new Logger(); + const env = validateEnv(logger); + logger.setLogLevel(env.LOG_LEVEL); const settingsService = new SettingsService(env); const envSettings = settingsService.loadSettings(); - logger = new Logger({ envSettings }); const { queueMode, queuePrimaryProcesses } = envSettings; logger.info({ @@ -51,7 +52,7 @@ const startApp = async () => { if (queueMode === "worker") { const { worker } = await buildWorker(shared, envSettings); logger.info({ message: "Worker instance started. API will not be started", service: SERVICE_NAME }); - initShutdownListener(null, { worker, db: shared.db }); + initShutdownListener(null, { worker, db: shared.db, logger }); return; } diff --git a/server/src/shutdown.ts b/server/src/shutdown.ts index 584cfea39..625547e85 100644 --- a/server/src/shutdown.ts +++ b/server/src/shutdown.ts @@ -1,5 +1,5 @@ import { IDb } from "@/db/db.interface.js"; -import { logger } from "./utils/logger.js"; +import type { ILogger } from "@/utils/logger.js"; import type { Server } from "http"; import { Mongoose } from "mongoose"; import { IJobScheduler } from "@/worker/worker.interface.js"; @@ -7,9 +7,11 @@ import { IJobScheduler } from "@/worker/worker.interface.js"; type ShutdownTargets = { worker: IJobScheduler; db: IDb; + logger: ILogger; }; export const initShutdownListener = (server: Server | null, services: ShutdownTargets) => { const SERVICE_NAME = "Server"; + const logger = services.logger; let isShuttingDown = false; diff --git a/server/src/utils/logger.ts b/server/src/utils/logger.ts index c53e7112c..41142fd92 100755 --- a/server/src/utils/logger.ts +++ b/server/src/utils/logger.ts @@ -1,7 +1,6 @@ import { createLogger, format, transports, Logger as WinstonLogger } from "winston"; import type { Logform } from "winston"; import dotenv from "dotenv"; -import { EnvConfig } from "@/domain/app-settings/app-settings.service.js"; dotenv.config(); const SERVICE_NAME = "Logger"; @@ -28,18 +27,17 @@ export interface ILogger { cacheLog(entry: LogEntry): void; getLogs(): LogEntry[]; buildLogEntry(level: string, config: LogConfig): LogEntry; + setLogLevel(level: string): void; } class Logger implements ILogger { static SERVICE_NAME = SERVICE_NAME; private logger: WinstonLogger; - private envSettings: Partial; private logCache: LogEntry[]; private maxCacheSize: number; - constructor({ envSettings }: { envSettings: Partial }) { - this.envSettings = envSettings; + constructor() { this.logCache = []; this.maxCacheSize = 1000; const consoleFormat = format.printf((info: Logform.TransformableInfo) => { @@ -84,10 +82,8 @@ class Logger implements ILogger { return msg; }); - const logLevel = this.envSettings.logLevel || "info"; - this.logger = createLogger({ - level: logLevel, + level: "info", format: format.combine(format.timestamp()), transports: [ new transports.Console({ @@ -151,11 +147,10 @@ class Logger implements ILogger { timestamp: new Date().toISOString(), }; } + + setLogLevel(level: string): void { + this.logger.level = level || "info"; + } } export default Logger; - -// Legacy logger -const logger = new Logger({ envSettings: { logLevel: "debug" } }); -export { logger }; -export type { LogConfig, LogEntry }; From 0f230c4d70e20c56e41c7edc2c17f514ab290ffe Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Wed, 24 Jun 2026 10:53:13 -0700 Subject: [PATCH 75/75] update tests --- .../test/integration/backfillLastEvaluatedAt.test.ts | 11 +++++++---- server/test/unit/middleware/handleErrors.test.ts | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/server/test/integration/backfillLastEvaluatedAt.test.ts b/server/test/integration/backfillLastEvaluatedAt.test.ts index ec0459b7d..f4c8a58c5 100644 --- a/server/test/integration/backfillLastEvaluatedAt.test.ts +++ b/server/test/integration/backfillLastEvaluatedAt.test.ts @@ -3,6 +3,9 @@ import mongoose from "mongoose"; import { MongoMemoryServer } from "mongodb-memory-server"; import { MonitorModel } from "../../src/domain/monitors/monitor.model.ts"; import { backfillMonitorLastEvaluatedAt } from "../../src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts"; +import { createMockLogger } from "../helpers/createMockLogger.ts"; + +const logger = createMockLogger(); // ── Real-Mongo harness ───────────────────────────────────────────────────────── // Legacy monitors are inserted through the raw driver, NOT MonitorModel.create — the @@ -30,7 +33,7 @@ describe("0008_backfillMonitorLastEvaluatedAt", () => { await MonitorModel.collection.insertOne({ name: "legacy" }); // pre-upgrade doc, no lastEvaluatedAt const before = Date.now(); - await backfillMonitorLastEvaluatedAt(); + await backfillMonitorLastEvaluatedAt(logger); const doc = await MonitorModel.collection.findOne({ name: "legacy" }); // Must be a recent timestamp — backfilling 0 would replay the whole check history on the first evaluate. @@ -42,7 +45,7 @@ describe("0008_backfillMonitorLastEvaluatedAt", () => { it("leaves monitors that already have lastEvaluatedAt untouched (idempotent)", async () => { await MonitorModel.collection.insertOne({ name: "current", lastEvaluatedAt: 12345 }); - await backfillMonitorLastEvaluatedAt(); + await backfillMonitorLastEvaluatedAt(logger); const doc = await MonitorModel.collection.findOne({ name: "current" }); expect(doc?.lastEvaluatedAt).toBe(12345); @@ -52,13 +55,13 @@ describe("0008_backfillMonitorLastEvaluatedAt", () => { const db = mongoose.connection.db!; await db.collection("jobs").insertOne({ id: "legacy-job", template: "monitor-job" }); // old-scheduler row - await backfillMonitorLastEvaluatedAt(); + await backfillMonitorLastEvaluatedAt(logger); const exists = await db.listCollections({ name: "jobs" }).toArray(); expect(exists).toHaveLength(0); }); it("does not throw when there is no jobs collection to drop", async () => { - await expect(backfillMonitorLastEvaluatedAt()).resolves.toBeUndefined(); + await expect(backfillMonitorLastEvaluatedAt(logger)).resolves.toBeUndefined(); }); }); diff --git a/server/test/unit/middleware/handleErrors.test.ts b/server/test/unit/middleware/handleErrors.test.ts index 5994cf7d3..3ce49adbb 100644 --- a/server/test/unit/middleware/handleErrors.test.ts +++ b/server/test/unit/middleware/handleErrors.test.ts @@ -3,6 +3,9 @@ import type { NextFunction, Request, Response } from "express"; import { MulterError } from "multer"; import { handleErrors } from "../../../src/api/middleware/handleErrors.ts"; import { AppError } from "../../../src/utils/AppError.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +const logger = createMockLogger(); const makeRes = (): Response => { const res = {} as Response; @@ -14,25 +17,25 @@ const makeRes = (): Response => { describe("handleErrors", () => { it("maps a multer file-size violation to 413 Payload Too Large", () => { const res = makeRes(); - handleErrors(new MulterError("LIMIT_FILE_SIZE", "profileImage"), {} as Request, res, (() => {}) as NextFunction); + handleErrors(logger)(new MulterError("LIMIT_FILE_SIZE", "profileImage"), {} as Request, res, (() => {}) as NextFunction); expect(res.status).toHaveBeenCalledWith(413); }); it("maps other multer errors to 400", () => { const res = makeRes(); - handleErrors(new MulterError("LIMIT_UNEXPECTED_FILE", "profileImage"), {} as Request, res, (() => {}) as NextFunction); + handleErrors(logger)(new MulterError("LIMIT_UNEXPECTED_FILE", "profileImage"), {} as Request, res, (() => {}) as NextFunction); expect(res.status).toHaveBeenCalledWith(400); }); it("preserves AppError status codes", () => { const res = makeRes(); - handleErrors(new AppError({ status: 415, message: "bad image" }), {} as Request, res, (() => {}) as NextFunction); + handleErrors(logger)(new AppError({ status: 415, message: "bad image" }), {} as Request, res, (() => {}) as NextFunction); expect(res.status).toHaveBeenCalledWith(415); }); it("falls back to 500 for unknown errors", () => { const res = makeRes(); - handleErrors(new Error("boom"), {} as Request, res, (() => {}) as NextFunction); + handleErrors(logger)(new Error("boom"), {} as Request, res, (() => {}) as NextFunction); expect(res.status).toHaveBeenCalledWith(500); }); });