diff --git a/server/src/config/services.ts b/server/src/config/services.ts index ebc17eaa0..346f77209 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -338,6 +338,7 @@ export const initializeServices = async ({ const maintenanceWindowService = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, + jobsRepository, }); const monitorService = new MonitorService({ worker, diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index 3ec95d4c2..190323eed 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -52,6 +52,9 @@ export interface IJobsRepository { // Edit job schedule updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise; + // Re-arm the check/geo-check rows. Add jitter to prevent herd + markMonitorsDue(monitorIds: string[], now: number): Promise; + // Delete, drop all rows deleteById(refId: string): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index e2dfd2e2a..1dd912d10 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -2,6 +2,9 @@ import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.inte import JobModel, { JobDocument } from "@/domain/jobs/job.model.js"; import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js"; +// Cap for rearm interval, never want to wait more than 15s. +const REARM_JITTER_MAX_MS = 15_000; + class MongoJobsRepository implements IJobsRepository { constructor(private readonly workerId: string) {} @@ -249,6 +252,21 @@ class MongoJobsRepository implements IJobsRepository { return res.modifiedCount === 1; }; + markMonitorsDue = async (monitorIds: string[], now: number) => { + if (monitorIds.length === 0) return 0; + const res = await JobModel.updateMany({ refId: { $in: monitorIds }, type: { $in: ["check", "geo-check"] } }, [ + { + // Spread rearmed jobs, but never for more than rearm jitter max + $set: { + nextScheduledAt: { + $add: [now, { $floor: { $multiply: [{ $rand: {} }, { $min: ["$intervalMs", REARM_JITTER_MAX_MS] }] } }], + }, + }, + }, + ]); + return res.modifiedCount; + }; + deleteById = async (refId: string) => { const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor return res.deletedCount > 0; diff --git a/server/src/domain/maintenance-windows/maintenance-window.service.ts b/server/src/domain/maintenance-windows/maintenance-window.service.ts index e1158dec9..aef0f9fb4 100644 --- a/server/src/domain/maintenance-windows/maintenance-window.service.ts +++ b/server/src/domain/maintenance-windows/maintenance-window.service.ts @@ -1,5 +1,6 @@ import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js"; import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js"; import { AppError } from "@/utils/AppError.js"; import { isWindowActive } from "@/utils/maintenanceWindow.js"; @@ -40,16 +41,20 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { static SERVICE_NAME = SERVICE_NAME; private monitorsRepository: IMonitorsRepository; private maintenanceWindowsRepository: IMaintenanceWindowsRepository; + private jobsRepository: IJobsRepository; constructor({ monitorsRepository, maintenanceWindowsRepository, + jobsRepository, }: { monitorsRepository: IMonitorsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; + jobsRepository: IJobsRepository; }) { this.monitorsRepository = monitorsRepository; this.maintenanceWindowsRepository = maintenanceWindowsRepository; + this.jobsRepository = jobsRepository; } get serviceName() { @@ -71,6 +76,8 @@ export class MaintenanceWindowService implements IMaintenanceWindowService { const toInitializing = monitorIds.filter((monitorId) => !stillCovered.has(monitorId)); if (toInitializing.length > 0) { await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]); + // Rearm jobs to run soon (now + jitter to avoid herding) + await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime()); } }; diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index 9c2f54f94..41781114a 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -396,6 +396,44 @@ describe("MongoJobsRepository", () => { expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched }); + it("markMonitorsDue re-arms check and geo rows, capping the jitter for long intervals", async () => { + const armAt = NOW + 5_000_000; // well past the seeded nextScheduledAt so the jittered value is unambiguous + await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", intervalMs: 10_000, nextScheduledAt: NOW }); // shorter than the cap + await seedJob({ _id: "geo:mon-1", refId: "mon-1", type: "geo-check", intervalMs: 3_600_000, nextScheduledAt: NOW }); // far longer than the cap + + const modified = await repo.markMonitorsDue(["mon-1"], armAt); + + expect(modified).toBe(2); + const check = await readRow("check:mon-1"); + const geo = await readRow("geo:mon-1"); + // Short interval spreads over its own interval; long interval is capped so it still runs soon + expect(check!.nextScheduledAt).toBeGreaterThanOrEqual(armAt); + expect(check!.nextScheduledAt).toBeLessThan(armAt + 10_000); + expect(geo!.nextScheduledAt).toBeGreaterThanOrEqual(armAt); + expect(geo!.nextScheduledAt).toBeLessThan(armAt + 15_000); // REARM_JITTER_MS, not the hour-long interval + }); + + it("markMonitorsDue only touches the listed monitors and ignores evaluate/cleanup rows", async () => { + await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW }); + await seedJob({ _id: "check:mon-2", refId: "mon-2", type: "check", nextScheduledAt: NOW }); + await seedJob({ _id: "evaluate:mon-1", refId: "mon-1", type: "evaluate", nextScheduledAt: NOW }); + + const modified = await repo.markMonitorsDue(["mon-1"], NOW + 1_000); + + expect(modified).toBe(1); // only check:mon-1 + expect((await readRow("check:mon-2"))?.nextScheduledAt).toBe(NOW); // other monitor untouched + expect((await readRow("evaluate:mon-1"))?.nextScheduledAt).toBe(NOW); // evaluate row untouched + }); + + it("markMonitorsDue is a no-op for an empty monitor list", async () => { + await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW }); + + const modified = await repo.markMonitorsDue([], NOW + 1_000); + + expect(modified).toBe(0); + expect((await readRow("check:mon-1"))?.nextScheduledAt).toBe(NOW); + }); + it("deleteById drops every row for the monitor", async () => { await seedJob({ _id: "check:mon-1", type: "check" }); await seedJob({ _id: "geo:mon-1", type: "geo-check" }); diff --git a/server/test/unit/services/maintenanceWindowService.test.ts b/server/test/unit/services/maintenanceWindowService.test.ts index 9d381ae74..07d19e405 100644 --- a/server/test/unit/services/maintenanceWindowService.test.ts +++ b/server/test/unit/services/maintenanceWindowService.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals"; import { MaintenanceWindowService } from "../../../src/domain/maintenance-windows/maintenance-window.service.ts"; import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; +import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts"; import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -27,14 +28,21 @@ const createMonitorsRepo = () => updateByIds: jest.fn().mockResolvedValue(0), }) as unknown as jest.Mocked; +const createJobsRepo = () => + ({ + markMonitorsDue: jest.fn().mockResolvedValue(0), + }) as unknown as jest.Mocked; + const createService = (overrides?: { monitorsRepository?: ReturnType; maintenanceWindowsRepository?: ReturnType; + jobsRepository?: ReturnType; }) => { const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo(); const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo(); - const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository }); - return { service, monitorsRepository, maintenanceWindowsRepository }; + const jobsRepository = overrides?.jobsRepository ?? createJobsRepo(); + const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository }); + return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository }; }; const makeWindow = (overrides?: Partial): MaintenanceWindow => ({ @@ -267,12 +275,14 @@ describe("MaintenanceWindowService", () => { it("flips covered monitors to initializing when deleting an active window", async () => { const maintenanceWindowsRepository = createMaintenanceWindowsRepo(); (maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] })); - const { service, monitorsRepository } = createService({ maintenanceWindowsRepository }); + const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository }); await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" }); expect(monitorsRepository.updateByIds).toHaveBeenCalledTimes(1); expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]); + // Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval + expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number)); }); it("excludes the deleted window from the overlap check", async () => {