rearm jobs on maintenance window pause/delete

server/src/config/services.ts

@@ -338,6 +338,7 @@ export const initializeServices = async ({
 	const maintenanceWindowService = new MaintenanceWindowService({
 		monitorsRepository,
 		maintenanceWindowsRepository,
+		jobsRepository,
 	});
 	const monitorService = new MonitorService({
 		worker,

server/src/domain/jobs/job.repository.interface.ts

@@ -52,6 +52,9 @@ export interface IJobsRepository {
 	// Edit job schedule
 	updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
 
+	// Re-arm the check/geo-check rows.  Add jitter to prevent herd
+	markMonitorsDue(monitorIds: string[], now: number): Promise<number>;
+
 	// Delete, drop all rows
 	deleteById(refId: string): Promise<boolean>;
 

server/src/domain/jobs/job.repository.mongo.ts

@@ -2,6 +2,9 @@ import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.inte
 import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
 import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
 
+// Cap for rearm interval, never want to wait more than 15s.
+const REARM_JITTER_MAX_MS = 15_000;
+
 class MongoJobsRepository implements IJobsRepository {
 	constructor(private readonly workerId: string) {}
 
@@ -249,6 +252,21 @@ class MongoJobsRepository implements IJobsRepository {
 		return res.modifiedCount === 1;
 	};
 
+	markMonitorsDue = async (monitorIds: string[], now: number) => {
+		if (monitorIds.length === 0) return 0;
+		const res = await JobModel.updateMany({ refId: { $in: monitorIds }, type: { $in: ["check", "geo-check"] } }, [
+			{
+				// Spread rearmed jobs, but never for more than rearm jitter max
+				$set: {
+					nextScheduledAt: {
+						$add: [now, { $floor: { $multiply: [{ $rand: {} }, { $min: ["$intervalMs", REARM_JITTER_MAX_MS] }] } }],
+					},
+				},
+			},
+		]);
+		return res.modifiedCount;
+	};
+
 	deleteById = async (refId: string) => {
 		const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
 		return res.deletedCount > 0;

server/src/domain/maintenance-windows/maintenance-window.service.ts

@@ -1,5 +1,6 @@
 import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
 import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
+import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
 import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js";
 import { AppError } from "@/utils/AppError.js";
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
@@ -40,16 +41,20 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
 	static SERVICE_NAME = SERVICE_NAME;
 	private monitorsRepository: IMonitorsRepository;
 	private maintenanceWindowsRepository: IMaintenanceWindowsRepository;
+	private jobsRepository: IJobsRepository;
 
 	constructor({
 		monitorsRepository,
 		maintenanceWindowsRepository,
+		jobsRepository,
 	}: {
 		monitorsRepository: IMonitorsRepository;
 		maintenanceWindowsRepository: IMaintenanceWindowsRepository;
+		jobsRepository: IJobsRepository;
 	}) {
 		this.monitorsRepository = monitorsRepository;
 		this.maintenanceWindowsRepository = maintenanceWindowsRepository;
+		this.jobsRepository = jobsRepository;
 	}
 
 	get serviceName() {
@@ -71,6 +76,8 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
 		const toInitializing = monitorIds.filter((monitorId) => !stillCovered.has(monitorId));
 		if (toInitializing.length > 0) {
 			await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]);
+			// Rearm jobs to run soon (now + jitter to avoid herding)
+			await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime());
 		}
 	};
 

server/test/integration/jobsRepository.test.ts

@@ -396,6 +396,44 @@ describe("MongoJobsRepository", () => {
 			expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched
 		});
 
+		it("markMonitorsDue re-arms check and geo rows, capping the jitter for long intervals", async () => {
+			const armAt = NOW + 5_000_000; // well past the seeded nextScheduledAt so the jittered value is unambiguous
+			await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", intervalMs: 10_000, nextScheduledAt: NOW }); // shorter than the cap
+			await seedJob({ _id: "geo:mon-1", refId: "mon-1", type: "geo-check", intervalMs: 3_600_000, nextScheduledAt: NOW }); // far longer than the cap
+
+			const modified = await repo.markMonitorsDue(["mon-1"], armAt);
+
+			expect(modified).toBe(2);
+			const check = await readRow("check:mon-1");
+			const geo = await readRow("geo:mon-1");
+			// Short interval spreads over its own interval; long interval is capped so it still runs soon
+			expect(check!.nextScheduledAt).toBeGreaterThanOrEqual(armAt);
+			expect(check!.nextScheduledAt).toBeLessThan(armAt + 10_000);
+			expect(geo!.nextScheduledAt).toBeGreaterThanOrEqual(armAt);
+			expect(geo!.nextScheduledAt).toBeLessThan(armAt + 15_000); // REARM_JITTER_MS, not the hour-long interval
+		});
+
+		it("markMonitorsDue only touches the listed monitors and ignores evaluate/cleanup rows", async () => {
+			await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW });
+			await seedJob({ _id: "check:mon-2", refId: "mon-2", type: "check", nextScheduledAt: NOW });
+			await seedJob({ _id: "evaluate:mon-1", refId: "mon-1", type: "evaluate", nextScheduledAt: NOW });
+
+			const modified = await repo.markMonitorsDue(["mon-1"], NOW + 1_000);
+
+			expect(modified).toBe(1); // only check:mon-1
+			expect((await readRow("check:mon-2"))?.nextScheduledAt).toBe(NOW); // other monitor untouched
+			expect((await readRow("evaluate:mon-1"))?.nextScheduledAt).toBe(NOW); // evaluate row untouched
+		});
+
+		it("markMonitorsDue is a no-op for an empty monitor list", async () => {
+			await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW });
+
+			const modified = await repo.markMonitorsDue([], NOW + 1_000);
+
+			expect(modified).toBe(0);
+			expect((await readRow("check:mon-1"))?.nextScheduledAt).toBe(NOW);
+		});
+
 		it("deleteById drops every row for the monitor", async () => {
 			await seedJob({ _id: "check:mon-1", type: "check" });
 			await seedJob({ _id: "geo:mon-1", type: "geo-check" });

server/test/unit/services/maintenanceWindowService.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
 import { MaintenanceWindowService } from "../../../src/domain/maintenance-windows/maintenance-window.service.ts";
 import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts";
 import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
+import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts";
 import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts";
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
@@ -27,14 +28,21 @@ const createMonitorsRepo = () =>
 		updateByIds: jest.fn().mockResolvedValue(0),
 	}) as unknown as jest.Mocked<IMonitorsRepository>;
 
+const createJobsRepo = () =>
+	({
+		markMonitorsDue: jest.fn().mockResolvedValue(0),
+	}) as unknown as jest.Mocked<IJobsRepository>;
+
 const createService = (overrides?: {
 	monitorsRepository?: ReturnType<typeof createMonitorsRepo>;
 	maintenanceWindowsRepository?: ReturnType<typeof createMaintenanceWindowsRepo>;
+	jobsRepository?: ReturnType<typeof createJobsRepo>;
 }) => {
 	const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo();
 	const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo();
-	const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository });
-	return { service, monitorsRepository, maintenanceWindowsRepository };
+	const jobsRepository = overrides?.jobsRepository ?? createJobsRepo();
+	const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository });
+	return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository };
 };
 
 const makeWindow = (overrides?: Partial<MaintenanceWindow>): MaintenanceWindow => ({
@@ -267,12 +275,14 @@ describe("MaintenanceWindowService", () => {
 		it("flips covered monitors to initializing when deleting an active window", async () => {
 			const maintenanceWindowsRepository = createMaintenanceWindowsRepo();
 			(maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] }));
-			const { service, monitorsRepository } = createService({ maintenanceWindowsRepository });
+			const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository });
 
 			await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" });
 
 			expect(monitorsRepository.updateByIds).toHaveBeenCalledTimes(1);
 			expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]);
+			// Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval
+			expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number));
 		});
 
 		it("excludes the deleted window from the overlap check", async () => {