rearm jobs on maintenance window pause/delete

This commit is contained in:
Alex Holliday committed 2026-06-18 10:17:33 -07:00
1 parent 6aa9360b98
commit a54fdd89ab
6 files changed
+80 -3

No files matched your search

@@ -396,6 +396,44 @@ describe("MongoJobsRepository", () => {
expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched
});
it("markMonitorsDue re-arms check and geo rows, capping the jitter for long intervals", async () => {
const armAt = NOW + 5_000_000; // well past the seeded nextScheduledAt so the jittered value is unambiguous
await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", intervalMs: 10_000, nextScheduledAt: NOW }); // shorter than the cap
await seedJob({ _id: "geo:mon-1", refId: "mon-1", type: "geo-check", intervalMs: 3_600_000, nextScheduledAt: NOW }); // far longer than the cap
const modified = await repo.markMonitorsDue(["mon-1"], armAt);
expect(modified).toBe(2);
const check = await readRow("check:mon-1");
const geo = await readRow("geo:mon-1");
// Short interval spreads over its own interval; long interval is capped so it still runs soon
expect(check!.nextScheduledAt).toBeGreaterThanOrEqual(armAt);
expect(check!.nextScheduledAt).toBeLessThan(armAt + 10_000);
expect(geo!.nextScheduledAt).toBeGreaterThanOrEqual(armAt);
expect(geo!.nextScheduledAt).toBeLessThan(armAt + 15_000); // REARM_JITTER_MS, not the hour-long interval
});
it("markMonitorsDue only touches the listed monitors and ignores evaluate/cleanup rows", async () => {
await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW });
await seedJob({ _id: "check:mon-2", refId: "mon-2", type: "check", nextScheduledAt: NOW });
await seedJob({ _id: "evaluate:mon-1", refId: "mon-1", type: "evaluate", nextScheduledAt: NOW });
const modified = await repo.markMonitorsDue(["mon-1"], NOW + 1_000);
expect(modified).toBe(1); // only check:mon-1
expect((await readRow("check:mon-2"))?.nextScheduledAt).toBe(NOW); // other monitor untouched
expect((await readRow("evaluate:mon-1"))?.nextScheduledAt).toBe(NOW); // evaluate row untouched
});
it("markMonitorsDue is a no-op for an empty monitor list", async () => {
await seedJob({ _id: "check:mon-1", refId: "mon-1", type: "check", nextScheduledAt: NOW });
const modified = await repo.markMonitorsDue([], NOW + 1_000);
expect(modified).toBe(0);
expect((await readRow("check:mon-1"))?.nextScheduledAt).toBe(NOW);
});
it("deleteById drops every row for the monitor", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
import { MaintenanceWindowService } from "../../../src/domain/maintenance-windows/maintenance-window.service.ts";
import type { IMaintenanceWindowsRepository } from "../../../src/domain/maintenance-windows/maintenance-window.repository.interface.ts";
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts";
import type { MaintenanceWindow } from "../../../src/domain/maintenance-windows/maintenance-window.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -27,14 +28,21 @@ const createMonitorsRepo = () =>
updateByIds: jest.fn().mockResolvedValue(0),
}) as unknown as jest.Mocked<IMonitorsRepository>;
const createJobsRepo = () =>
({
markMonitorsDue: jest.fn().mockResolvedValue(0),
}) as unknown as jest.Mocked<IJobsRepository>;
const createService = (overrides?: {
monitorsRepository?: ReturnType<typeof createMonitorsRepo>;
maintenanceWindowsRepository?: ReturnType<typeof createMaintenanceWindowsRepo>;
jobsRepository?: ReturnType<typeof createJobsRepo>;
}) => {
const monitorsRepository = overrides?.monitorsRepository ?? createMonitorsRepo();
const maintenanceWindowsRepository = overrides?.maintenanceWindowsRepository ?? createMaintenanceWindowsRepo();
const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository });
return { service, monitorsRepository, maintenanceWindowsRepository };
const jobsRepository = overrides?.jobsRepository ?? createJobsRepo();
const service = new MaintenanceWindowService({ monitorsRepository, maintenanceWindowsRepository, jobsRepository });
return { service, monitorsRepository, maintenanceWindowsRepository, jobsRepository };
};
const makeWindow = (overrides?: Partial<MaintenanceWindow>): MaintenanceWindow => ({
@@ -267,12 +275,14 @@ describe("MaintenanceWindowService", () => {
it("flips covered monitors to initializing when deleting an active window", async () => {
const maintenanceWindowsRepository = createMaintenanceWindowsRepo();
(maintenanceWindowsRepository.deleteById as jest.Mock).mockResolvedValue(makeActiveWindow({ monitorIds: ["mon-1", "mon-2"] }));
const { service, monitorsRepository } = createService({ maintenanceWindowsRepository });
const { service, monitorsRepository, jobsRepository } = createService({ maintenanceWindowsRepository });
await service.deleteMaintenanceWindow({ id: "mw-1", teamId: "team-1" });
expect(monitorsRepository.updateByIds).toHaveBeenCalledTimes(1);
expect(monitorsRepository.updateByIds).toHaveBeenCalledWith(["mon-1", "mon-2"], "team-1", { status: "initializing" }, ["paused"]);
// Monitors leaving maintenance are re-armed to run soon instead of waiting out the interval
expect(jobsRepository.markMonitorsDue).toHaveBeenCalledWith(["mon-1", "mon-2"], expect.any(Number));
});
it("excludes the deleted window from the overlap check", async () => {