add upsertCleanup to schedule immediate cleanup jobs, cleanup orphaned jobs

This commit is contained in:
Alex Holliday committed 2026-06-17 09:38:21 -07:00
1 parent 699883e0b1
commit 7244b35091
10 files changed
+871 -880

No files matched your search

@@ -347,6 +347,34 @@ describe("MongoJobsRepository", () => {
expect(row?.runCount).toBe(7);
});
it("upsertCleanupJob resets nextScheduledAt on an existing row so it is due again on restart", async () => {
const job: Job = {
id: "cleanup-orphaned",
type: "cleanup-orphaned",
refId: null,
isActive: true,
nextScheduledAt: NOW,
intervalMs: 86_400_000,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
};
await repo.upsertCleanupJob(job);
// row has already advanced a day into the future after a previous run
await JobModel.updateOne({ _id: "cleanup-orphaned" }, { $set: { nextScheduledAt: NOW + 86_400_000, runCount: 3 } });
// a restart reconcile re-upserts and pulls it due now
await repo.upsertCleanupJob({ ...job, nextScheduledAt: NOW + 1_000 });
const row = await readRow("cleanup-orphaned");
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // schedule reset ($set)
expect(row?.runCount).toBe(3); // run history untouched ($setOnInsert)
});
it("setActiveById flips check and geo rows together", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
@@ -378,6 +406,20 @@ describe("MongoJobsRepository", () => {
expect(await JobModel.countDocuments({ refId: "mon-1" })).toBe(0);
});
it("deleteByMonitorIdsNotIn drops jobs for missing monitors but keeps valid and global rows", async () => {
await seedJob({ _id: "check:keep", refId: "keep", type: "check" });
await seedJob({ _id: "geo:keep", refId: "keep", type: "geo-check" });
await seedJob({ _id: "check:gone", refId: "gone", type: "check" });
await seedJob({ _id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" });
await seedJob({ _id: "cleanup-retention", refId: null, type: "cleanup-retention" });
const deleted = await repo.deleteByMonitorIdsNotIn(["keep"]);
expect(deleted).toBe(1); // only check:gone
const remaining = (await JobModel.find().lean<JobDocument[]>()).map((d) => d._id).sort();
expect(remaining).toEqual(["check:keep", "cleanup-orphaned", "cleanup-retention", "geo:keep"]);
});
it("findById returns all of a monitor's rows", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
@@ -65,6 +65,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
const jobsRepository = {
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
upsertJob: jest.fn<any>().mockResolvedValue(true),
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
recordSuccess: jest.fn<any>().mockResolvedValue(true),
recordFailure: jest.fn<any>().mockResolvedValue(true),
@@ -183,8 +184,9 @@ describe("DBQueueWorker", () => {
const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
expect(seededTypes).toContain("check");
expect(seededTypes).toContain("cleanup-orphaned");
expect(seededTypes).toContain("cleanup-retention");
const cleanupTypes = mocks.jobsRepository.upsertCleanupJob.mock.calls.map((c: any[]) => c[0].type);
expect(cleanupTypes).toContain("cleanup-orphaned");
expect(cleanupTypes).toContain("cleanup-retention");
});
it("primary mode also seeds a geo-check row for geo-enabled monitors", async () => {
@@ -204,6 +206,7 @@ describe("DBQueueWorker", () => {
it("worker mode does not reconcile (never seeds jobs)", async () => {
const { mocks } = await start({ queueMode: "worker" });
expect(mocks.jobsRepository.upsertJob).not.toHaveBeenCalled();
expect(mocks.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("starts a polling loop per job type, each claiming up to its concurrency cap", async () => {
@@ -17,6 +17,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
findAllMonitorIds: jest.fn().mockResolvedValue(["m1"]),
deleteByTeamIdsNotIn: jest.fn().mockResolvedValue(0),
};
const jobsRepository = {
deleteByMonitorIdsNotIn: jest.fn().mockResolvedValue(0),
};
const teamsRepository = {
findAllTeamIds: jest.fn().mockResolvedValue(["team"]),
};
@@ -48,6 +51,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
checkService,
settingsService,
monitorsRepository,
jobsRepository,
teamsRepository,
monitorStatsRepository,
checksRepository,
@@ -64,6 +68,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
defaults.checkService as any,
defaults.settingsService as any,
defaults.monitorsRepository as any,
defaults.jobsRepository as any,
defaults.teamsRepository as any,
defaults.monitorStatsRepository as any,
defaults.checksRepository as any,
@@ -170,6 +175,7 @@ describe("WorkerHelper", () => {
(defaults.checksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(4);
(defaults.incidentsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(1);
(defaults.geoChecksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(5);
(defaults.jobsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(6);
const job = helper.getCleanupOrphanedJob();
await job();
@@ -181,11 +187,13 @@ describe("WorkerHelper", () => {
expect(defaults.checksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.incidentsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.geoChecksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.jobsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("2 orphaned monitors") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("3 orphaned monitor stats") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("4 orphaned checks") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 orphaned incidents") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("5 orphaned geo checks") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("6 orphaned jobs") }));
});
it("skips info logs when no orphaned data is found", async () => {