bug fixes

This commit is contained in:
Alex Holliday committed 2026-09-17 12:30:56 -07:00
1 parent fa0c0abcac
commit f299098548
9 files changed
+118 -26

No files matched your search

@@ -17,20 +17,20 @@ import type { MonitorStatusResponse } from "../../src/types/network.ts";
import type { ILogger } from "../../src/utils/logger.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The evaluate step hands work from the check job to the evaluator through the database:
// each worker's BufferService flushes its own in-memory batch on its own timer, then queues
// one evaluate job per monitor. The evaluator reads checks with createdAt strictly after the
// monitor's lastEvaluatedAt and advances that cursor to each check's createdAt.
// Regression test for the dropped-check bug in the check-to-evaluate handoff.
//
// With two processing workers the flush timers are not aligned, so a later check produced on
// worker B can be flushed and evaluated before an earlier check produced on worker A has
// been flushed. When A's batch finally lands, its check is older than the cursor and the
// evaluate query never returns it. The check exists in the collection but never reaches the
// status window, stats, incidents or notifications.
// The evaluate step used to be driven by a timestamp cursor: the evaluator read checks with
// createdAt strictly after the monitor's lastEvaluatedAt and advanced that cursor to each
// check's createdAt. Each worker's BufferService flushes its own in-memory batch on its own
// timer, so with two processing workers a later check produced on worker B could be flushed and
// evaluated before an earlier check produced on worker A had been flushed. When A's batch landed,
// its check was older than the cursor and the evaluate query never returned it. The check existed
// in the collection but never reached the status window, stats, incidents or notifications.
//
// This drives the real time-series query, the real jobs collection, the real buffer flush and
// the real DBQueueWorker evaluate path against an in-process mongod, so the cursor semantics
// under test are MongoDB's, not a mock's.
// The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each
// is applied, so a late batch is always evaluated. This drives the real time-series query, the
// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an
// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's.
let mongod: MongoMemoryServer;
@@ -376,6 +376,14 @@ describe("MongoJobsRepository", () => {
expect(row?.nextScheduledAt).toBe(NOW);
});
it("never duplicates an entry when the same batch is armed twice (retry after a failed arm)", async () => {
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW + 1_000);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
});
it("accumulates ids across flushes so a late batch is never lost", async () => {
await repo.upsertEvaluate("mon-1", pending("c2"), NOW); // later check, flushed first
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 15_000); // earlier check, flushed later
@@ -445,6 +453,19 @@ describe("MongoJobsRepository", () => {
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1"]);
});
it("reports the lease as held even when the ids were already pulled", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: [],
});
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(true);
});
it("is a no-op for an empty id list", async () => {
await seedJob({
_id: "evaluate:mon-1",
@@ -474,6 +495,7 @@ describe("MongoJobsRepository", () => {
intervalMs: 60_000,
lockedBy: null,
lockedUntil: null,
pendingChecks: [],
runCount: 0,
failCount: 0,
lastFinishedAt: null,
@@ -505,6 +527,7 @@ describe("MongoJobsRepository", () => {
intervalMs: 86_400_000,
lockedBy: null,
lockedUntil: null,
pendingChecks: [],
runCount: 0,
failCount: 0,
lastFinishedAt: null,
@@ -407,6 +407,24 @@ describe("BufferService", () => {
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
});
it("retries a failed arm on the next flush without re-storing the checks", async () => {
const { service, checkService, jobsRepository, logger } = createService();
(jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed"));
service.addToBuffer(makeCheck({ id: "c1" }));
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
await service.flushBuffer(); // buffer is empty, only the retry runs
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number));
await service.flushBuffer(); // nothing left to retry
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
});
it("ingestChecks with an empty batch neither writes nor arms", async () => {
const { service, checkService, jobsRepository } = createService();
@@ -421,6 +421,22 @@ describe("DBQueueWorker", () => {
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
});
it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
const job = makeEvaluateJob("c1", "c2", "c3");
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) };
const jobsRepository = {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
};
const { mocks } = await start({ mocks: { jobsRepository, checksRepository } });
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2);
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" }));
});
it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
const job = makeEvaluateJob("c1", "c-gone");
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up