bug fixes
This commit is contained in:
1 parent
fa0c0abcac
commit
f299098548
9 files changed
+118
-26
No files matched your search
@@ -17,20 +17,20 @@ import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||
import type { ILogger } from "../../src/utils/logger.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The evaluate step hands work from the check job to the evaluator through the database:
|
||||
// each worker's BufferService flushes its own in-memory batch on its own timer, then queues
|
||||
// one evaluate job per monitor. The evaluator reads checks with createdAt strictly after the
|
||||
// monitor's lastEvaluatedAt and advances that cursor to each check's createdAt.
|
||||
// Regression test for the dropped-check bug in the check-to-evaluate handoff.
|
||||
//
|
||||
// With two processing workers the flush timers are not aligned, so a later check produced on
|
||||
// worker B can be flushed and evaluated before an earlier check produced on worker A has
|
||||
// been flushed. When A's batch finally lands, its check is older than the cursor and the
|
||||
// evaluate query never returns it. The check exists in the collection but never reaches the
|
||||
// status window, stats, incidents or notifications.
|
||||
// The evaluate step used to be driven by a timestamp cursor: the evaluator read checks with
|
||||
// createdAt strictly after the monitor's lastEvaluatedAt and advanced that cursor to each
|
||||
// check's createdAt. Each worker's BufferService flushes its own in-memory batch on its own
|
||||
// timer, so with two processing workers a later check produced on worker B could be flushed and
|
||||
// evaluated before an earlier check produced on worker A had been flushed. When A's batch landed,
|
||||
// its check was older than the cursor and the evaluate query never returned it. The check existed
|
||||
// in the collection but never reached the status window, stats, incidents or notifications.
|
||||
//
|
||||
// This drives the real time-series query, the real jobs collection, the real buffer flush and
|
||||
// the real DBQueueWorker evaluate path against an in-process mongod, so the cursor semantics
|
||||
// under test are MongoDB's, not a mock's.
|
||||
// The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each
|
||||
// is applied, so a late batch is always evaluated. This drives the real time-series query, the
|
||||
// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an
|
||||
// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
|
||||
@@ -376,6 +376,14 @@ describe("MongoJobsRepository", () => {
|
||||
expect(row?.nextScheduledAt).toBe(NOW);
|
||||
});
|
||||
|
||||
it("never duplicates an entry when the same batch is armed twice (retry after a failed arm)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
|
||||
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW + 1_000);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
|
||||
});
|
||||
|
||||
it("accumulates ids across flushes so a late batch is never lost", async () => {
|
||||
await repo.upsertEvaluate("mon-1", pending("c2"), NOW); // later check, flushed first
|
||||
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 15_000); // earlier check, flushed later
|
||||
@@ -445,6 +453,19 @@ describe("MongoJobsRepository", () => {
|
||||
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1"]);
|
||||
});
|
||||
|
||||
it("reports the lease as held even when the ids were already pulled", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
lockedBy: ownedBy(repo),
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
pendingChecks: [],
|
||||
});
|
||||
|
||||
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(true);
|
||||
});
|
||||
|
||||
it("is a no-op for an empty id list", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
@@ -474,6 +495,7 @@ describe("MongoJobsRepository", () => {
|
||||
intervalMs: 60_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
pendingChecks: [],
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
@@ -505,6 +527,7 @@ describe("MongoJobsRepository", () => {
|
||||
intervalMs: 86_400_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
pendingChecks: [],
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
|
||||
@@ -407,6 +407,24 @@ describe("BufferService", () => {
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
|
||||
});
|
||||
|
||||
it("retries a failed arm on the next flush without re-storing the checks", async () => {
|
||||
const { service, checkService, jobsRepository, logger } = createService();
|
||||
(jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed"));
|
||||
service.addToBuffer(makeCheck({ id: "c1" }));
|
||||
|
||||
await service.flushBuffer();
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
|
||||
|
||||
await service.flushBuffer(); // buffer is empty, only the retry runs
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number));
|
||||
|
||||
await service.flushBuffer(); // nothing left to retry
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("ingestChecks with an empty batch neither writes nor arms", async () => {
|
||||
const { service, checkService, jobsRepository } = createService();
|
||||
|
||||
|
||||
@@ -421,6 +421,22 @@ describe("DBQueueWorker", () => {
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
|
||||
});
|
||||
|
||||
it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2", "c3");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) };
|
||||
const jobsRepository = {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository, checksRepository } });
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
|
||||
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" }));
|
||||
});
|
||||
|
||||
it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
|
||||
const job = makeEvaluateJob("c1", "c-gone");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up
|
||||
|
||||
Reference in new issue
Block a user