Fix: Use Check IDs instead of timestamp to process checks

This commit is contained in:
Alex Holliday committed 2026-09-17 11:57:19 -07:00
1 parent 9102335d3e
commit fa0c0abcac
13 files changed
+491 -56

No files matched your search

@@ -0,0 +1,200 @@
import { describe, expect, it, jest, beforeAll, afterAll, beforeEach, afterEach } from "@jest/globals";
import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
import { CheckModel } from "../../src/domain/checks/check.model.ts";
import JobModel from "../../src/domain/jobs/job.model.ts";
import { CheckService } from "../../src/domain/checks/check.service.ts";
import { BufferService } from "../../src/service/bufferService.ts";
import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts";
import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts";
import { createMockLogger } from "../helpers/createMockLogger.ts";
import type { Job } from "../../src/domain/jobs/job.type.ts";
import type { Check } from "../../src/domain/checks/check.type.ts";
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
import type { MonitorStatusResponse } from "../../src/types/network.ts";
import type { ILogger } from "../../src/utils/logger.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The evaluate step hands work from the check job to the evaluator through the database:
// each worker's BufferService flushes its own in-memory batch on its own timer, then queues
// one evaluate job per monitor. The evaluator reads checks with createdAt strictly after the
// monitor's lastEvaluatedAt and advances that cursor to each check's createdAt.
//
// With two processing workers the flush timers are not aligned, so a later check produced on
// worker B can be flushed and evaluated before an earlier check produced on worker A has
// been flushed. When A's batch finally lands, its check is older than the cursor and the
// evaluate query never returns it. The check exists in the collection but never reaches the
// status window, stats, incidents or notifications.
//
// This drives the real time-series query, the real jobs collection, the real buffer flush and
// the real DBQueueWorker evaluate path against an in-process mongod, so the cursor semantics
// under test are MongoDB's, not a mock's.
let mongod: MongoMemoryServer;
beforeAll(async () => {
mongod = await MongoMemoryServer.create();
await mongoose.connect(mongod.getUri());
await CheckModel.createCollection(); // timeseries collections must exist before insert
await JobModel.init();
}, 120_000);
afterAll(async () => {
await mongoose.disconnect();
await mongod.stop();
});
beforeEach(async () => {
await CheckModel.deleteMany({});
await JobModel.deleteMany({});
});
// ── Helpers ──────────────────────────────────────────────────────────────────
const MONITOR_ID = new mongoose.Types.ObjectId().toString();
const TEAM_ID = new mongoose.Types.ObjectId().toString();
const T0 = Date.parse("2026-01-01T00:00:00.000Z");
const INTERVAL_MS = 30_000;
const makeMonitor = (): Monitor =>
({
id: MONITOR_ID,
teamId: TEAM_ID,
type: "http",
interval: INTERVAL_MS,
status: "up",
isActive: true,
statusWindow: [],
statusWindowSize: 5,
statusWindowThreshold: 60,
recentChecks: [],
lastEvaluatedAt: 0,
}) as unknown as Monitor;
const makeStatusResponse = (): MonitorStatusResponse => ({
monitorId: MONITOR_ID,
teamId: TEAM_ID,
type: "http",
status: true,
code: 200,
message: "OK",
responseTime: 100,
});
// Production stamps createdAt at the moment the probe ran, on the producing worker; pin it so
// the two workers' checks have a known order regardless of wall clock.
const producedAt = (check: Check, at: number): Check => {
const createdAt = new Date(at).toISOString();
return { ...check, createdAt, updatedAt: createdAt };
};
const stubSettingsService = { getSettings: () => ({ nodeEnv: "production" }) };
const stubGeoChecksService = { createGeoChecks: jest.fn<any>().mockResolvedValue([]) };
const stubDockerLogsService = { createDockerLogs: jest.fn<any>().mockResolvedValue(0) };
describe("Evaluate cursor with more than one processing worker", () => {
let logger: ILogger;
let monitorsRepository: InMemoryMonitorsRepository;
let checksRepository: MongoChecksRepository;
let checkService: CheckService;
let bufferA: BufferService;
let bufferB: BufferService;
let evaluatedCheckIds: string[];
let evaluator: DBQueueWorker;
let evaluatorJobsRepository: MongoJobsRepository;
// One BufferService per worker process, each flushing into the same collection and jobs table.
const createWorkerBuffer = (workerId: string) =>
new BufferService(
logger,
checkService,
stubGeoChecksService as any,
stubDockerLogsService as any,
stubSettingsService as any,
new MongoJobsRepository(workerId)
);
// Claim every due evaluate job and run it through the worker's real per-job path.
const runDueEvaluateJobs = async () => {
const jobs = await evaluatorJobsRepository.claimDueBatch("evaluate", 20, Date.now());
const runJob = (evaluator as unknown as { runJob: (job: Job) => Promise<void> }).runJob;
for (const job of jobs) await runJob(job);
return jobs.length;
};
beforeEach(() => {
logger = createMockLogger() as unknown as ILogger;
monitorsRepository = new InMemoryMonitorsRepository();
monitorsRepository.seed(makeMonitor());
checksRepository = new MongoChecksRepository(logger);
checkService = new CheckService(monitorsRepository, logger, checksRepository);
bufferA = createWorkerBuffer("worker-a");
bufferB = createWorkerBuffer("worker-b");
evaluatedCheckIds = [];
evaluatorJobsRepository = new MongoJobsRepository("worker-b");
evaluator = new DBQueueWorker({
logger,
isDbConnected: () => true,
jobsRepository: evaluatorJobsRepository,
monitorsRepository,
checksRepository,
checkService,
bufferService: bufferB,
checkProducer: { produce: jest.fn<any>() },
checkEvaluator: {
evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
evaluatedCheckIds.push(check.id);
return {
monitor,
status,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 },
decision: {},
};
},
} as any,
geoCheckPipeline: { run: jest.fn<any>() },
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
helper: {} as any,
queueWorkersRepository: {} as any,
queueMode: "worker",
queuePrimaryProcesses: true,
workerId: "worker-b",
});
});
afterEach(async () => {
await bufferA.shutdown();
await bufferB.shutdown();
});
it("evaluates every persisted check even when a later worker's batch flushes first", async () => {
// t=0: worker A holds the check job, probes, and buffers the result. Its flush timer has not fired.
const c1 = producedAt(checkService.toCheck(makeStatusResponse())!, T0);
bufferA.addToBuffer(c1);
// t=30s: worker B claims the next run of the same job, probes, and buffers the result.
const c2 = producedAt(checkService.toCheck(makeStatusResponse())!, T0 + INTERVAL_MS);
bufferB.addToBuffer(c2);
// t=45s: worker B's flush timer fires first. c2 is inserted and an evaluate job is queued.
await bufferB.flushBuffer();
expect(await runDueEvaluateJobs()).toBe(1);
expect(evaluatedCheckIds).toEqual([c2.id]);
// t=60s: worker A's flush timer fires. c1 is inserted and the evaluate job is pulled due again.
await bufferA.flushBuffer();
expect(await runDueEvaluateJobs()).toBe(1);
// Both checks are durably stored and visible in history.
const stored = await CheckModel.find({ "metadata.monitorId": new mongoose.Types.ObjectId(MONITOR_ID) }).lean();
expect(stored.map((doc) => String(doc._id)).sort()).toEqual([c1.id, c2.id].sort());
// Every stored check must reach the evaluator exactly once.
expect(evaluatedCheckIds).toHaveLength(2);
expect(evaluatedCheckIds).toContain(c1.id);
});
});
+112 -14
View File
@@ -3,7 +3,7 @@ import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
import { BACKOFF_MS, LOCK_MS, PARKED, type Job, type JobType } from "../../src/domain/jobs/job.type.ts";
import { BACKOFF_MS, LOCK_MS, PARKED, type Job, type JobType, type PendingCheck } from "../../src/domain/jobs/job.type.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
@@ -58,6 +58,9 @@ const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string
const WORKER_ID = "worker-under-test";
// Pending entries as ingest pushes them: one per inserted check, stamped with the check's createdAt.
const pending = (...checkIds: string[]): PendingCheck[] => checkIds.map((checkId, i) => ({ checkId, createdAt: NOW + i }));
describe("MongoJobsRepository", () => {
let repo: MongoJobsRepository;
@@ -239,6 +242,23 @@ describe("MongoJobsRepository", () => {
expect(row?.failCount).toBe(2);
expect(row?.lastFailReason).toBe("timeout");
});
it("re-arms instead of parking when checks were pushed while the run held the lease", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c-late"), // arrived mid-run, not yet pulled
});
expect(await repo.recordOneShot("evaluate:mon-1", NOW + 1_000)).toBe(true);
const row = await readRow("evaluate:mon-1");
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // due now, not PARKED
expect(row?.lockedBy).toBeNull();
expect(row?.runCount).toBe(1);
});
});
// ── renewLocks (lease renewal) ───────────────────────────────────────────────
@@ -325,10 +345,10 @@ describe("MongoJobsRepository", () => {
});
});
// ── upsertEvaluate (handoff + $min coalescing) ───────────────────────────────
// ── upsertEvaluate (handoff: $push ids + $min due time) ──────────────────────
describe("upsertEvaluate", () => {
it("creates a single evaluate row keyed by monitor", async () => {
await repo.upsertEvaluate("mon-1", NOW);
it("creates a single evaluate row keyed by monitor carrying the pushed ids", async () => {
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.type).toBe("evaluate");
@@ -336,11 +356,12 @@ describe("MongoJobsRepository", () => {
expect(row?.nextScheduledAt).toBe(NOW);
expect(row?.intervalMs).toBeNull();
expect(row?.isActive).toBe(true);
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
});
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
await repo.upsertEvaluate("mon-1", NOW);
await repo.upsertEvaluate("mon-1", NOW + 9_000);
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
await repo.upsertEvaluate("mon-1", pending("c2"), NOW + 9_000);
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
expect(rows).toHaveLength(1);
@@ -348,19 +369,96 @@ describe("MongoJobsRepository", () => {
});
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
await repo.upsertEvaluate("mon-1", NOW + 9_000);
await repo.upsertEvaluate("mon-1", NOW);
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 9_000);
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.nextScheduledAt).toBe(NOW);
});
it("concurrent upserts for one monitor produce exactly one row", async () => {
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW)));
it("accumulates ids across flushes so a late batch is never lost", async () => {
await repo.upsertEvaluate("mon-1", pending("c2"), NOW); // later check, flushed first
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 15_000); // earlier check, flushed later
const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" });
expect(count).toBe(1);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2", "c1"]);
});
it("keeps pushing onto a row that is currently leased by a worker", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: "other-worker",
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1"),
});
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.lockedBy).toBe("other-worker"); // lease untouched
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1", "c2"]);
});
it("concurrent upserts for one monitor produce exactly one row holding every id", async () => {
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
await Promise.all(workers.map((w, i) => w.upsertEvaluate("mon-1", pending(`c${i}`), NOW)));
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
expect(rows).toHaveLength(1);
expect(rows[0].pendingChecks).toHaveLength(20);
});
});
// ── pullEvaluated (evaluator removes applied ids) ───────────────────────────
describe("pullEvaluated", () => {
it("removes only the listed ids and leaves the rest", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1", "c2", "c3"),
});
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1", "c3"])).toBe(true);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2"]);
});
it("is fenced on the lease: another worker's row is untouched", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: "other-worker",
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1"),
});
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(false);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1"]);
});
it("is a no-op for an empty id list", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1"),
});
expect(await repo.pullEvaluated("evaluate:mon-1", [])).toBe(true);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks).toHaveLength(1);
});
});
@@ -487,7 +585,7 @@ describe("MongoJobsRepository", () => {
it("deleteById drops every row for the monitor", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
await repo.upsertEvaluate("mon-1", NOW);
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
await repo.deleteById("mon-1");