Fix: Use Check IDs instead of timestamp to process checks
This commit is contained in:
1 parent
9102335d3e
commit
fa0c0abcac
13 files changed
+491
-56
No files matched your search
@@ -0,0 +1,200 @@
|
||||
import { describe, expect, it, jest, beforeAll, afterAll, beforeEach, afterEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
|
||||
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
||||
import { CheckModel } from "../../src/domain/checks/check.model.ts";
|
||||
import JobModel from "../../src/domain/jobs/job.model.ts";
|
||||
import { CheckService } from "../../src/domain/checks/check.service.ts";
|
||||
import { BufferService } from "../../src/service/bufferService.ts";
|
||||
import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts";
|
||||
import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts";
|
||||
import { createMockLogger } from "../helpers/createMockLogger.ts";
|
||||
import type { Job } from "../../src/domain/jobs/job.type.ts";
|
||||
import type { Check } from "../../src/domain/checks/check.type.ts";
|
||||
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||
import type { ILogger } from "../../src/utils/logger.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The evaluate step hands work from the check job to the evaluator through the database:
|
||||
// each worker's BufferService flushes its own in-memory batch on its own timer, then queues
|
||||
// one evaluate job per monitor. The evaluator reads checks with createdAt strictly after the
|
||||
// monitor's lastEvaluatedAt and advances that cursor to each check's createdAt.
|
||||
//
|
||||
// With two processing workers the flush timers are not aligned, so a later check produced on
|
||||
// worker B can be flushed and evaluated before an earlier check produced on worker A has
|
||||
// been flushed. When A's batch finally lands, its check is older than the cursor and the
|
||||
// evaluate query never returns it. The check exists in the collection but never reaches the
|
||||
// status window, stats, incidents or notifications.
|
||||
//
|
||||
// This drives the real time-series query, the real jobs collection, the real buffer flush and
|
||||
// the real DBQueueWorker evaluate path against an in-process mongod, so the cursor semantics
|
||||
// under test are MongoDB's, not a mock's.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await CheckModel.createCollection(); // timeseries collections must exist before insert
|
||||
await JobModel.init();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await CheckModel.deleteMany({});
|
||||
await JobModel.deleteMany({});
|
||||
});
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const MONITOR_ID = new mongoose.Types.ObjectId().toString();
|
||||
const TEAM_ID = new mongoose.Types.ObjectId().toString();
|
||||
const T0 = Date.parse("2026-01-01T00:00:00.000Z");
|
||||
const INTERVAL_MS = 30_000;
|
||||
|
||||
const makeMonitor = (): Monitor =>
|
||||
({
|
||||
id: MONITOR_ID,
|
||||
teamId: TEAM_ID,
|
||||
type: "http",
|
||||
interval: INTERVAL_MS,
|
||||
status: "up",
|
||||
isActive: true,
|
||||
statusWindow: [],
|
||||
statusWindowSize: 5,
|
||||
statusWindowThreshold: 60,
|
||||
recentChecks: [],
|
||||
lastEvaluatedAt: 0,
|
||||
}) as unknown as Monitor;
|
||||
|
||||
const makeStatusResponse = (): MonitorStatusResponse => ({
|
||||
monitorId: MONITOR_ID,
|
||||
teamId: TEAM_ID,
|
||||
type: "http",
|
||||
status: true,
|
||||
code: 200,
|
||||
message: "OK",
|
||||
responseTime: 100,
|
||||
});
|
||||
|
||||
// Production stamps createdAt at the moment the probe ran, on the producing worker; pin it so
|
||||
// the two workers' checks have a known order regardless of wall clock.
|
||||
const producedAt = (check: Check, at: number): Check => {
|
||||
const createdAt = new Date(at).toISOString();
|
||||
return { ...check, createdAt, updatedAt: createdAt };
|
||||
};
|
||||
|
||||
const stubSettingsService = { getSettings: () => ({ nodeEnv: "production" }) };
|
||||
const stubGeoChecksService = { createGeoChecks: jest.fn<any>().mockResolvedValue([]) };
|
||||
const stubDockerLogsService = { createDockerLogs: jest.fn<any>().mockResolvedValue(0) };
|
||||
|
||||
describe("Evaluate cursor with more than one processing worker", () => {
|
||||
let logger: ILogger;
|
||||
let monitorsRepository: InMemoryMonitorsRepository;
|
||||
let checksRepository: MongoChecksRepository;
|
||||
let checkService: CheckService;
|
||||
let bufferA: BufferService;
|
||||
let bufferB: BufferService;
|
||||
let evaluatedCheckIds: string[];
|
||||
let evaluator: DBQueueWorker;
|
||||
let evaluatorJobsRepository: MongoJobsRepository;
|
||||
|
||||
// One BufferService per worker process, each flushing into the same collection and jobs table.
|
||||
const createWorkerBuffer = (workerId: string) =>
|
||||
new BufferService(
|
||||
logger,
|
||||
checkService,
|
||||
stubGeoChecksService as any,
|
||||
stubDockerLogsService as any,
|
||||
stubSettingsService as any,
|
||||
new MongoJobsRepository(workerId)
|
||||
);
|
||||
|
||||
// Claim every due evaluate job and run it through the worker's real per-job path.
|
||||
const runDueEvaluateJobs = async () => {
|
||||
const jobs = await evaluatorJobsRepository.claimDueBatch("evaluate", 20, Date.now());
|
||||
const runJob = (evaluator as unknown as { runJob: (job: Job) => Promise<void> }).runJob;
|
||||
for (const job of jobs) await runJob(job);
|
||||
return jobs.length;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
logger = createMockLogger() as unknown as ILogger;
|
||||
monitorsRepository = new InMemoryMonitorsRepository();
|
||||
monitorsRepository.seed(makeMonitor());
|
||||
checksRepository = new MongoChecksRepository(logger);
|
||||
checkService = new CheckService(monitorsRepository, logger, checksRepository);
|
||||
bufferA = createWorkerBuffer("worker-a");
|
||||
bufferB = createWorkerBuffer("worker-b");
|
||||
|
||||
evaluatedCheckIds = [];
|
||||
evaluatorJobsRepository = new MongoJobsRepository("worker-b");
|
||||
evaluator = new DBQueueWorker({
|
||||
logger,
|
||||
isDbConnected: () => true,
|
||||
jobsRepository: evaluatorJobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
bufferService: bufferB,
|
||||
checkProducer: { produce: jest.fn<any>() },
|
||||
checkEvaluator: {
|
||||
evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
|
||||
evaluatedCheckIds.push(check.id);
|
||||
return {
|
||||
monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 },
|
||||
decision: {},
|
||||
};
|
||||
},
|
||||
} as any,
|
||||
geoCheckPipeline: { run: jest.fn<any>() },
|
||||
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
|
||||
helper: {} as any,
|
||||
queueWorkersRepository: {} as any,
|
||||
queueMode: "worker",
|
||||
queuePrimaryProcesses: true,
|
||||
workerId: "worker-b",
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await bufferA.shutdown();
|
||||
await bufferB.shutdown();
|
||||
});
|
||||
|
||||
it("evaluates every persisted check even when a later worker's batch flushes first", async () => {
|
||||
// t=0: worker A holds the check job, probes, and buffers the result. Its flush timer has not fired.
|
||||
const c1 = producedAt(checkService.toCheck(makeStatusResponse())!, T0);
|
||||
bufferA.addToBuffer(c1);
|
||||
|
||||
// t=30s: worker B claims the next run of the same job, probes, and buffers the result.
|
||||
const c2 = producedAt(checkService.toCheck(makeStatusResponse())!, T0 + INTERVAL_MS);
|
||||
bufferB.addToBuffer(c2);
|
||||
|
||||
// t=45s: worker B's flush timer fires first. c2 is inserted and an evaluate job is queued.
|
||||
await bufferB.flushBuffer();
|
||||
expect(await runDueEvaluateJobs()).toBe(1);
|
||||
expect(evaluatedCheckIds).toEqual([c2.id]);
|
||||
|
||||
// t=60s: worker A's flush timer fires. c1 is inserted and the evaluate job is pulled due again.
|
||||
await bufferA.flushBuffer();
|
||||
expect(await runDueEvaluateJobs()).toBe(1);
|
||||
|
||||
// Both checks are durably stored and visible in history.
|
||||
const stored = await CheckModel.find({ "metadata.monitorId": new mongoose.Types.ObjectId(MONITOR_ID) }).lean();
|
||||
expect(stored.map((doc) => String(doc._id)).sort()).toEqual([c1.id, c2.id].sort());
|
||||
|
||||
// Every stored check must reach the evaluator exactly once.
|
||||
expect(evaluatedCheckIds).toHaveLength(2);
|
||||
expect(evaluatedCheckIds).toContain(c1.id);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user