Merge pull request #3728 from bluewave-labs/feat/db-queue
feat: db queue
This commit is contained in:
42 files changed
+3499
-3514
No files matched your search
@@ -2,6 +2,8 @@ import { jest } from "@jest/globals";
|
||||
import { WorkerHelper } from "../../src/worker/worker.helper.ts";
|
||||
import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "../../src/worker/worker.check-pipeline.ts";
|
||||
import { CheckProducer } from "../../src/worker/worker.check-producer.ts";
|
||||
import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts";
|
||||
import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts";
|
||||
import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts";
|
||||
import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts";
|
||||
@@ -131,7 +133,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
}),
|
||||
};
|
||||
const checkService = {
|
||||
buildCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => {
|
||||
toCheck: jest.fn().mockImplementation((response: MonitorStatusResponse) => {
|
||||
return makeCheck(response.status, response.code);
|
||||
}),
|
||||
};
|
||||
@@ -149,16 +151,16 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
|
||||
const geoChecksService = { buildGeoCheck: jest.fn() };
|
||||
|
||||
const checkPipeline = new CheckPipeline(
|
||||
const checkProducer = new CheckProducer(
|
||||
monitorsRepo as any,
|
||||
maintenanceWindowsRepo as any,
|
||||
checkService as any,
|
||||
networkService as any,
|
||||
bufferStub as any,
|
||||
new MonitorStatusPolicy(),
|
||||
statusService as any,
|
||||
logger
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
|
||||
const checkPipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepo as any, geoChecksService as any, bufferStub as any, logger);
|
||||
|
||||
const notificationReactor = new NotificationReactor(notificationsService as any);
|
||||
@@ -170,6 +172,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
checkService as any,
|
||||
{ getDBSettings: jest.fn() } as any,
|
||||
monitorsRepo as any,
|
||||
{ deleteByMonitorIdsNotIn: jest.fn() } as any,
|
||||
{ findAllTeamIds: jest.fn() } as any,
|
||||
createStubMonitorStatsRepo() as any,
|
||||
createStubChecksRepo() as any,
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import { MonitorModel } from "../../src/domain/monitors/monitor.model.ts";
|
||||
import { backfillMonitorLastEvaluatedAt } from "../../src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// Legacy monitors are inserted through the raw driver, NOT MonitorModel.create — the
|
||||
// schema default (lastEvaluatedAt: 0) would otherwise stamp the field and the migration's
|
||||
// { $exists: false } filter would skip it, defeating the test.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await MonitorModel.collection.deleteMany({});
|
||||
});
|
||||
|
||||
describe("0008_backfillMonitorLastEvaluatedAt", () => {
|
||||
it("stamps the upgrade moment (not 0) on monitors missing lastEvaluatedAt", async () => {
|
||||
await MonitorModel.collection.insertOne({ name: "legacy" }); // pre-upgrade doc, no lastEvaluatedAt
|
||||
|
||||
const before = Date.now();
|
||||
await backfillMonitorLastEvaluatedAt();
|
||||
|
||||
const doc = await MonitorModel.collection.findOne({ name: "legacy" });
|
||||
// Must be a recent timestamp — backfilling 0 would replay the whole check history on the first evaluate.
|
||||
expect(doc?.lastEvaluatedAt).not.toBe(0);
|
||||
expect(doc?.lastEvaluatedAt).toBeGreaterThanOrEqual(before);
|
||||
expect(doc?.lastEvaluatedAt).toBeLessThanOrEqual(Date.now());
|
||||
});
|
||||
|
||||
it("leaves monitors that already have lastEvaluatedAt untouched (idempotent)", async () => {
|
||||
await MonitorModel.collection.insertOne({ name: "current", lastEvaluatedAt: 12345 });
|
||||
|
||||
await backfillMonitorLastEvaluatedAt();
|
||||
|
||||
const doc = await MonitorModel.collection.findOne({ name: "current" });
|
||||
expect(doc?.lastEvaluatedAt).toBe(12345);
|
||||
});
|
||||
|
||||
it("drops the legacy jobs collection left behind by the old scheduler", async () => {
|
||||
const db = mongoose.connection.db!;
|
||||
await db.collection("jobs").insertOne({ id: "legacy-job", template: "monitor-job" }); // old-scheduler row
|
||||
|
||||
await backfillMonitorLastEvaluatedAt();
|
||||
|
||||
const exists = await db.listCollections({ name: "jobs" }).toArray();
|
||||
expect(exists).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not throw when there is no jobs collection to drop", async () => {
|
||||
await expect(backfillMonitorLastEvaluatedAt()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,461 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
||||
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
|
||||
import { BACKOFF_MS, LOCK_MS, type Job, type JobType } from "../../src/domain/jobs/job.type.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
|
||||
// single-document atomicity on findOneAndUpdate, so it can only be exercised against a
|
||||
// live engine. We boot a throwaway in-process mongod for this suite.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await JobModel.init(); // build the indexes the claim scan relies on
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await JobModel.deleteMany({});
|
||||
});
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const NOW = 1_000_000;
|
||||
|
||||
// Insert a job row directly, full control over every field. Defaults describe a due,
|
||||
// unlocked, active check job.
|
||||
const seedJob = (overrides: Partial<JobDocument> = {}): Promise<JobDocument> =>
|
||||
JobModel.create({
|
||||
_id: overrides._id ?? `check:${overrides.refId ?? "mon-1"}`,
|
||||
type: "check" as JobType,
|
||||
refId: "mon-1",
|
||||
isActive: true,
|
||||
nextScheduledAt: NOW,
|
||||
intervalMs: 60_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const readRow = (id: string) => JobModel.findById(id).lean<JobDocument>();
|
||||
|
||||
// record*/renewLocks are fenced on the repo's private workerId, so a pre-seeded "held"
|
||||
// lock must be stamped with the same id the repo will check against.
|
||||
const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string }).workerId;
|
||||
|
||||
const WORKER_ID = "worker-under-test";
|
||||
|
||||
describe("MongoJobsRepository", () => {
|
||||
let repo: MongoJobsRepository;
|
||||
|
||||
beforeEach(() => {
|
||||
repo = new MongoJobsRepository(WORKER_ID);
|
||||
});
|
||||
|
||||
// ── claimDue ───────────────────────────────────────────────────────────────
|
||||
describe("claimDue", () => {
|
||||
it("claims a due, unlocked job and leases it", async () => {
|
||||
await seedJob();
|
||||
|
||||
const claimed = await repo.claimDue("check", NOW);
|
||||
|
||||
expect(claimed?.id).toBe("check:mon-1");
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).not.toBeNull();
|
||||
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
});
|
||||
|
||||
it("returns null when nothing is due yet", async () => {
|
||||
await seedJob({ nextScheduledAt: NOW + 1 });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("skips inactive (paused) jobs", async () => {
|
||||
await seedJob({ isActive: false });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("only claims the requested type", async () => {
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not claim a job held by a live lease", async () => {
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
|
||||
expect(await repo.claimDue("check", NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("reclaims a job whose lease has expired (crash recovery)", async () => {
|
||||
await seedJob({ lockedBy: "dead-worker", lockedUntil: NOW - 1 });
|
||||
|
||||
const claimed = await repo.claimDue("check", NOW);
|
||||
|
||||
expect(claimed?.id).toBe("check:mon-1");
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).not.toBe("dead-worker");
|
||||
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
});
|
||||
|
||||
it("returns the oldest-due job first", async () => {
|
||||
await seedJob({ _id: "check:newer", refId: "newer", nextScheduledAt: NOW });
|
||||
await seedJob({ _id: "check:older", refId: "older", nextScheduledAt: NOW - 5_000 });
|
||||
|
||||
const claimed = await repo.claimDue("check", NOW);
|
||||
expect(claimed?.id).toBe("check:older");
|
||||
});
|
||||
|
||||
it("lets exactly one of many concurrent claimers win a single due job", async () => {
|
||||
await seedJob();
|
||||
|
||||
// Distinct repo instances = distinct workerIds = realistic competing workers.
|
||||
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
||||
const results = await Promise.all(workers.map((w) => w.claimDue("check", NOW)));
|
||||
|
||||
const winners = results.filter((r): r is Job => r !== null);
|
||||
expect(winners).toHaveLength(1);
|
||||
expect(winners[0].id).toBe("check:mon-1");
|
||||
});
|
||||
|
||||
it("hands two concurrent claimers two distinct due jobs (no double-claim)", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a" });
|
||||
await seedJob({ _id: "check:b", refId: "b" });
|
||||
|
||||
const [w1, w2] = [new MongoJobsRepository("worker-1"), new MongoJobsRepository("worker-2")];
|
||||
const [r1, r2] = await Promise.all([w1.claimDue("check", NOW), w2.claimDue("check", NOW)]);
|
||||
|
||||
expect(r1).not.toBeNull();
|
||||
expect(r2).not.toBeNull();
|
||||
expect(r1?.id).not.toBe(r2?.id);
|
||||
expect([r1?.id, r2?.id].sort()).toEqual(["check:a", "check:b"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordSuccess (fixed-rate reschedule) ────────────────────────────────────
|
||||
describe("recordSuccess", () => {
|
||||
it("reschedules fixed-rate from the scheduled tick and releases the lease", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, runCount: 4 });
|
||||
|
||||
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_500);
|
||||
|
||||
expect(ok).toBe(true);
|
||||
const row = await readRow("check:mon-1");
|
||||
// next tick measured from the scheduled time (NOW), not completion time
|
||||
expect(row?.nextScheduledAt).toBe(NOW + 60_000);
|
||||
expect(row?.lockedBy).toBeNull();
|
||||
expect(row?.lockedUntil).toBeNull();
|
||||
expect(row?.lastFinishedAt).toBe(NOW + 1_500);
|
||||
expect(row?.runCount).toBe(5);
|
||||
});
|
||||
|
||||
it("skips missed ticks when the worker has fallen behind, with jitter to avoid a herd", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
// scheduled NOW, interval 60s, but completion is 200s late → NOW+60s is already past
|
||||
const now = NOW + 200_000;
|
||||
await repo.recordSuccess("check:mon-1", NOW, 60_000, now);
|
||||
|
||||
const row = await readRow("check:mon-1");
|
||||
// no catch-up burst: rebased to one interval from now, plus up to one interval of jitter
|
||||
// so behind-jobs don't re-synchronize. Never earlier than now + interval (no over-checking).
|
||||
expect(row?.nextScheduledAt).toBeGreaterThanOrEqual(now + 60_000);
|
||||
expect(row?.nextScheduledAt).toBeLessThan(now + 120_000);
|
||||
});
|
||||
|
||||
it("returns false for an unknown id", async () => {
|
||||
expect(await repo.recordSuccess("missing", NOW, 60_000, NOW)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not release a lock owned by another worker (fenced no-op)", async () => {
|
||||
// A worker whose lease expired and was reclaimed must not clobber the new owner's lock.
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7 });
|
||||
|
||||
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000);
|
||||
|
||||
expect(ok).toBe(false);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).toBe("other-worker"); // lock untouched
|
||||
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
expect(row?.runCount).toBe(7); // not double-counted
|
||||
});
|
||||
});
|
||||
|
||||
// ── renewLocks (lease renewal) ───────────────────────────────────────────────
|
||||
describe("renewLocks", () => {
|
||||
it("extends the lease on a lock this worker holds", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000);
|
||||
|
||||
expect(renewed).toBe(1);
|
||||
expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS);
|
||||
});
|
||||
|
||||
it("does not renew a lock held by another worker (fenced)", async () => {
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
const renewed = await repo.renewLocks(["check:mon-1"], NOW + 1_000);
|
||||
|
||||
expect(renewed).toBe(0);
|
||||
expect((await readRow("check:mon-1"))?.lockedUntil).toBe(NOW + LOCK_MS); // unchanged
|
||||
});
|
||||
|
||||
it("renews only the locks this worker holds out of a mixed batch", async () => {
|
||||
await seedJob({ _id: "check:mine", refId: "mine", lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
await seedJob({ _id: "check:theirs", refId: "theirs", lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS });
|
||||
|
||||
const renewed = await repo.renewLocks(["check:mine", "check:theirs"], NOW + 1_000);
|
||||
|
||||
expect(renewed).toBe(1);
|
||||
expect((await readRow("check:mine"))?.lockedUntil).toBe(NOW + 1_000 + LOCK_MS);
|
||||
expect((await readRow("check:theirs"))?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
});
|
||||
|
||||
it("returns 0 for an empty id list without touching the db", async () => {
|
||||
expect(await repo.renewLocks([], NOW)).toBe(0);
|
||||
});
|
||||
|
||||
// Regression: a renewed lock keeps a slow-but-alive job from being reclaimed once its
|
||||
// original lease would have expired — preventing the duplicate-execution path.
|
||||
it("keeps a job from being reclaimed after the original lease window passes", async () => {
|
||||
await seedJob();
|
||||
await repo.claimDue("check", NOW); // this worker now holds the lease until NOW + LOCK_MS
|
||||
|
||||
// just before expiry, the still-running job renews its own lock
|
||||
const renewAt = NOW + LOCK_MS - 1_000;
|
||||
expect(await repo.renewLocks(["check:mon-1"], renewAt)).toBe(1);
|
||||
|
||||
// a second worker scans PAST the original lease end — without renewal it would reclaim
|
||||
const other = new MongoJobsRepository("competing-worker");
|
||||
expect(await other.claimDue("check", NOW + LOCK_MS + 1)).toBeNull();
|
||||
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lockedBy).toBe(ownedBy(repo)); // still held by the original worker
|
||||
expect(row?.lockedUntil).toBe(renewAt + LOCK_MS);
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordFailure (backoff) ──────────────────────────────────────────────────
|
||||
describe("recordFailure", () => {
|
||||
it("backs off, records the reason, bumps failCount, releases the lease", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, failCount: 2 });
|
||||
|
||||
const ok = await repo.recordFailure("check:mon-1", new Error("connection refused"), NOW);
|
||||
|
||||
expect(ok).toBe(true);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW + BACKOFF_MS);
|
||||
expect(row?.lastFailReason).toBe("connection refused");
|
||||
expect(row?.failCount).toBe(3);
|
||||
expect(row?.lockedBy).toBeNull();
|
||||
expect(row?.lockedUntil).toBeNull();
|
||||
});
|
||||
|
||||
it("stringifies a non-Error reason", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS });
|
||||
await repo.recordFailure("check:mon-1", "weird non-error", NOW);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.lastFailReason).toBe("weird non-error");
|
||||
});
|
||||
|
||||
it("returns false for an unknown id", async () => {
|
||||
expect(await repo.recordFailure("missing", new Error("x"), NOW)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── upsertEvaluate (handoff + $min coalescing) ───────────────────────────────
|
||||
describe("upsertEvaluate", () => {
|
||||
it("creates a single evaluate row keyed by monitor", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.type).toBe("evaluate");
|
||||
expect(row?.refId).toBe("mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW);
|
||||
expect(row?.intervalMs).toBeNull();
|
||||
expect(row?.isActive).toBe(true);
|
||||
});
|
||||
|
||||
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
||||
|
||||
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].nextScheduledAt).toBe(NOW); // earliest pending wins
|
||||
});
|
||||
|
||||
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW);
|
||||
});
|
||||
|
||||
it("concurrent upserts for one monitor produce exactly one row", async () => {
|
||||
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
||||
await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW)));
|
||||
|
||||
const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" });
|
||||
expect(count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── monitor lifecycle CRUD ───────────────────────────────────────────────────
|
||||
describe("CRUD", () => {
|
||||
it("upsertJob inserts, then refreshes config without clobbering scheduling state", async () => {
|
||||
const job: Job = {
|
||||
id: "check:mon-1",
|
||||
type: "check",
|
||||
refId: "mon-1",
|
||||
isActive: true,
|
||||
nextScheduledAt: NOW,
|
||||
intervalMs: 60_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
};
|
||||
await repo.upsertJob(job);
|
||||
|
||||
// simulate a running job that has advanced + holds a lease
|
||||
await JobModel.updateOne({ _id: "check:mon-1" }, { $set: { nextScheduledAt: NOW + 5_000, lockedBy: "live", runCount: 7 } });
|
||||
|
||||
// a reconcile sweep re-upserts the same id with new config
|
||||
await repo.upsertJob({ ...job, intervalMs: 30_000, isActive: false });
|
||||
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.intervalMs).toBe(30_000); // config refreshed ($set)
|
||||
expect(row?.isActive).toBe(false);
|
||||
expect(row?.nextScheduledAt).toBe(NOW + 5_000); // scheduling untouched ($setOnInsert)
|
||||
expect(row?.lockedBy).toBe("live");
|
||||
expect(row?.runCount).toBe(7);
|
||||
});
|
||||
|
||||
it("upsertCleanupJob resets nextScheduledAt on an existing row so it is due again on restart", async () => {
|
||||
const job: Job = {
|
||||
id: "cleanup-orphaned",
|
||||
type: "cleanup-orphaned",
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: NOW,
|
||||
intervalMs: 86_400_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
};
|
||||
await repo.upsertCleanupJob(job);
|
||||
|
||||
// row has already advanced a day into the future after a previous run
|
||||
await JobModel.updateOne({ _id: "cleanup-orphaned" }, { $set: { nextScheduledAt: NOW + 86_400_000, runCount: 3 } });
|
||||
|
||||
// a restart reconcile re-upserts and pulls it due now
|
||||
await repo.upsertCleanupJob({ ...job, nextScheduledAt: NOW + 1_000 });
|
||||
|
||||
const row = await readRow("cleanup-orphaned");
|
||||
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // schedule reset ($set)
|
||||
expect(row?.runCount).toBe(3); // run history untouched ($setOnInsert)
|
||||
});
|
||||
|
||||
it("setActiveById flips check and geo rows together", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
|
||||
await repo.setActiveById("mon-1", false);
|
||||
|
||||
const rows = await JobModel.find({ refId: "mon-1" }).lean<JobDocument[]>();
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(rows.every((r) => r.isActive === false)).toBe(true);
|
||||
});
|
||||
|
||||
it("updateScheduleById changes the interval on a single typed row", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check", intervalMs: 60_000 });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check", intervalMs: 60_000 });
|
||||
|
||||
await repo.updateScheduleById("mon-1", "check", 15_000);
|
||||
|
||||
expect((await readRow("check:mon-1"))?.intervalMs).toBe(15_000);
|
||||
expect((await readRow("geo:mon-1"))?.intervalMs).toBe(60_000); // untouched
|
||||
});
|
||||
|
||||
it("deleteById drops every row for the monitor", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
|
||||
await repo.deleteById("mon-1");
|
||||
|
||||
expect(await JobModel.countDocuments({ refId: "mon-1" })).toBe(0);
|
||||
});
|
||||
|
||||
it("deleteByMonitorIdsNotIn drops jobs for missing monitors but keeps valid and global rows", async () => {
|
||||
await seedJob({ _id: "check:keep", refId: "keep", type: "check" });
|
||||
await seedJob({ _id: "geo:keep", refId: "keep", type: "geo-check" });
|
||||
await seedJob({ _id: "check:gone", refId: "gone", type: "check" });
|
||||
await seedJob({ _id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" });
|
||||
await seedJob({ _id: "cleanup-retention", refId: null, type: "cleanup-retention" });
|
||||
|
||||
const deleted = await repo.deleteByMonitorIdsNotIn(["keep"]);
|
||||
|
||||
expect(deleted).toBe(1); // only check:gone
|
||||
const remaining = (await JobModel.find().lean<JobDocument[]>()).map((d) => d._id).sort();
|
||||
expect(remaining).toEqual(["check:keep", "cleanup-orphaned", "cleanup-retention", "geo:keep"]);
|
||||
});
|
||||
|
||||
it("findById returns all of a monitor's rows", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
|
||||
const jobs = await repo.findById("mon-1");
|
||||
expect(jobs.map((j) => j.id).sort()).toEqual(["check:mon-1", "geo:mon-1"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── observability ────────────────────────────────────────────────────────────
|
||||
describe("observability", () => {
|
||||
it("findPage returns a page plus the total count", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a", nextScheduledAt: NOW + 1 });
|
||||
await seedJob({ _id: "check:b", refId: "b", nextScheduledAt: NOW + 2 });
|
||||
await seedJob({ _id: "check:c", refId: "c", nextScheduledAt: NOW + 3 });
|
||||
|
||||
const { jobs, count } = await repo.findPage({ page: 0, rowsPerPage: 2 });
|
||||
expect(count).toBe(3);
|
||||
expect(jobs).toHaveLength(2);
|
||||
expect(jobs.map((j) => j.id)).toEqual(["check:a", "check:b"]); // sorted by nextScheduledAt
|
||||
});
|
||||
|
||||
it("findPage returns everything when rowsPerPage is 0", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a" });
|
||||
await seedJob({ _id: "check:b", refId: "b" });
|
||||
|
||||
const { jobs, count } = await repo.findPage({});
|
||||
expect(count).toBe(2);
|
||||
expect(jobs).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("findAll returns every row", async () => {
|
||||
await seedJob({ _id: "check:a", refId: "a" });
|
||||
await seedJob({ _id: "geo:b", refId: "b", type: "geo-check" });
|
||||
|
||||
expect(await repo.findAll()).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckPipeline, GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
@@ -30,23 +32,23 @@ const createCheckPipeline = (overrides?: Record<string, any>) => {
|
||||
statusService: {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: { id: "m1", status: "up" }, statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
checkService: { buildCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) },
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
...overrides,
|
||||
};
|
||||
const pipeline = new CheckPipeline(
|
||||
const checkProducer = new CheckProducer(
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.maintenanceWindowsRepository as any,
|
||||
defaults.checkService as any,
|
||||
defaults.networkService as any,
|
||||
defaults.buffer as any,
|
||||
defaults.monitorStatusPolicy as any,
|
||||
defaults.statusService as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any);
|
||||
const pipeline = new CheckPipeline(checkProducer, checkEvaluator);
|
||||
return { pipeline, defaults };
|
||||
};
|
||||
|
||||
@@ -85,9 +87,9 @@ describe("CheckPipeline", () => {
|
||||
await expect(pipeline.run(makeMonitor())).rejects.toThrow("No network response");
|
||||
});
|
||||
|
||||
it("returns null and warns when buildCheck returns null", async () => {
|
||||
it("returns null and warns when toCheck returns null", async () => {
|
||||
const { pipeline, defaults } = createCheckPipeline({
|
||||
checkService: { buildCheck: jest.fn().mockReturnValue(null) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(null) },
|
||||
});
|
||||
|
||||
const result = await pipeline.run(makeMonitor());
|
||||
|
||||
@@ -4,6 +4,8 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
|
||||
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload, PageSpeedStatusPayload } from "../../../src/types/network.ts";
|
||||
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -53,12 +55,12 @@ describe("CheckService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildCheck ───────────────────────────────────────────────────────────
|
||||
// ── toCheck ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("buildCheck", () => {
|
||||
describe("toCheck", () => {
|
||||
it("builds a basic HTTP check with correct fields", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse());
|
||||
const check = service.toCheck(makeStatusResponse());
|
||||
|
||||
expect(check).toBeDefined();
|
||||
expect(check!.metadata).toEqual({ monitorId: "mon-1", teamId: "team-1", type: "http" });
|
||||
@@ -72,14 +74,14 @@ describe("CheckService", () => {
|
||||
|
||||
it("defaults responseTime to 0 when falsy", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ responseTime: undefined }));
|
||||
const check = service.toCheck(makeStatusResponse({ responseTime: undefined }));
|
||||
expect(check!.responseTime).toBe(0);
|
||||
});
|
||||
|
||||
it("includes timings when provided", () => {
|
||||
const timings = { start: 0, socket: 10, lookup: 20 };
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ timings } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ timings } as any));
|
||||
expect(check!.timings).toBe(timings);
|
||||
});
|
||||
|
||||
@@ -106,7 +108,7 @@ describe("CheckService", () => {
|
||||
} as any;
|
||||
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.accessibility).toBe(90);
|
||||
expect(check!.bestPractices).toBe(80);
|
||||
@@ -119,7 +121,7 @@ describe("CheckService", () => {
|
||||
|
||||
it("returns undefined when pagespeed payload is missing", () => {
|
||||
const { service, logger } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: undefined } as any));
|
||||
|
||||
expect(check).toBeUndefined();
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build check" }));
|
||||
@@ -127,7 +129,7 @@ describe("CheckService", () => {
|
||||
|
||||
it("defaults categories and audits to empty when lighthouseResult is missing", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload: {} } as any));
|
||||
|
||||
expect(check!.accessibility).toBe(0);
|
||||
expect(check!.bestPractices).toBe(0);
|
||||
@@ -146,7 +148,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.cls!.score).toBe("informative");
|
||||
});
|
||||
@@ -161,7 +163,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.cls!.score).toBeNull();
|
||||
});
|
||||
@@ -176,7 +178,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.si!.numericValue).toBeUndefined();
|
||||
});
|
||||
@@ -192,7 +194,7 @@ describe("CheckService", () => {
|
||||
},
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "pagespeed", payload } as any));
|
||||
|
||||
expect(check!.audits!.cls).toBeUndefined();
|
||||
expect(check!.audits!.si).toBeUndefined();
|
||||
@@ -214,7 +216,7 @@ describe("CheckService", () => {
|
||||
capture: { screenshot: "base64data" },
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
|
||||
expect(check!.cpu).toEqual({ usage_percent: 0.5 });
|
||||
expect(check!.memory).toEqual({ usage_percent: 0.6 });
|
||||
@@ -230,7 +232,7 @@ describe("CheckService", () => {
|
||||
errors: [{ message: "timeout" }],
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
|
||||
expect(check!.errors).toEqual([{ message: "timeout" }]);
|
||||
});
|
||||
@@ -241,14 +243,14 @@ describe("CheckService", () => {
|
||||
errors: { errors: [{ message: "nested error" }] },
|
||||
} as any;
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload } as any));
|
||||
|
||||
expect(check!.errors).toEqual([{ message: "nested error" }]);
|
||||
});
|
||||
|
||||
it("handles undefined payload gracefully", () => {
|
||||
const { service } = createService();
|
||||
const check = service.buildCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any));
|
||||
const check = service.toCheck(makeStatusResponse({ type: "hardware", payload: undefined } as any));
|
||||
|
||||
expect(check).toBeDefined();
|
||||
expect(check!.cpu).toBeUndefined();
|
||||
@@ -259,6 +261,84 @@ describe("CheckService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── toStatusResponse round-trip (Phase 2 §5 guard) ─────────────────────────
|
||||
// The DB-queue split persists a Check, then the evaluator rebuilds the status response from it.
|
||||
// toStatusResponse is only correct because every field a downstream consumer reads survives
|
||||
// toCheck → toStatusResponse. These tests pin that: if a future change drops a consumed field
|
||||
// from the round-trip, they fail instead of silently regressing the evaluator/reactors.
|
||||
|
||||
describe("toStatusResponse round-trip", () => {
|
||||
const makeHardwareStatus = (overrides?: Partial<MonitorStatusResponse>): MonitorStatusResponse =>
|
||||
makeStatusResponse({
|
||||
type: "hardware",
|
||||
status: true,
|
||||
code: 200,
|
||||
message: "OK",
|
||||
responseTime: 42,
|
||||
payload: {
|
||||
data: {
|
||||
cpu: { usage_percent: 0.92 },
|
||||
memory: { usage_percent: 0.81 },
|
||||
disk: [{ device: "/dev/sda", usage_percent: 0.75 }],
|
||||
host: { os: "linux" },
|
||||
net: [{ name: "eth0" }],
|
||||
},
|
||||
} as HardwareStatusPayload,
|
||||
...overrides,
|
||||
} as any);
|
||||
|
||||
it("uptime: preserves every consumed scalar field", () => {
|
||||
const { service } = createService();
|
||||
const status = makeStatusResponse({ type: "http", status: false, code: 503, message: "Service Unavailable", responseTime: 87 });
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
expect(rebuilt.monitorId).toBe(status.monitorId);
|
||||
expect(rebuilt.teamId).toBe(status.teamId);
|
||||
expect(rebuilt.type).toBe(status.type);
|
||||
expect(rebuilt.status).toBe(status.status);
|
||||
expect(rebuilt.code).toBe(status.code);
|
||||
expect(rebuilt.message).toBe(status.message);
|
||||
expect(rebuilt.responseTime).toBe(status.responseTime);
|
||||
});
|
||||
|
||||
it("hardware: preserves payload.data metrics verbatim, same units (usage_percent stays a 0–1 decimal)", () => {
|
||||
const { service } = createService();
|
||||
const status = makeHardwareStatus();
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
const original = status.payload as HardwareStatusPayload;
|
||||
const roundtripped = rebuilt.payload as HardwareStatusPayload;
|
||||
expect(roundtripped.data).toEqual(original.data);
|
||||
expect(roundtripped.data.cpu!.usage_percent).toBe(0.92); // not rescaled to 92
|
||||
});
|
||||
|
||||
it("hardware: a real consumer (extractThresholdBreaches) behaves identically on raw vs round-tripped status", () => {
|
||||
const { service } = createService();
|
||||
const builder = new NotificationMessageBuilder();
|
||||
const monitor = {
|
||||
id: "mon-1",
|
||||
type: "hardware",
|
||||
cpuAlertThreshold: 90, // 0.92 → 92% breaches
|
||||
memoryAlertThreshold: 90, // 0.81 → 81% does not
|
||||
diskAlertThreshold: 90, // 0.75 → 75% does not
|
||||
tempAlertThreshold: 100,
|
||||
} as Monitor;
|
||||
const status = makeHardwareStatus();
|
||||
|
||||
const rebuilt = service.toStatusResponse(service.toCheck(status)!);
|
||||
|
||||
const fromRaw = builder.extractThresholdBreaches(monitor, status as MonitorStatusResponse<HardwareStatusPayload>);
|
||||
const fromRebuilt = builder.extractThresholdBreaches(monitor, rebuilt as MonitorStatusResponse<HardwareStatusPayload>);
|
||||
|
||||
// sanity: the fixture actually triggers a breach, so the equality below is not vacuous
|
||||
expect(fromRaw).toHaveLength(1);
|
||||
expect(fromRaw[0].metric).toBe("cpu");
|
||||
expect(fromRebuilt).toEqual(fromRaw);
|
||||
});
|
||||
});
|
||||
|
||||
// ── createChecks ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("createChecks", () => {
|
||||
|
||||
@@ -0,0 +1,536 @@
|
||||
import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals";
|
||||
import { DBQueueWorker } from "../../../src/worker/worker.db-queue.ts";
|
||||
import { LOCK_MS } from "../../../src/domain/jobs/job.type.ts";
|
||||
import type { Job } from "../../../src/domain/jobs/job.type.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
import type { QueueMode } from "../../../src/domain/app-settings/app-settings.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Notes ──────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
|
||||
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
|
||||
// renews their locks, and registers itself in the worker heartbeat. The repository
|
||||
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
|
||||
// test only the orchestration around it, with the repository fully mocked.
|
||||
//
|
||||
// Fake timers are mandatory: init() starts self-re-arming setTimeout loops and a
|
||||
// heartbeat setInterval, so without them the loops would leak across tests. The
|
||||
// loop re-arms at POLL_MS (250ms) and the lock renews at LOCK_MS / 3 (20s).
|
||||
|
||||
const POLL_MS = 250;
|
||||
const LOCK_RENEW_MS = LOCK_MS / 3;
|
||||
|
||||
const deferred = <T>() => {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
isActive: true,
|
||||
geoCheckEnabled: false,
|
||||
geoCheckInterval: 300000,
|
||||
lastEvaluatedAt: 0,
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
id: "check:m1",
|
||||
type: "check",
|
||||
refId: "m1",
|
||||
isActive: true,
|
||||
nextScheduledAt: 1000,
|
||||
intervalMs: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
|
||||
const jobsRepository = {
|
||||
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
|
||||
upsertJob: jest.fn<any>().mockResolvedValue(true),
|
||||
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
|
||||
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
|
||||
recordSuccess: jest.fn<any>().mockResolvedValue(true),
|
||||
recordFailure: jest.fn<any>().mockResolvedValue(true),
|
||||
recordOneShot: jest.fn<any>().mockResolvedValue(true),
|
||||
renewLocks: jest.fn<any>().mockResolvedValue(1),
|
||||
deleteById: jest.fn<any>().mockResolvedValue(true),
|
||||
deleteByIdAndType: jest.fn<any>().mockResolvedValue(true),
|
||||
setActiveById: jest.fn<any>().mockResolvedValue(true),
|
||||
updateScheduleById: jest.fn<any>().mockResolvedValue(true),
|
||||
findAll: jest.fn<any>().mockResolvedValue([]),
|
||||
findPage: jest.fn<any>().mockResolvedValue({ jobs: [], count: 0 }),
|
||||
};
|
||||
const monitorsRepository = {
|
||||
findByIds: jest.fn<any>().mockResolvedValue([makeMonitor()]),
|
||||
findAll: jest.fn<any>().mockResolvedValue([]),
|
||||
updateById: jest.fn<any>().mockResolvedValue({}),
|
||||
};
|
||||
const checksRepository = {
|
||||
findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([]),
|
||||
};
|
||||
const checkService = {
|
||||
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
|
||||
toLastEvaluatedAt: jest.fn<any>().mockReturnValue(12345),
|
||||
};
|
||||
const checkProducer = {
|
||||
produce: jest.fn<any>().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }),
|
||||
};
|
||||
const checkEvaluator = {
|
||||
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), decision: {} }),
|
||||
};
|
||||
const geoCheckPipeline = { run: jest.fn<any>().mockResolvedValue(null) };
|
||||
const dispatcher = { dispatch: jest.fn<any>().mockResolvedValue(undefined) };
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const queueWorkersRepository = {
|
||||
upsert: jest.fn<any>().mockResolvedValue(undefined),
|
||||
deleteById: jest.fn<any>().mockResolvedValue(undefined),
|
||||
findRecent: jest.fn<any>().mockResolvedValue([]),
|
||||
};
|
||||
const logger = createMockLogger();
|
||||
|
||||
const mocks = {
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
dispatcher,
|
||||
helper,
|
||||
queueWorkersRepository,
|
||||
logger,
|
||||
...overrides?.mocks,
|
||||
};
|
||||
|
||||
const worker = new DBQueueWorker(
|
||||
mocks.logger as any,
|
||||
mocks.jobsRepository as any,
|
||||
mocks.monitorsRepository as any,
|
||||
mocks.checksRepository as any,
|
||||
mocks.checkService as any,
|
||||
mocks.checkProducer as any,
|
||||
mocks.checkEvaluator as any,
|
||||
mocks.geoCheckPipeline as any,
|
||||
mocks.dispatcher as any,
|
||||
mocks.helper as any,
|
||||
mocks.queueWorkersRepository as any,
|
||||
overrides?.queueMode ?? "worker",
|
||||
overrides?.queuePrimaryProcesses ?? true,
|
||||
"worker-1"
|
||||
);
|
||||
return { worker, mocks };
|
||||
};
|
||||
|
||||
describe("DBQueueWorker", () => {
|
||||
let active: DBQueueWorker | null = null;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (active) await active.shutdown();
|
||||
active = null;
|
||||
jest.clearAllTimers();
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
const start = async (opts?: Parameters<typeof createWorker>[0]) => {
|
||||
const ctx = createWorker(opts);
|
||||
active = ctx.worker;
|
||||
await ctx.worker.init();
|
||||
await jest.advanceTimersByTimeAsync(1); // flush the immediate tick + any job it launched
|
||||
return ctx;
|
||||
};
|
||||
|
||||
// ── init() wiring ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("init", () => {
|
||||
it("registers the worker in the heartbeat registry on startup", async () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker");
|
||||
});
|
||||
|
||||
it("primary mode reconciles: seeds a check row per monitor plus the two cleanup rows", async () => {
|
||||
const { mocks } = await start({
|
||||
queueMode: "primary",
|
||||
queuePrimaryProcesses: false, // isolate reconcile from the polling loops
|
||||
mocks: {
|
||||
monitorsRepository: { findByIds: jest.fn<any>(), findAll: jest.fn<any>().mockResolvedValue([makeMonitor()]), updateById: jest.fn<any>() },
|
||||
},
|
||||
});
|
||||
|
||||
const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(seededTypes).toContain("check");
|
||||
const cleanupTypes = mocks.jobsRepository.upsertCleanupJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(cleanupTypes).toContain("cleanup-orphaned");
|
||||
expect(cleanupTypes).toContain("cleanup-retention");
|
||||
});
|
||||
|
||||
it("primary mode also seeds a geo-check row for geo-enabled monitors", async () => {
|
||||
const geoMonitor = makeMonitor({ type: "http", geoCheckEnabled: true });
|
||||
const { mocks } = await start({
|
||||
queueMode: "primary",
|
||||
queuePrimaryProcesses: false,
|
||||
mocks: {
|
||||
monitorsRepository: { findByIds: jest.fn<any>(), findAll: jest.fn<any>().mockResolvedValue([geoMonitor]), updateById: jest.fn<any>() },
|
||||
},
|
||||
});
|
||||
|
||||
const seededTypes = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(seededTypes).toContain("geo-check");
|
||||
});
|
||||
|
||||
it("worker mode does not reconcile (never seeds jobs)", async () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
expect(mocks.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("starts a polling loop per job type, each claiming up to its concurrency cap", async () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
|
||||
const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
|
||||
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
|
||||
});
|
||||
|
||||
it("does not start polling loops when primary and queuePrimaryProcesses is false", async () => {
|
||||
const { mocks } = await start({ queueMode: "primary", queuePrimaryProcesses: false });
|
||||
expect(mocks.jobsRepository.claimDueBatch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("create() constructs the worker and runs init", async () => {
|
||||
const { mocks } = createWorker(); // reuse the mock set; the un-init'd instance is discarded
|
||||
const worker = await DBQueueWorker.create(
|
||||
mocks.logger as any,
|
||||
mocks.jobsRepository as any,
|
||||
mocks.monitorsRepository as any,
|
||||
mocks.checksRepository as any,
|
||||
mocks.checkService as any,
|
||||
mocks.checkProducer as any,
|
||||
mocks.checkEvaluator as any,
|
||||
mocks.geoCheckPipeline as any,
|
||||
mocks.dispatcher as any,
|
||||
mocks.helper as any,
|
||||
mocks.queueWorkersRepository as any,
|
||||
"worker",
|
||||
true,
|
||||
"worker-1"
|
||||
);
|
||||
active = worker;
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledWith("worker-1", "worker");
|
||||
});
|
||||
});
|
||||
|
||||
// ── runJob: the per-type execution paths ────────────────────────────────────
|
||||
|
||||
describe("runJob", () => {
|
||||
// Hand the "check" loop exactly one job, then nothing, so the assertion target runs once.
|
||||
const claimOnce = (job: Job) => {
|
||||
let handed = false;
|
||||
return jest.fn<any>(async (type: string) => {
|
||||
if (type === job.type && !handed) {
|
||||
handed = true;
|
||||
return [job];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
};
|
||||
|
||||
it("runs a check job and reschedules it on success", async () => {
|
||||
const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.monitorsRepository.findByIds).toHaveBeenCalledWith(["m1"]);
|
||||
expect(mocks.checkProducer.produce).toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("m1", expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("records a failure (and not a success) when the job handler throws", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: failingProducer },
|
||||
});
|
||||
|
||||
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluate job dispatches each unevaluated check and advances lastEvaluatedAt", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||
});
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
|
||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
|
||||
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
|
||||
});
|
||||
|
||||
it("cleanup-orphaned job runs the helper's cleanup function", async () => {
|
||||
const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 });
|
||||
const cleanupFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(cleanupFn),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(cleanupFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cleanup-retention job runs the helper's retention function", async () => {
|
||||
const job = makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 });
|
||||
const retentionFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(retentionFn),
|
||||
};
|
||||
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(retentionFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
|
||||
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.geoCheckPipeline.run).toHaveBeenCalled();
|
||||
expect(mocks.dispatcher.dispatch).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
});
|
||||
|
||||
it("renews the lock while a slow job is still running", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer },
|
||||
});
|
||||
|
||||
// Job is parked on produce(); push past one renewal interval.
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledWith([job.id], expect.any(Number));
|
||||
|
||||
// Let it finish; success is still recorded.
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops renewing once the lock is lost to another worker", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const jobsRepository = {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
renewLocks: jest.fn<any>().mockResolvedValue(0),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository, checkProducer: slowProducer } });
|
||||
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); // lease lost → renewal timer stops
|
||||
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Lost lock") }));
|
||||
|
||||
// A second interval passes without another renewal attempt.
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1);
|
||||
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── monitor lifecycle delegation ────────────────────────────────────────────
|
||||
|
||||
describe("job lifecycle methods", () => {
|
||||
it("addJob seeds a check row, plus a geo row for geo-enabled monitors", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.addJob("m1", makeMonitor({ geoCheckEnabled: true }));
|
||||
|
||||
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(types).toContain("check");
|
||||
expect(types).toContain("geo-check");
|
||||
});
|
||||
|
||||
it("addJob seeds only a check row for non-geo monitors", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.addJob("m1", makeMonitor({ geoCheckEnabled: false }));
|
||||
|
||||
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(types).toEqual(["check"]);
|
||||
});
|
||||
|
||||
it("deleteJob removes every row for the monitor", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.deleteJob(makeMonitor());
|
||||
expect(mocks.jobsRepository.deleteById).toHaveBeenCalledWith("m1");
|
||||
});
|
||||
|
||||
it("pauseJob and resumeJob flip the active flag", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.pauseJob(makeMonitor());
|
||||
expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", false);
|
||||
await worker.resumeJob(makeMonitor());
|
||||
expect(mocks.jobsRepository.setActiveById).toHaveBeenCalledWith("m1", true);
|
||||
});
|
||||
|
||||
it("updateJob reschedules the check row and drops the geo row when geo is disabled", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.updateJob(makeMonitor({ geoCheckEnabled: false, interval: 30000 }));
|
||||
expect(mocks.jobsRepository.updateScheduleById).toHaveBeenCalledWith("m1", "check", 30000);
|
||||
expect(mocks.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith("m1", "geo-check");
|
||||
});
|
||||
|
||||
it("updateJob upserts the geo row when geo is enabled", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
await worker.updateJob(makeMonitor({ geoCheckEnabled: true }));
|
||||
const types = mocks.jobsRepository.upsertJob.mock.calls.map((c: any[]) => c[0].type);
|
||||
expect(types).toContain("geo-check");
|
||||
expect(mocks.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── shutdown ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("shutdown", () => {
|
||||
it("deregisters the worker and stops the polling loops", async () => {
|
||||
const { worker, mocks } = createWorker({ queueMode: "worker" });
|
||||
await worker.init();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
const claimsAfterInit = mocks.jobsRepository.claimDueBatch.mock.calls.length;
|
||||
|
||||
await worker.shutdown();
|
||||
expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1");
|
||||
|
||||
// Loops must not re-arm after shutdown.
|
||||
await jest.advanceTimersByTimeAsync(POLL_MS * 4);
|
||||
expect(mocks.jobsRepository.claimDueBatch.mock.calls.length).toBe(claimsAfterInit);
|
||||
// already shut down; keep afterEach from double-shutting-down
|
||||
active = null;
|
||||
});
|
||||
});
|
||||
|
||||
// ── flushQueues ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("flushQueues", () => {
|
||||
it("restarts the worker and reports success", async () => {
|
||||
const { worker, mocks } = createWorker({ queueMode: "worker" });
|
||||
active = worker;
|
||||
await worker.init();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
const result = await worker.flushQueues();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(result).toEqual({ success: true });
|
||||
expect(mocks.queueWorkersRepository.deleteById).toHaveBeenCalledWith("worker-1"); // shutdown leg
|
||||
expect(mocks.queueWorkersRepository.upsert).toHaveBeenCalledTimes(2); // init + re-init
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics", () => {
|
||||
it("aggregates run/fail counts, active leases, and failing jobs across all rows", async () => {
|
||||
const now = Date.now();
|
||||
const rows = [
|
||||
makeJob({ id: "a", runCount: 5, failCount: 0, lockedBy: "w", lockedUntil: now + 10000 }), // active lease
|
||||
makeJob({ id: "b", runCount: 2, failCount: 3, lastFailReason: "boom", lockedBy: null, lockedUntil: null }), // failing
|
||||
makeJob({ id: "c", runCount: 1, failCount: 0, lockedBy: "w", lockedUntil: now - 10000 }), // expired lease, not active
|
||||
];
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
mocks.jobsRepository.findAll.mockResolvedValue(rows);
|
||||
mocks.queueWorkersRepository.findRecent.mockResolvedValue([{ workerId: "w" }]);
|
||||
|
||||
const metrics = await worker.getMetrics();
|
||||
|
||||
expect(metrics.jobs).toBe(3);
|
||||
expect(metrics.totalRuns).toBe(8);
|
||||
expect(metrics.totalFailures).toBe(3);
|
||||
expect(metrics.activeJobs).toBe(1);
|
||||
expect(metrics.failingJobs).toBe(1);
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ failCount: 3, failReason: "boom" }));
|
||||
expect(metrics.workers).toEqual([{ workerId: "w" }]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getJobs ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getJobs", () => {
|
||||
it("maps rows to summaries and passes the total count through", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
mocks.jobsRepository.findPage.mockResolvedValue({
|
||||
jobs: [makeJob({ id: "check:m1", refId: "m1", type: "check", intervalMs: 60000 })],
|
||||
count: 1,
|
||||
});
|
||||
|
||||
const page = await worker.getJobs({ page: 0, rowsPerPage: 10 });
|
||||
|
||||
expect(page.count).toBe(1);
|
||||
expect(page.jobs).toHaveLength(1);
|
||||
expect(page.jobs[0]).toEqual(expect.objectContaining({ monitorId: "m1", monitorType: "check", monitorInterval: 60000, repeat: 60000 }));
|
||||
});
|
||||
|
||||
it("falls back to the job id for monitorId when refId is null (global jobs)", async () => {
|
||||
const { worker, mocks } = createWorker();
|
||||
active = worker;
|
||||
mocks.jobsRepository.findPage.mockResolvedValue({
|
||||
jobs: [makeJob({ id: "cleanup-orphaned", refId: null, type: "cleanup-orphaned" })],
|
||||
count: 1,
|
||||
});
|
||||
|
||||
const page = await worker.getJobs({ page: 0, rowsPerPage: 10 });
|
||||
expect(page.jobs[0].monitorId).toBe("cleanup-orphaned");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,791 +0,0 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
|
||||
// ── Mock Scheduler / MongoStore ──────────────────────────────────────────────
|
||||
|
||||
const createScheduler = () => {
|
||||
const listeners: Record<string, Function[]> = {};
|
||||
return {
|
||||
workerId: "host-1:1234:abcd",
|
||||
on: jest.fn((event: string, cb: Function) => {
|
||||
listeners[event] = listeners[event] || [];
|
||||
listeners[event].push(cb);
|
||||
}),
|
||||
off: jest.fn((event: string, cb: Function) => {
|
||||
listeners[event] = (listeners[event] || []).filter((fn) => fn !== cb);
|
||||
}),
|
||||
emit: (event: string, ...args: unknown[]) => {
|
||||
(listeners[event] || []).forEach((cb) => cb(...args));
|
||||
},
|
||||
start: jest.fn().mockResolvedValue(true),
|
||||
stop: jest.fn().mockResolvedValue(true),
|
||||
addTemplate: jest.fn(),
|
||||
addJob: jest.fn(),
|
||||
addJobs: jest.fn().mockResolvedValue({ inserted: 0, upserted: 0, modified: 0, failures: [] }),
|
||||
removeJob: jest.fn(),
|
||||
getJob: jest.fn().mockResolvedValue(null),
|
||||
getJobs: jest.fn().mockResolvedValue([]),
|
||||
countJobs: jest.fn().mockResolvedValue(0),
|
||||
getStats: jest.fn().mockResolvedValue({ jobs: 0, activeJobs: 0, failingJobs: 0, totalRuns: 0, totalFailures: 0, jobsWithFailures: [] }),
|
||||
pauseJob: jest.fn().mockResolvedValue(true),
|
||||
resumeJob: jest.fn().mockResolvedValue(true),
|
||||
updateJob: jest.fn(),
|
||||
flushJobs: jest.fn().mockResolvedValue(true),
|
||||
};
|
||||
};
|
||||
|
||||
const mockSchedulerInstance = createScheduler();
|
||||
const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance);
|
||||
const MockMongoStore = jest.fn();
|
||||
|
||||
jest.unstable_mockModule("less-simple-scheduler", () => ({
|
||||
Scheduler: MockScheduler,
|
||||
MongoStore: MockMongoStore,
|
||||
}));
|
||||
|
||||
const { LessSimpleWorker: LessSimpleQueue } = await import("../../../src/worker/worker.less-simple.ts");
|
||||
|
||||
const createQueueHelper = () => ({
|
||||
getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
});
|
||||
|
||||
const createMonitorsRepo = () => ({
|
||||
findAll: jest.fn().mockResolvedValue([]),
|
||||
});
|
||||
|
||||
const createWorkersRepo = () => ({
|
||||
upsert: jest.fn().mockResolvedValue(undefined),
|
||||
findRecent: jest.fn().mockResolvedValue([]),
|
||||
deleteById: jest.fn().mockResolvedValue(undefined),
|
||||
});
|
||||
|
||||
const createQueue = (overrides?: Record<string, unknown>) => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const workersRepository = createWorkersRepo();
|
||||
const scheduler = createScheduler();
|
||||
|
||||
const defaults = { logger, helper, monitorsRepository, workersRepository, scheduler, queueMode: "primary", ...overrides };
|
||||
|
||||
const queue = new LessSimpleQueue(
|
||||
defaults.logger as any,
|
||||
defaults.helper as any,
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.workersRepository as any,
|
||||
defaults.scheduler as any,
|
||||
defaults.queueMode as any
|
||||
);
|
||||
return { queue, ...defaults };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "mon-1",
|
||||
teamId: "team-1",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
isActive: true,
|
||||
geoCheckEnabled: false,
|
||||
geoCheckInterval: 300000,
|
||||
geoCheckLocations: [],
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("LessSimpleQueue", () => {
|
||||
describe("serviceName", () => {
|
||||
it("returns JobQueue", () => {
|
||||
const { queue } = createQueue();
|
||||
expect(queue.serviceName).toBe("JobQueue");
|
||||
});
|
||||
});
|
||||
|
||||
// ── init (primary) ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("init - primary mode", () => {
|
||||
it("starts scheduler, registers all templates, seeds monitors and system jobs", async () => {
|
||||
const { queue, scheduler, monitorsRepository } = createQueue();
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.start).toHaveBeenCalled();
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
|
||||
expect(monitorsRepository.findAll).toHaveBeenCalled();
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
|
||||
});
|
||||
|
||||
it("registers existing monitors in a single bulk write", async () => {
|
||||
const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })];
|
||||
const { queue, scheduler, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors);
|
||||
|
||||
await queue.init();
|
||||
|
||||
expect(scheduler.addJobs).toHaveBeenCalledTimes(1);
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ id: "m1", template: "monitor-job", jitter: true }),
|
||||
expect.objectContaining({ id: "m2", template: "monitor-job", jitter: true }),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it("returns true and skips system jobs when findAll returns null", async () => {
|
||||
const { queue, monitorsRepository, scheduler } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(null);
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
});
|
||||
|
||||
it("returns false and logs error on failure", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") }));
|
||||
});
|
||||
|
||||
it("logs error with String(error) for non-Error exceptions", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error");
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── init (worker) ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("init - worker mode", () => {
|
||||
it("starts scheduler and registers all templates", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
|
||||
const result = await queue.init("worker");
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.start).toHaveBeenCalled();
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
|
||||
});
|
||||
|
||||
it("does NOT seed monitor jobs (no findAll, primary populates the queue)", async () => {
|
||||
const { queue, monitorsRepository } = createQueue();
|
||||
|
||||
await queue.init("worker");
|
||||
|
||||
expect(monitorsRepository.findAll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does NOT seed cleanup system jobs", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
|
||||
await queue.init("worker");
|
||||
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs the initialized worker with its workerId", async () => {
|
||||
const { queue, logger } = createQueue();
|
||||
|
||||
await queue.init("worker");
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Worker queue: host-1:1234:abcd initialized" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── registerListeners ──────────────────────────────────────────────────────
|
||||
|
||||
describe("registerListeners (via constructor)", () => {
|
||||
it("logs on scheduler:start", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:start", "host-1:1234:abcd");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:stop", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:stop", "host-1:1234:abcd");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:error", "w1", new Error("boom"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: boom" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with non-Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("scheduler:error", "w1", "string error");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 error: string error", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:locked", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:locked", "w1", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 locked" }));
|
||||
});
|
||||
|
||||
it("logs on job:abort", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:abort", "w1", { id: "m1" }, "timeout");
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 aborted: timeout" }));
|
||||
});
|
||||
|
||||
it("logs on job:attempt", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:attempt", "w1", { id: "m1" }, 2);
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 attempt 2" }));
|
||||
});
|
||||
|
||||
it("logs on job:complete", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:complete", "w1", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 completed successfully" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:exhausted", "w1", { id: "m1" }, new Error("gave up"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with non-Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:exhausted", "w1", { id: "m1" }, "gave up");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 exhausted all retries: gave up", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:fail", "w1", { id: "m1" }, new Error("oops"), 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops" }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with non-Error", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:fail", "w1", { id: "m1" }, "oops", 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 failed on attempt 1: oops", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:start", () => {
|
||||
const { scheduler, logger } = createQueue();
|
||||
scheduler.emit("job:start", "w1", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "w1 m1 started" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── addJob ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("addJob", () => {
|
||||
it("adds monitor job to scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor());
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true })])
|
||||
);
|
||||
});
|
||||
|
||||
it("adds geo check job when geoCheckEnabled and type supports it", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }));
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 })])
|
||||
);
|
||||
});
|
||||
|
||||
it("skips geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })]));
|
||||
});
|
||||
|
||||
it("skips geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.addJobs).toHaveBeenCalledWith(expect.not.arrayContaining([expect.objectContaining({ id: "mon-1-geo" })]));
|
||||
});
|
||||
});
|
||||
|
||||
// ── deleteJob ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("deleteJob", () => {
|
||||
it("removes monitor job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1");
|
||||
});
|
||||
|
||||
it("removes geo job if it exists", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not remove geo job if it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
});
|
||||
|
||||
// ── pauseJob ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("pauseJob", () => {
|
||||
it("pauses monitor and geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.pauseJob(makeMonitor());
|
||||
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("does not pause geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.pauseJob(makeMonitor());
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("throws when scheduler.pauseJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor");
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo pauseJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── resumeJob ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("resumeJob", () => {
|
||||
it("resumes monitor and geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.resumeJob(makeMonitor());
|
||||
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("does not resume geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.resumeJob(makeMonitor());
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("throws when scheduler.resumeJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor");
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo resumeJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── updateJob / syncGeoJob ─────────────────────────────────────────────────
|
||||
|
||||
describe("updateJob", () => {
|
||||
it("updates monitor job with new interval", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ interval: 120000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor }));
|
||||
});
|
||||
|
||||
it("removes existing geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("removes existing geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("updates existing geo job when geoCheckEnabled", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 }));
|
||||
});
|
||||
|
||||
it("creates new geo job when enabled but no existing job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── shutdown ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("shutdown", () => {
|
||||
it("stops the scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.shutdown();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics", () => {
|
||||
it("returns empty metrics when no jobs", async () => {
|
||||
const { queue } = createQueue();
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics).toEqual({ jobs: 0, activeJobs: 0, failingJobs: 0, jobsWithFailures: [], totalRuns: 0, totalFailures: 0, workers: [] });
|
||||
});
|
||||
|
||||
it("maps the store's aggregate counters through", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getStats as jest.Mock).mockResolvedValue({
|
||||
jobs: 2,
|
||||
activeJobs: 1,
|
||||
failingJobs: 1,
|
||||
totalRuns: 15,
|
||||
totalFailures: 2,
|
||||
jobsWithFailures: [],
|
||||
});
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics).toMatchObject({ jobs: 2, activeJobs: 1, failingJobs: 1, totalRuns: 15, totalFailures: 2 });
|
||||
});
|
||||
|
||||
it("derives monitor fields for failing monitor jobs from job data", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getStats as jest.Mock).mockResolvedValue({
|
||||
jobs: 1,
|
||||
activeJobs: 0,
|
||||
failingJobs: 1,
|
||||
totalRuns: 10,
|
||||
totalFailures: 2,
|
||||
jobsWithFailures: [{ id: "m1", data: { url: "http://a.com", type: "http" }, failedAt: 200, failCount: 2, failReason: "timeout" }],
|
||||
});
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures).toEqual([
|
||||
{ monitorId: "m1", monitorUrl: "http://a.com", monitorType: "http", failedAt: 200, failCount: 2, failReason: "timeout" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("leaves monitor fields null for non-monitor (cleanup) failures", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getStats as jest.Mock).mockResolvedValue({
|
||||
jobs: 1,
|
||||
activeJobs: 0,
|
||||
failingJobs: 1,
|
||||
totalRuns: 1,
|
||||
totalFailures: 1,
|
||||
jobsWithFailures: [{ id: "cleanup-orphaned", data: undefined, failedAt: 100, failCount: 1, failReason: "boom" }],
|
||||
});
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures[0]).toEqual({
|
||||
monitorId: "cleanup-orphaned",
|
||||
monitorUrl: null,
|
||||
monitorType: null,
|
||||
failedAt: 100,
|
||||
failCount: 1,
|
||||
failReason: "boom",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics: worker registry ──────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics - worker registry", () => {
|
||||
it("returns the alive workers from the registry", async () => {
|
||||
const { queue, workersRepository } = createQueue();
|
||||
const alive = [
|
||||
{ workerId: "host:1:aaa", mode: "primary", lastSeenAt: 1000 },
|
||||
{ workerId: "host:2:bbb", mode: "worker", lastSeenAt: 2000 },
|
||||
];
|
||||
(workersRepository.findRecent as jest.Mock).mockResolvedValue(alive);
|
||||
|
||||
const metrics = await queue.getMetrics();
|
||||
|
||||
expect(workersRepository.findRecent).toHaveBeenCalledWith(30000);
|
||||
expect(metrics.workers).toEqual(alive);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getJobs ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getJobs", () => {
|
||||
it("maps scheduler jobs to summaries", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{
|
||||
id: "m1",
|
||||
template: "monitor-job",
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
lastError: "timeout",
|
||||
lastStartedAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastFailedAt: 900,
|
||||
data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true },
|
||||
},
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]).toEqual({
|
||||
monitorId: "m1",
|
||||
monitorUrl: "http://a.com",
|
||||
monitorType: "http",
|
||||
monitorInterval: 60000,
|
||||
monitorGeoInterval: 300000,
|
||||
monitorActive: true,
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
failReason: "timeout",
|
||||
lastRunAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastRunTook: 100,
|
||||
lastFailedAt: 900,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null lastRunTook when job is locked", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{
|
||||
id: "m1",
|
||||
template: "monitor-job",
|
||||
active: true,
|
||||
lockedAt: 999,
|
||||
runCount: 1,
|
||||
lastStartedAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
data: { url: "a", type: "http", interval: 60000 },
|
||||
},
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].lastRunTook).toBeNull();
|
||||
});
|
||||
|
||||
it("leaves monitor fields null for non-monitor jobs", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, data: undefined }]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].monitorUrl).toBeNull();
|
||||
expect(jobs[0].monitorType).toBeNull();
|
||||
expect(jobs[0].monitorInterval).toBeNull();
|
||||
expect(jobs[0].monitorGeoInterval).toBeNull();
|
||||
expect(jobs[0].monitorActive).toBeNull();
|
||||
});
|
||||
|
||||
it("pushes pagination to the scheduler and returns its total count", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", template: "monitor-job", active: true, data: { url: "a", type: "http", interval: 60000 } },
|
||||
]);
|
||||
(scheduler.countJobs as jest.Mock).mockResolvedValue(42);
|
||||
|
||||
const { jobs, count } = await queue.getJobs({ page: 2, rowsPerPage: 10 });
|
||||
|
||||
expect(scheduler.getJobs).toHaveBeenCalledWith({ skip: 20, limit: 10 });
|
||||
expect(count).toBe(42);
|
||||
expect(jobs).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("requests all jobs when rowsPerPage is omitted", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.countJobs as jest.Mock).mockResolvedValue(7);
|
||||
|
||||
const { count } = await queue.getJobs({});
|
||||
|
||||
expect(scheduler.getJobs).toHaveBeenCalledWith(undefined);
|
||||
expect(count).toBe(7);
|
||||
});
|
||||
});
|
||||
|
||||
// ── flushQueues ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("flushQueues", () => {
|
||||
it("stops, flushes, and reinitializes", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const result = await queue.flushQueues();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
expect(scheduler.flushJobs).toHaveBeenCalled();
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("flushes before stopping, so the store is still connected for removeAll", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const order: string[] = [];
|
||||
(scheduler.flushJobs as jest.Mock).mockImplementation(async () => {
|
||||
order.push("flush");
|
||||
return true;
|
||||
});
|
||||
(scheduler.stop as jest.Mock).mockImplementation(async () => {
|
||||
order.push("stop");
|
||||
return true;
|
||||
});
|
||||
await queue.flushQueues();
|
||||
expect(order).toEqual(["flush", "stop"]);
|
||||
});
|
||||
|
||||
it("refuses to flush on a non-primary worker without touching the scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue({ queueMode: "worker" });
|
||||
const result = await queue.flushQueues();
|
||||
expect(result.success).toBe(false);
|
||||
expect(scheduler.flushJobs).not.toHaveBeenCalled();
|
||||
expect(scheduler.stop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns false when flush fails", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.flushJobs as jest.Mock).mockResolvedValue(false);
|
||||
const result = await queue.flushQueues();
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── static create ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("static create", () => {
|
||||
const envSettings = { dbConnectionString: "mongodb://localhost:27017/test_db" } as any;
|
||||
|
||||
it("creates a MongoStore + Scheduler and inits in primary mode", async () => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const workersRepository = createWorkersRepo();
|
||||
|
||||
const instance = await LessSimpleQueue.create(
|
||||
logger as any,
|
||||
helper as any,
|
||||
monitorsRepository as any,
|
||||
workersRepository as any,
|
||||
envSettings,
|
||||
"primary"
|
||||
);
|
||||
|
||||
expect(MockMongoStore).toHaveBeenCalledWith(expect.objectContaining({ url: "mongodb://localhost:27017/test_db" }));
|
||||
expect(MockScheduler).toHaveBeenCalled();
|
||||
expect(instance).toBeInstanceOf(LessSimpleQueue);
|
||||
expect(mockSchedulerInstance.start).toHaveBeenCalled();
|
||||
expect(monitorsRepository.findAll).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("inits in worker mode without seeding monitors", async () => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const workersRepository = createWorkersRepo();
|
||||
|
||||
const instance = await LessSimpleQueue.create(
|
||||
logger as any,
|
||||
helper as any,
|
||||
monitorsRepository as any,
|
||||
workersRepository as any,
|
||||
envSettings,
|
||||
"worker"
|
||||
);
|
||||
|
||||
expect(instance).toBeInstanceOf(LessSimpleQueue);
|
||||
expect(monitorsRepository.findAll).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── worker heartbeat / deregistration ────────────────────────────────────────
|
||||
|
||||
describe("worker heartbeat", () => {
|
||||
it("writes a heartbeat on init with the workerId and mode (primary)", async () => {
|
||||
const { queue, workersRepository } = createQueue();
|
||||
await queue.init("primary");
|
||||
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "primary");
|
||||
});
|
||||
|
||||
it("writes a heartbeat on init with the workerId and mode (worker)", async () => {
|
||||
const { queue, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker");
|
||||
});
|
||||
|
||||
it("beats again on each scheduler:heartbeat event", async () => {
|
||||
const { queue, scheduler, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
(workersRepository.upsert as jest.Mock).mockClear();
|
||||
|
||||
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
|
||||
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
|
||||
|
||||
expect(workersRepository.upsert).toHaveBeenCalledTimes(2);
|
||||
expect(workersRepository.upsert).toHaveBeenCalledWith("host-1:1234:abcd", "worker");
|
||||
});
|
||||
|
||||
it("stops beating after shutdown", async () => {
|
||||
const { queue, scheduler, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
await queue.shutdown();
|
||||
(workersRepository.upsert as jest.Mock).mockClear();
|
||||
|
||||
scheduler.emit("scheduler:heartbeat", "host-1:1234:abcd");
|
||||
|
||||
expect(workersRepository.upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("deregisters the worker and stops the scheduler on shutdown", async () => {
|
||||
const { queue, scheduler, workersRepository } = createQueue();
|
||||
await queue.init("worker");
|
||||
await queue.shutdown();
|
||||
expect(workersRepository.deleteById).toHaveBeenCalledWith("host-1:1234:abcd");
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,712 +0,0 @@
|
||||
import { describe, expect, it, jest, beforeEach } from "@jest/globals";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.types.ts";
|
||||
|
||||
// ── Mock Scheduler ───────────────────────────────────────────────────────────
|
||||
|
||||
const createScheduler = () => {
|
||||
const listeners: Record<string, Function[]> = {};
|
||||
return {
|
||||
on: jest.fn((event: string, cb: Function) => {
|
||||
listeners[event] = listeners[event] || [];
|
||||
listeners[event].push(cb);
|
||||
}),
|
||||
emit: (event: string, ...args: unknown[]) => {
|
||||
(listeners[event] || []).forEach((cb) => cb(...args));
|
||||
},
|
||||
start: jest.fn(),
|
||||
stop: jest.fn().mockResolvedValue(true),
|
||||
addTemplate: jest.fn(),
|
||||
addJob: jest.fn(),
|
||||
removeJob: jest.fn(),
|
||||
getJob: jest.fn().mockResolvedValue(null),
|
||||
getJobs: jest.fn().mockResolvedValue([]),
|
||||
pauseJob: jest.fn().mockResolvedValue(true),
|
||||
resumeJob: jest.fn().mockResolvedValue(true),
|
||||
updateJob: jest.fn(),
|
||||
flushJobs: jest.fn().mockResolvedValue(true),
|
||||
};
|
||||
};
|
||||
|
||||
const mockSchedulerInstance = createScheduler();
|
||||
const MockScheduler = jest.fn().mockReturnValue(mockSchedulerInstance);
|
||||
|
||||
jest.unstable_mockModule("super-simple-scheduler", () => ({
|
||||
default: MockScheduler,
|
||||
}));
|
||||
|
||||
const { SuperSimpleQueue } = await import("../../../src/worker/worker.super-simple.ts");
|
||||
|
||||
const createQueueHelper = () => ({
|
||||
getHeartbeatJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getHeartbeatGeoJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupOrphanedJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
getCleanupRetentionJob: jest.fn().mockReturnValue(() => Promise.resolve()),
|
||||
});
|
||||
|
||||
const createMonitorsRepo = () => ({
|
||||
findAll: jest.fn().mockResolvedValue([]),
|
||||
});
|
||||
|
||||
const createQueue = (overrides?: Record<string, unknown>) => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
const scheduler = createScheduler();
|
||||
|
||||
const defaults = { logger, helper, monitorsRepository, scheduler, ...overrides };
|
||||
|
||||
const queue = new SuperSimpleQueue(defaults.logger as any, defaults.helper as any, defaults.monitorsRepository as any, defaults.scheduler as any);
|
||||
return { queue, ...defaults };
|
||||
};
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "mon-1",
|
||||
teamId: "team-1",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
isActive: true,
|
||||
geoCheckEnabled: false,
|
||||
geoCheckInterval: 300000,
|
||||
geoCheckLocations: [],
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("SuperSimpleQueue", () => {
|
||||
describe("serviceName", () => {
|
||||
it("returns JobQueue", () => {
|
||||
const { queue } = createQueue();
|
||||
expect(queue.serviceName).toBe("JobQueue");
|
||||
});
|
||||
});
|
||||
|
||||
// ── init ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("init", () => {
|
||||
it("registers listeners, starts scheduler, adds templates and system jobs", async () => {
|
||||
const { queue, scheduler, helper } = createQueue();
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(scheduler.start).toHaveBeenCalled();
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("monitor-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("geo-check-job", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-orphaned", expect.any(Function));
|
||||
expect(scheduler.addTemplate).toHaveBeenCalledWith("cleanup-retention-job", expect.any(Function));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-retention" }));
|
||||
});
|
||||
|
||||
it("schedules existing monitors with staggered offsets", async () => {
|
||||
jest.useFakeTimers();
|
||||
const monitors = [makeMonitor({ id: "m1" }), makeMonitor({ id: "m2" })];
|
||||
const { queue, scheduler, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(monitors);
|
||||
|
||||
await queue.init();
|
||||
jest.runAllTimers();
|
||||
|
||||
// Each monitor gets addJob called (once for the monitor itself via addJob method)
|
||||
// The init schedules via setTimeout → addJob, plus the two system jobs
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m1", template: "monitor-job" }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "m2", template: "monitor-job" }));
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it("returns true when findAll returns null", async () => {
|
||||
const { queue, monitorsRepository, scheduler } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockResolvedValue(null);
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(true);
|
||||
// System jobs should not be added when monitors is null (early return)
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "cleanup-orphaned" }));
|
||||
});
|
||||
|
||||
it("returns false and logs error on failure", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("db down") }));
|
||||
});
|
||||
|
||||
it("logs error with String(error) for non-Error exceptions", async () => {
|
||||
const { queue, logger, monitorsRepository } = createQueue();
|
||||
(monitorsRepository.findAll as jest.Mock).mockRejectedValue("string error");
|
||||
|
||||
const result = await queue.init();
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("string error") }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── registerListeners ────────────────────────────────────────────────────
|
||||
|
||||
describe("registerListeners (via init)", () => {
|
||||
it("logs on scheduler:start", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:start");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler started" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:stop", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:stop");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler stopped" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:error", new Error("boom"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: boom" }));
|
||||
});
|
||||
|
||||
it("logs on scheduler:error with non-Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("scheduler:error", "string error");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Scheduler error: string error", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:abort", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:abort", { id: "m1" }, "timeout");
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 aborted: timeout" }));
|
||||
});
|
||||
|
||||
it("logs on job:attempt", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:attempt", { id: "m1" }, 2);
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 attempt 2" }));
|
||||
});
|
||||
|
||||
it("logs on job:complete", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:complete", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 completed successfully" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:exhausted", { id: "m1" }, new Error("gave up"));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up" }));
|
||||
});
|
||||
|
||||
it("logs on job:exhausted with non-Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:exhausted", { id: "m1" }, "gave up");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 exhausted all retries: gave up", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:fail", { id: "m1" }, new Error("oops"), 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops" }));
|
||||
});
|
||||
|
||||
it("logs on job:fail with non-Error", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:fail", { id: "m1" }, "oops", 1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 failed on attempt 1: oops", stack: undefined }));
|
||||
});
|
||||
|
||||
it("logs on job:start", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
await queue.init();
|
||||
scheduler.emit("job:start", { id: "m1" });
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "m1 started" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── addJob ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("addJob", () => {
|
||||
it("adds monitor job to scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor());
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1", template: "monitor-job", repeat: 60000, active: true }));
|
||||
});
|
||||
|
||||
it("adds geo check job when geoCheckEnabled and type supports it", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 }));
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job", repeat: 300000 }));
|
||||
});
|
||||
|
||||
it("skips geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" }));
|
||||
});
|
||||
|
||||
it("skips geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.addJob("mon-1", makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.addJob).not.toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── deleteJob ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("deleteJob", () => {
|
||||
it("removes monitor job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1");
|
||||
});
|
||||
|
||||
it("removes geo job if it exists", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not remove geo job if it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.deleteJob(makeMonitor());
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
});
|
||||
|
||||
// ── pauseJob ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("pauseJob", () => {
|
||||
it("pauses monitor and pauses geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.pauseJob(makeMonitor());
|
||||
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Paused monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("refreshes scheduler data with current monitor on pause", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ isActive: false, status: "paused" });
|
||||
|
||||
await queue.pauseJob(monitor);
|
||||
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor });
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor });
|
||||
});
|
||||
|
||||
it("does not pause geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.pauseJob(makeMonitor());
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.pauseJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not refresh geo data when geo job does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.pauseJob(makeMonitor());
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() }));
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything());
|
||||
});
|
||||
|
||||
it("throws when scheduler.pauseJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow("Failed to pause monitor");
|
||||
});
|
||||
|
||||
it("does not refresh data when main pauseJob throws", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.pauseJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.pauseJob(makeMonitor())).rejects.toThrow();
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo pauseJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.pauseJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.pauseJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to pause geo check job for monitor mon-1" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── resumeJob ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("resumeJob", () => {
|
||||
it("resumes monitor and geo job when geo exists", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.resumeJob(makeMonitor());
|
||||
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: "Resumed monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("refreshes scheduler data with current monitor on resume", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ isActive: true, status: "initializing" });
|
||||
|
||||
await queue.resumeJob(monitor);
|
||||
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", { data: monitor });
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", { data: monitor });
|
||||
});
|
||||
|
||||
it("does not resume geo job when it does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.resumeJob(makeMonitor());
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1");
|
||||
expect(scheduler.resumeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not refresh geo data when geo job does not exist", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.resumeJob(makeMonitor());
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ data: expect.anything() }));
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalledWith("mon-1-geo", expect.anything());
|
||||
});
|
||||
|
||||
it("throws when scheduler.resumeJob returns false for the main job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow("Failed to resume monitor");
|
||||
});
|
||||
|
||||
it("does not refresh data when main resumeJob throws", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.resumeJob as jest.Mock).mockResolvedValue(false);
|
||||
await expect(queue.resumeJob(makeMonitor())).rejects.toThrow();
|
||||
expect(scheduler.updateJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs error but does not throw when geo resumeJob returns false", async () => {
|
||||
const { queue, scheduler, logger } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
(scheduler.resumeJob as jest.Mock).mockImplementation(async (id: string) => id !== "mon-1-geo");
|
||||
|
||||
await expect(queue.resumeJob(makeMonitor())).resolves.toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to resume geo check job for monitor mon-1" }));
|
||||
});
|
||||
|
||||
it("preserves geo job through a pause/resume cycle", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
|
||||
await queue.pauseJob(makeMonitor());
|
||||
await queue.resumeJob(makeMonitor());
|
||||
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(scheduler.pauseJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
expect(scheduler.resumeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
});
|
||||
|
||||
// ── updateJob ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("updateJob", () => {
|
||||
it("updates monitor job and syncs geo job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ interval: 120000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1", expect.objectContaining({ repeat: 120000, data: monitor }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── syncGeoJob (via updateJob) ───────────────────────────────────────────
|
||||
|
||||
describe("syncGeoJob (via updateJob)", () => {
|
||||
it("removes existing geo job when geoCheckEnabled is false", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: false, type: "http" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("removes existing geo job when type does not support geo checks", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: true, type: "hardware" }));
|
||||
expect(scheduler.removeJob).toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("does not remove when geo disabled and no existing geo job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.updateJob(makeMonitor({ geoCheckEnabled: false }));
|
||||
expect(scheduler.removeJob).not.toHaveBeenCalledWith("mon-1-geo");
|
||||
});
|
||||
|
||||
it("updates existing geo job when geoCheckEnabled", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJob as jest.Mock).mockResolvedValue({ id: "mon-1-geo" });
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.updateJob).toHaveBeenCalledWith("mon-1-geo", expect.objectContaining({ repeat: 300000 }));
|
||||
});
|
||||
|
||||
it("creates new geo job when enabled but no existing job", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const monitor = makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckInterval: 300000 });
|
||||
await queue.updateJob(monitor);
|
||||
expect(scheduler.addJob).toHaveBeenCalledWith(expect.objectContaining({ id: "mon-1-geo", template: "geo-check-job" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── shutdown ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("shutdown", () => {
|
||||
it("stops the scheduler", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
await queue.shutdown();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── getMetrics ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("getMetrics", () => {
|
||||
it("returns empty job metrics plus this process as the sole worker when no jobs", async () => {
|
||||
const { queue } = createQueue();
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics).toEqual({
|
||||
jobs: 0,
|
||||
activeJobs: 0,
|
||||
failingJobs: 0,
|
||||
jobsWithFailures: [],
|
||||
totalRuns: 0,
|
||||
totalFailures: 0,
|
||||
workers: [
|
||||
{
|
||||
// hostname:pid:uuid, matching less-simple-scheduler's workerId format
|
||||
workerId: expect.stringMatching(/^.+:\d+:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/),
|
||||
mode: "primary",
|
||||
lastSeenAt: expect.any(Number),
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("aggregates metrics from jobs", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", runCount: 10, failCount: 2, lastFailedAt: 200, lastRunAt: 100, lockedAt: null, data: { url: "http://a.com", type: "http" } },
|
||||
{ id: "m2", runCount: 5, failCount: 0, lastFailedAt: 0, lastRunAt: 50, lockedAt: 999, data: { url: "http://b.com", type: "ping" } },
|
||||
]);
|
||||
const metrics = await queue.getMetrics();
|
||||
|
||||
expect(metrics.jobs).toBe(2);
|
||||
expect(metrics.totalRuns).toBe(15);
|
||||
expect(metrics.totalFailures).toBe(2);
|
||||
expect(metrics.activeJobs).toBe(1);
|
||||
expect(metrics.failingJobs).toBe(1);
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
expect(metrics.jobsWithFailures[0]).toEqual(expect.objectContaining({ monitorId: "m1", failCount: 2, monitorUrl: "http://a.com" }));
|
||||
});
|
||||
|
||||
it("handles jobs with undefined fields", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", data: null }]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobs).toBe(1);
|
||||
expect(metrics.totalRuns).toBe(0);
|
||||
});
|
||||
|
||||
it("handles failing job with null data", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", runCount: 1, failCount: 1, lastFailedAt: 100, lastRunAt: 50, data: null, lastFailReason: "timeout" },
|
||||
]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
expect(metrics.jobsWithFailures[0].monitorUrl).toBeNull();
|
||||
expect(metrics.jobsWithFailures[0].monitorType).toBeNull();
|
||||
expect(metrics.jobsWithFailures[0].failReason).toBe("timeout");
|
||||
});
|
||||
|
||||
it("handles failing job with undefined lastFailedAt and lastFailReason", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", runCount: 1, failCount: 1, data: { url: "http://a.com", type: "http" } }]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.jobsWithFailures[0].failedAt).toBeNull();
|
||||
expect(metrics.jobsWithFailures[0].failReason).toBeNull();
|
||||
});
|
||||
|
||||
it("does not count as failing when lastFailedAt < lastRunAt", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", runCount: 10, failCount: 1, lastFailedAt: 50, lastRunAt: 100, data: { url: "a" } },
|
||||
]);
|
||||
const metrics = await queue.getMetrics();
|
||||
expect(metrics.failingJobs).toBe(0);
|
||||
expect(metrics.jobsWithFailures).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── getJobs ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getJobs", () => {
|
||||
it("maps scheduler jobs to summaries", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{
|
||||
id: "m1",
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
lastFailReason: "timeout",
|
||||
lastRunAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastFailedAt: 900,
|
||||
data: { url: "http://a.com", type: "http", interval: 60000, geoCheckInterval: 300000, isActive: true },
|
||||
},
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]).toEqual({
|
||||
monitorId: "m1",
|
||||
monitorUrl: "http://a.com",
|
||||
monitorType: "http",
|
||||
monitorInterval: 60000,
|
||||
monitorGeoInterval: 300000,
|
||||
monitorActive: true,
|
||||
active: true,
|
||||
repeat: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
lockedAt: null,
|
||||
runCount: 5,
|
||||
failCount: 1,
|
||||
failReason: "timeout",
|
||||
lastRunAt: 1000,
|
||||
lastFinishedAt: 1100,
|
||||
lastRunTook: 100,
|
||||
lastFailedAt: 900,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null lastRunTook when job is locked", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", active: true, lockedAt: 999, runCount: 1, data: { url: "a", type: "http", interval: 60000 } },
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].lastRunTook).toBeNull();
|
||||
});
|
||||
|
||||
it("handles jobs with missing data", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([{ id: "m1", active: true, data: null }]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].monitorUrl).toBeNull();
|
||||
expect(jobs[0].monitorType).toBeNull();
|
||||
expect(jobs[0].monitorInterval).toBeNull();
|
||||
expect(jobs[0].monitorGeoInterval).toBeNull();
|
||||
expect(jobs[0].monitorActive).toBeNull();
|
||||
expect(jobs[0].repeat).toBeNull();
|
||||
});
|
||||
|
||||
it("exposes monitor.isActive as monitorActive in the summary", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", active: true, data: { url: "a", type: "http", interval: 60000, isActive: true } },
|
||||
{ id: "m2", active: false, data: { url: "b", type: "http", interval: 60000, isActive: false } },
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].monitorActive).toBe(true);
|
||||
expect(jobs[1].monitorActive).toBe(false);
|
||||
});
|
||||
|
||||
it("exposes geo check interval and repeat", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue([
|
||||
{ id: "m1", active: true, repeat: 120000, data: { url: "a", type: "http", interval: 60000, geoCheckInterval: 600000 } },
|
||||
]);
|
||||
const { jobs } = await queue.getJobs({});
|
||||
expect(jobs[0].repeat).toBe(120000);
|
||||
expect(jobs[0].monitorGeoInterval).toBe(600000);
|
||||
});
|
||||
|
||||
it("paginates jobs and returns the total count", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue(
|
||||
Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } }))
|
||||
);
|
||||
|
||||
const { jobs, count } = await queue.getJobs({ page: 1, rowsPerPage: 10 });
|
||||
expect(count).toBe(25);
|
||||
expect(jobs).toHaveLength(10);
|
||||
expect(jobs[0].monitorId).toBe("m10");
|
||||
});
|
||||
|
||||
it("returns all jobs when rowsPerPage is omitted", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.getJobs as jest.Mock).mockResolvedValue(
|
||||
Array.from({ length: 25 }, (_, i) => ({ id: `m${i}`, active: true, data: { url: `a${i}`, type: "http", interval: 60000 } }))
|
||||
);
|
||||
|
||||
const { jobs, count } = await queue.getJobs({});
|
||||
expect(count).toBe(25);
|
||||
expect(jobs).toHaveLength(25);
|
||||
});
|
||||
});
|
||||
|
||||
// ── flushQueues ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("flushQueues", () => {
|
||||
it("stops, flushes, and reinitializes", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
const result = await queue.flushQueues();
|
||||
expect(scheduler.stop).toHaveBeenCalled();
|
||||
expect(scheduler.flushJobs).toHaveBeenCalled();
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false when any step fails", async () => {
|
||||
const { queue, scheduler } = createQueue();
|
||||
(scheduler.flushJobs as jest.Mock).mockResolvedValue(false);
|
||||
const result = await queue.flushQueues();
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── static create ────────────────────────────────────────────────────────
|
||||
|
||||
describe("static create", () => {
|
||||
it("creates a SuperSimpleQueue instance with a Scheduler and calls init", async () => {
|
||||
const logger = createMockLogger();
|
||||
const helper = createQueueHelper();
|
||||
const monitorsRepository = createMonitorsRepo();
|
||||
|
||||
const instance = await SuperSimpleQueue.create(logger as any, helper as any, monitorsRepository as any);
|
||||
|
||||
expect(MockScheduler).toHaveBeenCalled();
|
||||
expect(instance).toBeInstanceOf(SuperSimpleQueue);
|
||||
// init was called, which calls scheduler.start
|
||||
expect(mockSchedulerInstance.start).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
+8
@@ -17,6 +17,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
findAllMonitorIds: jest.fn().mockResolvedValue(["m1"]),
|
||||
deleteByTeamIdsNotIn: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
const jobsRepository = {
|
||||
deleteByMonitorIdsNotIn: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
const teamsRepository = {
|
||||
findAllTeamIds: jest.fn().mockResolvedValue(["team"]),
|
||||
};
|
||||
@@ -48,6 +51,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
checkService,
|
||||
settingsService,
|
||||
monitorsRepository,
|
||||
jobsRepository,
|
||||
teamsRepository,
|
||||
monitorStatsRepository,
|
||||
checksRepository,
|
||||
@@ -64,6 +68,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
defaults.checkService as any,
|
||||
defaults.settingsService as any,
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.jobsRepository as any,
|
||||
defaults.teamsRepository as any,
|
||||
defaults.monitorStatsRepository as any,
|
||||
defaults.checksRepository as any,
|
||||
@@ -170,6 +175,7 @@ describe("WorkerHelper", () => {
|
||||
(defaults.checksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(4);
|
||||
(defaults.incidentsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(1);
|
||||
(defaults.geoChecksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(5);
|
||||
(defaults.jobsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(6);
|
||||
|
||||
const job = helper.getCleanupOrphanedJob();
|
||||
await job();
|
||||
@@ -181,11 +187,13 @@ describe("WorkerHelper", () => {
|
||||
expect(defaults.checksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.incidentsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.geoChecksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.jobsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("2 orphaned monitors") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("3 orphaned monitor stats") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("4 orphaned checks") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 orphaned incidents") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("5 orphaned geo checks") }));
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("6 orphaned jobs") }));
|
||||
});
|
||||
|
||||
it("skips info logs when no orphaned data is found", async () => {
|
||||
Reference in new issue
Block a user