port egress to pipeline
This commit is contained in:
1 parent
01e9655f42
commit
8dd57d45e2
27 files changed
+19071
-15939
No files matched your search
@@ -1,10 +1,8 @@
|
||||
import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals";
|
||||
import { BufferService } from "../../../src/service/bufferService.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { ICheckService } from "../../../src/domain/checks/check.service.ts";
|
||||
import type { IGeoChecksService } from "../../../src/domain/geo-checks/geo-check.service.ts";
|
||||
import type { ISettingsService } from "../../../src/domain/app-settings/app-settings.service.ts";
|
||||
import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts";
|
||||
import type { Check } from "../../../src/domain/checks/check.type.ts";
|
||||
import type { GeoCheck } from "../../../src/domain/geo-checks/geo-check.type.ts";
|
||||
import type { IDockerLogsService } from "../../../src/domain/docker/docker-log.service.ts";
|
||||
@@ -12,11 +10,6 @@ import type { DockerLog } from "../../../src/domain/docker/docker-log.type.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const createMockCheckService = () =>
|
||||
({
|
||||
createChecks: jest.fn().mockResolvedValue([]),
|
||||
}) as unknown as jest.Mocked<ICheckService>;
|
||||
|
||||
const createMockGeoChecksService = () =>
|
||||
({
|
||||
createGeoChecks: jest.fn().mockResolvedValue([]),
|
||||
@@ -32,11 +25,6 @@ const createMockSettingsService = (nodeEnv: string = "development") =>
|
||||
getSettings: jest.fn().mockReturnValue({ nodeEnv }),
|
||||
}) as unknown as jest.Mocked<ISettingsService>;
|
||||
|
||||
const createMockJobsRepository = () =>
|
||||
({
|
||||
upsertEvaluate: jest.fn().mockResolvedValue(true),
|
||||
}) as unknown as jest.Mocked<IJobsRepository>;
|
||||
|
||||
const makeCheck = (overrides?: Partial<Check>): Check =>
|
||||
({
|
||||
id: "check-1",
|
||||
@@ -72,13 +60,12 @@ const makeDockerLog = (overrides?: Partial<DockerLog>): DockerLog =>
|
||||
|
||||
const createService = (nodeEnv: string = "development") => {
|
||||
const logger = createMockLogger();
|
||||
const checkService = createMockCheckService();
|
||||
const ingest = jest.fn<(checks: Check[]) => Promise<void>>().mockResolvedValue(undefined);
|
||||
const geoChecksService = createMockGeoChecksService();
|
||||
const dockerLogsService = createMockDockerLogsService();
|
||||
const settingsService = createMockSettingsService(nodeEnv);
|
||||
const jobsRepository = createMockJobsRepository();
|
||||
const service = new BufferService(logger as any, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
|
||||
return { service, logger, checkService, geoChecksService, dockerLogsService, jobsRepository };
|
||||
const service = new BufferService(logger as any, geoChecksService, dockerLogsService, settingsService, ingest);
|
||||
return { service, logger, ingest, geoChecksService, dockerLogsService };
|
||||
};
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
@@ -127,17 +114,17 @@ describe("BufferService", () => {
|
||||
|
||||
describe("addToBuffer", () => {
|
||||
it("adds a check to the buffer", async () => {
|
||||
const { service, checkService } = createService();
|
||||
const { service, ingest } = createService();
|
||||
const check = makeCheck();
|
||||
|
||||
service.addToBuffer(check);
|
||||
await service.flushBuffer();
|
||||
|
||||
expect(checkService.createChecks).toHaveBeenCalledWith([check]);
|
||||
expect(ingest).toHaveBeenCalledWith([check]);
|
||||
});
|
||||
|
||||
it("adds multiple checks to the buffer", async () => {
|
||||
const { service, checkService } = createService();
|
||||
const { service, ingest } = createService();
|
||||
const check1 = makeCheck({ id: "c1" });
|
||||
const check2 = makeCheck({ id: "c2" });
|
||||
|
||||
@@ -145,7 +132,7 @@ describe("BufferService", () => {
|
||||
service.addToBuffer(check2);
|
||||
await service.flushBuffer();
|
||||
|
||||
expect(checkService.createChecks).toHaveBeenCalledWith([check1, check2]);
|
||||
expect(ingest).toHaveBeenCalledWith([check1, check2]);
|
||||
});
|
||||
|
||||
it("logs error if push throws", () => {
|
||||
@@ -268,21 +255,20 @@ describe("BufferService", () => {
|
||||
});
|
||||
|
||||
it("flushes all buffers when timer fires", async () => {
|
||||
const { service, checkService, geoChecksService, dockerLogsService } = createService();
|
||||
const { service, ingest, geoChecksService, dockerLogsService } = createService();
|
||||
service.addToBuffer(makeCheck());
|
||||
service.addGeoCheckToBuffer(makeGeoCheck());
|
||||
service.addDockerLogToBuffer(makeDockerLog());
|
||||
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
expect(checkService.createChecks).toHaveBeenCalled();
|
||||
expect(ingest).toHaveBeenCalledWith([expect.objectContaining({ id: "check-1" })]);
|
||||
expect(geoChecksService.createGeoChecks).toHaveBeenCalled();
|
||||
expect(dockerLogsService.createDockerLogs).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reschedules after flush completes", async () => {
|
||||
const { service, checkService } = createService();
|
||||
(checkService.createChecks as jest.Mock).mockResolvedValue([]);
|
||||
const { service, ingest } = createService();
|
||||
|
||||
service.addToBuffer(makeCheck());
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
@@ -291,12 +277,12 @@ describe("BufferService", () => {
|
||||
service.addToBuffer(makeCheck({ id: "c2" }));
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(2);
|
||||
expect(ingest).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("reschedules even when flush throws", async () => {
|
||||
const { service, checkService, logger } = createService();
|
||||
(checkService.createChecks as jest.Mock).mockRejectedValueOnce(new Error("DB down"));
|
||||
const { service, ingest, logger } = createService();
|
||||
ingest.mockRejectedValueOnce(new Error("DB down"));
|
||||
service.addToBuffer(makeCheck());
|
||||
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
@@ -309,11 +295,10 @@ describe("BufferService", () => {
|
||||
);
|
||||
|
||||
// Should still reschedule — add another check and flush
|
||||
(checkService.createChecks as jest.Mock).mockResolvedValue([]);
|
||||
service.addToBuffer(makeCheck({ id: "c2" }));
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(2);
|
||||
expect(ingest).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("logs error and reschedules when flush throws past its own catch", async () => {
|
||||
@@ -350,25 +335,26 @@ describe("BufferService", () => {
|
||||
// ── flushBuffer ──────────────────────────────────────────────────────
|
||||
|
||||
describe("flushBuffer", () => {
|
||||
it("does nothing when buffer is empty", async () => {
|
||||
const { service, checkService } = createService();
|
||||
it("hands an empty batch to ingest so the pipeline can retry anything it left unarmed", async () => {
|
||||
const { service, ingest, logger } = createService();
|
||||
|
||||
await service.flushBuffer();
|
||||
|
||||
expect(checkService.createChecks).not.toHaveBeenCalled();
|
||||
expect(ingest).toHaveBeenCalledWith([]);
|
||||
expect(logger.debug).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("flushes checks to checksService and clears buffer", async () => {
|
||||
const { service, checkService } = createService();
|
||||
it("hands the batch to ingest and clears the buffer", async () => {
|
||||
const { service, ingest } = createService();
|
||||
const check = makeCheck();
|
||||
service.addToBuffer(check);
|
||||
|
||||
await service.flushBuffer();
|
||||
|
||||
expect(checkService.createChecks).toHaveBeenCalledWith([check]);
|
||||
expect(ingest).toHaveBeenNthCalledWith(1, [check]);
|
||||
// Buffer should be empty now
|
||||
await service.flushBuffer();
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(ingest).toHaveBeenNthCalledWith(2, []);
|
||||
});
|
||||
|
||||
it("logs debug message before flushing", async () => {
|
||||
@@ -386,68 +372,11 @@ describe("BufferService", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("arms the evaluate stage once per distinct monitor after the checks are stored", async () => {
|
||||
const { service, jobsRepository } = createService();
|
||||
service.addToBuffer(makeCheck({ id: "c1", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }));
|
||||
service.addToBuffer(makeCheck({ id: "c2", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }));
|
||||
service.addToBuffer(makeCheck({ id: "c3", metadata: { monitorId: "mon-2", teamId: "team-1", type: "http" } }));
|
||||
|
||||
await service.flushBuffer();
|
||||
|
||||
const createdAt = Date.parse("2026-01-01T00:00:00.000Z");
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith(
|
||||
"mon-1",
|
||||
[
|
||||
{ checkId: "c1", createdAt },
|
||||
{ checkId: "c2", createdAt },
|
||||
],
|
||||
expect.any(Number)
|
||||
);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
|
||||
});
|
||||
|
||||
it("retries a failed arm on the next flush without re-storing the checks", async () => {
|
||||
const { service, checkService, jobsRepository, logger } = createService();
|
||||
(jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed"));
|
||||
service.addToBuffer(makeCheck({ id: "c1" }));
|
||||
|
||||
await service.flushBuffer();
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
|
||||
|
||||
await service.flushBuffer(); // buffer is empty, only the retry runs
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number));
|
||||
|
||||
await service.flushBuffer(); // nothing left to retry
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("ingestChecks with an empty batch neither writes nor arms", async () => {
|
||||
const { service, checkService, jobsRepository } = createService();
|
||||
|
||||
await service.ingestChecks([]);
|
||||
|
||||
expect(checkService.createChecks).not.toHaveBeenCalled();
|
||||
expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not arm the evaluate stage when the check write fails", async () => {
|
||||
const { service, checkService, jobsRepository } = createService();
|
||||
(checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed"));
|
||||
service.addToBuffer(makeCheck());
|
||||
|
||||
await service.flushBuffer();
|
||||
|
||||
expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clears buffer even on error to prevent infinite retries", async () => {
|
||||
const { service, checkService, logger } = createService();
|
||||
(checkService.createChecks as jest.Mock).mockRejectedValue(new Error("DB write failed"));
|
||||
service.addToBuffer(makeCheck());
|
||||
const { service, ingest, logger } = createService();
|
||||
ingest.mockRejectedValueOnce(new Error("DB write failed"));
|
||||
const check = makeCheck();
|
||||
service.addToBuffer(check);
|
||||
|
||||
await service.flushBuffer();
|
||||
|
||||
@@ -457,14 +386,14 @@ describe("BufferService", () => {
|
||||
method: "flushBuffer",
|
||||
})
|
||||
);
|
||||
// Buffer should be cleared
|
||||
// Buffer should be cleared: the failed batch is not handed over again
|
||||
await service.flushBuffer();
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(ingest).toHaveBeenNthCalledWith(2, []);
|
||||
});
|
||||
|
||||
it("logs 'Unknown error' for non-Error thrown values", async () => {
|
||||
const { service, checkService, logger } = createService();
|
||||
(checkService.createChecks as jest.Mock).mockRejectedValue(null);
|
||||
const { service, ingest, logger } = createService();
|
||||
ingest.mockRejectedValue(null);
|
||||
service.addToBuffer(makeCheck());
|
||||
|
||||
await service.flushBuffer();
|
||||
@@ -609,28 +538,28 @@ describe("BufferService", () => {
|
||||
|
||||
describe("shutdown", () => {
|
||||
it("stops the flush timer and flushes all buffers", async () => {
|
||||
const { service, checkService, geoChecksService, dockerLogsService } = createService();
|
||||
const { service, ingest, geoChecksService, dockerLogsService } = createService();
|
||||
service.addToBuffer(makeCheck());
|
||||
service.addGeoCheckToBuffer(makeGeoCheck());
|
||||
service.addDockerLogToBuffer(makeDockerLog());
|
||||
|
||||
await service.shutdown();
|
||||
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(ingest).toHaveBeenCalledTimes(1);
|
||||
expect(geoChecksService.createGeoChecks).toHaveBeenCalledTimes(1);
|
||||
expect(dockerLogsService.createDockerLogs).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Timer is cleared: advancing past the flush interval triggers no further flush.
|
||||
await jest.advanceTimersByTimeAsync(60 * 1000);
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(ingest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("is safe to call with empty buffers", async () => {
|
||||
const { service, checkService, geoChecksService, dockerLogsService } = createService();
|
||||
const { service, ingest, geoChecksService, dockerLogsService } = createService();
|
||||
|
||||
await service.shutdown();
|
||||
|
||||
expect(checkService.createChecks).not.toHaveBeenCalled();
|
||||
expect(ingest).toHaveBeenCalledWith([]);
|
||||
expect(geoChecksService.createGeoChecks).not.toHaveBeenCalled();
|
||||
expect(dockerLogsService.createDockerLogs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckEvaluator } from "../../../src/worker/worker.check-evaluator.ts";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
const createEvaluator = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
statusService: {
|
||||
updateMonitorStatus: jest.fn().mockResolvedValue({ monitor: makeMonitor({ status: "up" }), statusChanged: false, prevStatus: "up", code: 200 }),
|
||||
},
|
||||
monitorStatusPolicy: new MonitorStatusPolicy(),
|
||||
logger: createMockLogger(),
|
||||
...overrides,
|
||||
};
|
||||
const evaluator = new CheckEvaluator(defaults.statusService as any, defaults.monitorStatusPolicy as any, defaults.logger as any);
|
||||
return { evaluator, defaults };
|
||||
};
|
||||
|
||||
describe("CheckEvaluator", () => {
|
||||
it("threads status, check, and monitor through to the status service", async () => {
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const check = { id: "check-1" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
|
||||
await evaluator.evaluate(status, check, monitor);
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor);
|
||||
});
|
||||
|
||||
it("returns an evaluation that opens an incident on a down transition", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1" } as any;
|
||||
const statusChange = { monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 };
|
||||
const { evaluator } = createEvaluator({
|
||||
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
|
||||
});
|
||||
|
||||
const result = await evaluator.evaluate(status, check, makeMonitor());
|
||||
|
||||
expect(result).toMatchObject({
|
||||
monitor: statusChange.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange,
|
||||
decision: expect.objectContaining({
|
||||
shouldCreateIncident: true,
|
||||
shouldSendNotification: true,
|
||||
incidentReason: "status_down",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("returns an evaluation that resolves an incident on recovery", async () => {
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const statusChange = { monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down", code: 200 };
|
||||
const { evaluator } = createEvaluator({
|
||||
statusService: { updateMonitorStatus: jest.fn().mockResolvedValue(statusChange) },
|
||||
});
|
||||
|
||||
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor({ status: "down" }));
|
||||
|
||||
expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
|
||||
});
|
||||
|
||||
it("produces a no-op decision when status did not change", async () => {
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const { evaluator } = createEvaluator();
|
||||
|
||||
const result = await evaluator.evaluate(status, { id: "check-1" } as any, makeMonitor());
|
||||
|
||||
expect(result.decision).toMatchObject({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
});
|
||||
});
|
||||
|
||||
// ── degraded egress ───────────────────────────────────────────────────────
|
||||
|
||||
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "degraded" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
const result = await evaluator.evaluate(status, check, monitor);
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).not.toHaveBeenCalled();
|
||||
expect(result.monitor).toBe(monitor);
|
||||
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
|
||||
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
|
||||
expect(result.decision).toEqual({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
});
|
||||
expect(defaults.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
|
||||
});
|
||||
|
||||
it("evaluates a check flagged with egress ok as normal", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "ok" } as any;
|
||||
const { evaluator, defaults } = createEvaluator();
|
||||
|
||||
await evaluator.evaluate(status, check, makeMonitor());
|
||||
|
||||
expect(defaults.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
|
||||
});
|
||||
});
|
||||
@@ -1,461 +0,0 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { CheckProducer } from "../../../src/worker/worker.check-producer.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import { NETWORK_ERROR } from "../../../src/types/network.ts";
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
// A maintenance window straddling "now" — active per isWindowActive.
|
||||
const activeWindow = () => {
|
||||
const now = Date.now();
|
||||
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
|
||||
};
|
||||
|
||||
const createProducer = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
monitorsRepository: { updateById: jest.fn().mockResolvedValue({}), findDockerTlsKeyById: jest.fn().mockResolvedValue(null) },
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
|
||||
buffer: { addToBuffer: jest.fn() },
|
||||
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue(null) },
|
||||
...overrides,
|
||||
};
|
||||
const producer = new CheckProducer(
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.maintenanceWindowsRepository as any,
|
||||
defaults.checkService as any,
|
||||
defaults.networkService as any,
|
||||
defaults.proxyResolver as any,
|
||||
defaults.buffer as any,
|
||||
defaults.dockerLogsService as any,
|
||||
defaults.egressService as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { producer, defaults };
|
||||
};
|
||||
|
||||
describe("CheckProducer", () => {
|
||||
it("throws when monitor id is missing", async () => {
|
||||
const { producer } = createProducer();
|
||||
await expect(producer.produce({} as Monitor)).rejects.toThrow("No monitor id");
|
||||
});
|
||||
|
||||
// ── maintenance gate ──────────────────────────────────────────────────────
|
||||
|
||||
it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-write status when already in maintenance", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor({ status: "maintenance" }));
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
|
||||
const monitor = makeMonitor({ status: "maintenance" });
|
||||
const { producer, defaults } = createProducer({
|
||||
monitorsRepository: {
|
||||
updateById: jest.fn().mockResolvedValue({ ...monitor, status: "initializing" }),
|
||||
findDockerTlsKeyById: jest.fn().mockResolvedValue(null),
|
||||
},
|
||||
});
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
|
||||
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not touch the status of a monitor that was not in maintenance", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
|
||||
await producer.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("proceeds with the check when the maintenance window is inactive", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([{ ...activeWindow(), active: false }]) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── proxy resolution ──────────────────────────────────────────────────────
|
||||
|
||||
it("passes the resolved proxy url into requestStatus", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue("http://proxy.example.com:8080") },
|
||||
});
|
||||
const monitor = makeMonitor();
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: "http://proxy.example.com:8080" });
|
||||
});
|
||||
|
||||
it("still produces a check when the resolver returns undefined", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
const monitor = makeMonitor();
|
||||
|
||||
const result = await producer.produce(monitor);
|
||||
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined });
|
||||
expect(result).toEqual({ status: expect.objectContaining({ monitorId: "m1" }), check: { id: "check-1" } });
|
||||
});
|
||||
|
||||
// ── acquire / record ──────────────────────────────────────────────────────
|
||||
|
||||
// ── docker tls key resolution ─────────────────────────────────────────────
|
||||
|
||||
it("fetches the stored key for a docker monitor with a key set and passes it in the context", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
monitorsRepository: { updateById: jest.fn(), findDockerTlsKeyById: jest.fn().mockResolvedValue("v1.abc123.iv.tag.data") },
|
||||
});
|
||||
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: "v1.abc123.iv.tag.data" });
|
||||
});
|
||||
|
||||
it("does not fetch a key for a docker monitor without one set", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKeySet: false });
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
|
||||
});
|
||||
|
||||
it("does not fetch a key for non-docker monitors even if the flag is set", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
|
||||
await producer.produce(makeMonitor({ type: "http", dockerTlsKeySet: true }));
|
||||
|
||||
expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes undefined when the flag is set but the repository has no key", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
|
||||
});
|
||||
|
||||
it("throws when the network response is null", async () => {
|
||||
const { producer } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(null) },
|
||||
});
|
||||
|
||||
await expect(producer.produce(makeMonitor())).rejects.toThrow("No network response");
|
||||
});
|
||||
|
||||
it("returns null, warns, and does not buffer when toCheck yields nothing", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(null) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
|
||||
expect(defaults.buffer.addToBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("buffers the built check and returns the status and check on success", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
|
||||
const check = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(result).toEqual({ status, check });
|
||||
});
|
||||
|
||||
it("builds and buffers docker logs for a docker status", async () => {
|
||||
const status = { type: "docker", monitorId: "m1", teamId: "team", status: true, code: 200, message: "OK", payload: {} };
|
||||
const dockerLogs = [{ id: "docker-log-1" }, { id: "docker-log-2" }];
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
buffer: { addToBuffer: jest.fn(), addDockerLogToBuffer: jest.fn() },
|
||||
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue(dockerLogs) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "docker" }));
|
||||
|
||||
expect(defaults.dockerLogsService.buildDockerLogs).toHaveBeenCalledWith(status);
|
||||
expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(1, dockerLogs[0]);
|
||||
expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(2, dockerLogs[1]);
|
||||
});
|
||||
|
||||
it("does not build docker logs for a non-docker status", async () => {
|
||||
const { producer, defaults } = createProducer();
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── egress self-check ─────────────────────────────────────────────────────
|
||||
|
||||
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
|
||||
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
|
||||
|
||||
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("does not consult egress when an HTTP response was received", async () => {
|
||||
for (const status of [
|
||||
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
|
||||
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("consults egress for a timeout or network error", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("never consults egress for a docker daemon on a local socket", async () => {
|
||||
for (const url of ["unix:///var/run/docker.sock", "/var/run/docker.sock"]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: "docker" }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "docker", url }));
|
||||
|
||||
// Local IPC never leaves the instance, so it cannot fail through egress.
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
|
||||
for (const monitor of [
|
||||
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
|
||||
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(monitor);
|
||||
|
||||
// Both requests leave the instance, so our own loss of egress can be the cause.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not consult egress when the provider reports that the peer answered", async () => {
|
||||
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
|
||||
// them from a connection failure.
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "dns" }));
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ type: "grpc" }));
|
||||
|
||||
// An explicit report from the provider outranks the HTTP-status heuristic.
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
|
||||
expect(result?.check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
// ── status window across a spell ─────────────────────────────────────────
|
||||
// Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either
|
||||
// side of a spell are never adjacent.
|
||||
|
||||
it("clears the status window on the first degraded check of a spell", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] });
|
||||
});
|
||||
|
||||
it("does not re-write the status window for later degraded checks in the same spell", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves the status window alone when the probe found egress fine", async () => {
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(defaults.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("flags a failing check as ok when the probe found egress fine", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(check.egressStatus).toBe("ok");
|
||||
});
|
||||
|
||||
it("buffers the check when the egress assessment rejects", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn().mockRejectedValue(new Error("settings unavailable")) },
|
||||
});
|
||||
|
||||
const result = await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(result).toEqual({ status: failingStatus, check });
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "CheckProducer", method: "assessEgress" })
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { producer, defaults } = createProducer({
|
||||
networkService: { requestStatus: jest.fn().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await producer.produce(makeMonitor());
|
||||
|
||||
expect(defaults.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
});
|
||||
@@ -9,10 +9,11 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
// ── Notes ──────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
|
||||
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and
|
||||
// renews their locks, and registers itself in the worker heartbeat. The repository
|
||||
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
|
||||
// test only the orchestration around it, with the repository fully mocked.
|
||||
// sized to free capacity, hands each job to the handler registered for its type,
|
||||
// renews locks while a job runs, and registers itself in the worker heartbeat. What a
|
||||
// handler does is covered in workerPipeline.test.ts and geoChecksPipeline.test.ts; the
|
||||
// repository claim/lock atomicity in integration/jobsRepository.test.ts. Here we test
|
||||
// only the orchestration, with the repository and every handler mocked.
|
||||
//
|
||||
// Fake timers are mandatory: init() starts self-re-arming setTimeout loops and a
|
||||
// heartbeat setInterval, so without them the loops would leak across tests. The
|
||||
@@ -73,6 +74,17 @@ const makeEvaluateJob = (...checkIds: string[]): Job =>
|
||||
pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })),
|
||||
});
|
||||
|
||||
// One resolved mock per job type; a test overrides the one it drives
|
||||
const makeHandlers = (overrides?: Record<string, any>) => ({
|
||||
check: jest.fn<any>().mockResolvedValue(undefined),
|
||||
evaluate: jest.fn<any>().mockResolvedValue(undefined),
|
||||
"geo-check": jest.fn<any>().mockResolvedValue(undefined),
|
||||
"cleanup-orphaned": jest.fn<any>().mockResolvedValue(undefined),
|
||||
"cleanup-retention": jest.fn<any>().mockResolvedValue(undefined),
|
||||
egress: jest.fn<any>().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
|
||||
const jobsRepository = {
|
||||
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
|
||||
@@ -98,12 +110,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
findAllForScheduling: jest.fn<any>().mockResolvedValue([]),
|
||||
updateById: jest.fn<any>().mockResolvedValue({}),
|
||||
};
|
||||
const checksRepository = {
|
||||
findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([]),
|
||||
};
|
||||
const checkService = {
|
||||
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
|
||||
};
|
||||
const bufferService = {
|
||||
addToBuffer: jest.fn<any>(),
|
||||
addGeoCheckToBuffer: jest.fn<any>(),
|
||||
@@ -112,19 +118,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
flushGeoBuffer: jest.fn<any>().mockResolvedValue(undefined),
|
||||
shutdown: jest.fn<any>().mockResolvedValue(undefined),
|
||||
};
|
||||
const checkProducer = {
|
||||
produce: jest.fn<any>().mockResolvedValue({ status: { status: "up" }, check: { id: "c1" } }),
|
||||
};
|
||||
const checkEvaluator = {
|
||||
evaluate: jest.fn<any>().mockResolvedValue({ monitor: makeMonitor(), statusChange: { monitor: makeMonitor() }, decision: {} }),
|
||||
};
|
||||
const geoCheckPipeline = { run: jest.fn<any>().mockResolvedValue(null) };
|
||||
const dispatcher = { dispatch: jest.fn<any>().mockResolvedValue(undefined) };
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const handlers = makeHandlers();
|
||||
const queueWorkersRepository = {
|
||||
upsert: jest.fn<any>().mockResolvedValue(undefined),
|
||||
deleteById: jest.fn<any>().mockResolvedValue(undefined),
|
||||
@@ -137,14 +131,8 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
isDbConnected,
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
bufferService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
dispatcher,
|
||||
helper,
|
||||
handlers,
|
||||
queueWorkersRepository,
|
||||
logger,
|
||||
...overrides?.mocks,
|
||||
@@ -282,19 +270,18 @@ describe("DBQueueWorker", () => {
|
||||
const job = makeJob({ type: "check", refId: "m1", intervalMs: 60000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
|
||||
expect(mocks.checkProducer.produce).toHaveBeenCalled();
|
||||
// The check stage only produces; the buffer arms evaluate once the check is durably stored
|
||||
expect(mocks.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||
expect(mocks.handlers.check).toHaveBeenCalledWith(job);
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordFailure).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("records a failure (and not a success) when the job handler throws", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: failingProducer },
|
||||
mocks: {
|
||||
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
|
||||
handlers: makeHandlers({ check: jest.fn<any>().mockRejectedValue(new Error("network down")) }),
|
||||
},
|
||||
});
|
||||
|
||||
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
|
||||
@@ -310,7 +297,6 @@ describe("DBQueueWorker", () => {
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
try {
|
||||
const job = makeJob({ type: "check" });
|
||||
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
|
||||
const { mocks, worker } = await start({
|
||||
mocks: {
|
||||
jobsRepository: {
|
||||
@@ -318,7 +304,7 @@ describe("DBQueueWorker", () => {
|
||||
claimDueBatch: claimOnce(job),
|
||||
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
|
||||
},
|
||||
checkProducer: failingProducer,
|
||||
handlers: makeHandlers({ check: jest.fn<any>().mockRejectedValue(new Error("network down")) }),
|
||||
},
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1); // let the recordFailure rejection settle
|
||||
@@ -336,7 +322,7 @@ describe("DBQueueWorker", () => {
|
||||
});
|
||||
|
||||
it("swallows the failure when both the success and failure writes reject", async () => {
|
||||
const job = makeJob({ type: "check", intervalMs: 60000 }); // produce succeeds → recordSuccess path
|
||||
const job = makeJob({ type: "check", intervalMs: 60000 }); // handler succeeds → recordSuccess path
|
||||
const { mocks, worker } = await start({
|
||||
mocks: {
|
||||
jobsRepository: {
|
||||
@@ -367,7 +353,7 @@ describe("DBQueueWorker", () => {
|
||||
claimDueBatch: claimOnce(job),
|
||||
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
|
||||
},
|
||||
checkProducer: { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) },
|
||||
handlers: makeHandlers({ check: jest.fn<any>().mockRejectedValue(new Error("network down")) }),
|
||||
},
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
@@ -386,173 +372,66 @@ describe("DBQueueWorker", () => {
|
||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluate job loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
|
||||
const job = makeEvaluateJob("c1");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||
it("routes every job type to the handler registered for it, with the claimed job", async () => {
|
||||
const jobs: Job[] = [
|
||||
makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null, pendingChecks: [{ checkId: "c1", createdAt: 1000 }] }),
|
||||
makeJob({ id: "geo-check:m1", type: "geo-check", intervalMs: 300000 }),
|
||||
makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 }),
|
||||
makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 }),
|
||||
makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 }),
|
||||
];
|
||||
const handed = new Set<string>();
|
||||
const claimDueBatch = jest.fn<any>(async (type: string) => {
|
||||
const job = jobs.find((j) => j.type === type);
|
||||
if (!job || handed.has(type)) return [];
|
||||
handed.add(type);
|
||||
return [job];
|
||||
});
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch } } });
|
||||
|
||||
expect(checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks);
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
|
||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||
// The monitor cursor is gone: nothing writes lastEvaluatedAt
|
||||
expect(mocks.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
for (const job of jobs) {
|
||||
expect(mocks.handlers[job.type]).toHaveBeenCalledWith(job);
|
||||
}
|
||||
expect(mocks.handlers.check).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluate job with no pending checks does nothing", async () => {
|
||||
const job = makeEvaluateJob();
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
|
||||
expect(mocks.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
||||
});
|
||||
|
||||
it("evaluate job for a deleted monitor drops every pending id", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2");
|
||||
const monitorsRepository = { findByIdLean: jest.fn<any>().mockResolvedValue(null), updateById: jest.fn<any>() };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, monitorsRepository },
|
||||
});
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
|
||||
});
|
||||
|
||||
it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2", "c3");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) };
|
||||
const jobsRepository = {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository, checksRepository } });
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
|
||||
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" }));
|
||||
});
|
||||
|
||||
it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
|
||||
const job = makeEvaluateJob("c1", "c-gone");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||
});
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]);
|
||||
});
|
||||
|
||||
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2");
|
||||
const initialMonitor = makeMonitor();
|
||||
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
|
||||
const monitorsRepository = {
|
||||
findByIdLean: jest.fn<any>().mockResolvedValue(initialMonitor),
|
||||
updateById: jest.fn<any>().mockResolvedValue({}),
|
||||
};
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) };
|
||||
const checkEvaluator = {
|
||||
evaluate: jest.fn<any>().mockResolvedValue({ monitor: postWriteMonitor, statusChange: { monitor: postWriteMonitor }, decision: {} }),
|
||||
};
|
||||
const { mocks } = await start({
|
||||
mocks: {
|
||||
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkEvaluator,
|
||||
},
|
||||
});
|
||||
|
||||
// The monitor is read once for the whole backlog, not once per check
|
||||
expect(mocks.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
|
||||
// First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first
|
||||
expect(mocks.checkEvaluator.evaluate.mock.calls[0][2]).toBe(initialMonitor);
|
||||
expect(mocks.checkEvaluator.evaluate.mock.calls[1][2]).toBe(postWriteMonitor);
|
||||
});
|
||||
|
||||
it("cleanup-orphaned job runs the helper's cleanup function", async () => {
|
||||
const job = makeJob({ id: "cleanup-orphaned", type: "cleanup-orphaned", refId: null, intervalMs: 86400000 });
|
||||
const cleanupFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(cleanupFn),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(cleanupFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cleanup-retention job runs the helper's retention function", async () => {
|
||||
const job = makeJob({ id: "cleanup-retention", type: "cleanup-retention", refId: null, intervalMs: 86400000 });
|
||||
const retentionFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(retentionFn),
|
||||
};
|
||||
await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(retentionFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("egress job runs the helper's egress recovery function with the claimed job", async () => {
|
||||
it("reschedules an interval job after its handler succeeds", async () => {
|
||||
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
|
||||
const egressFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
});
|
||||
|
||||
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
|
||||
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.geoCheckPipeline.run).toHaveBeenCalled();
|
||||
expect(mocks.dispatcher.dispatch).not.toHaveBeenCalled();
|
||||
expect(mocks.handlers.egress).toHaveBeenCalledWith(job);
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
});
|
||||
|
||||
it("renews the lock while a slow job is still running", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const gate = deferred<void>();
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer },
|
||||
mocks: {
|
||||
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
|
||||
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
|
||||
},
|
||||
});
|
||||
|
||||
// Job is parked on produce(); push past one renewal interval.
|
||||
// Job is parked in its handler; push past one renewal interval.
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledWith([job.id], expect.any(Number));
|
||||
|
||||
// Let it finish; success is still recorded.
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
gate.resolve();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("stops renewing once the lock is lost to another worker", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const gate = deferred<void>();
|
||||
const jobsRepository = {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
renewLocks: jest.fn<any>().mockResolvedValue(0),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository, checkProducer: slowProducer } });
|
||||
const { mocks } = await start({ mocks: { jobsRepository, handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }) } });
|
||||
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1); // lease lost → renewal timer stops
|
||||
@@ -562,7 +441,7 @@ describe("DBQueueWorker", () => {
|
||||
await jest.advanceTimersByTimeAsync(LOCK_RENEW_MS);
|
||||
expect(mocks.jobsRepository.renewLocks).toHaveBeenCalledTimes(1);
|
||||
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
gate.resolve();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
});
|
||||
});
|
||||
@@ -683,11 +562,8 @@ describe("DBQueueWorker", () => {
|
||||
}
|
||||
return [];
|
||||
});
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>(() => gate.promise)), // blocks → keeps the slot full
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, helper } });
|
||||
const handlers = makeHandlers({ "cleanup-orphaned": jest.fn<any>(() => gate.promise) }); // blocks → keeps the slot full
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch }, handlers } });
|
||||
|
||||
const before = claims(mocks, "cleanup-orphaned"); // the blocking job is now in-flight → capacity 0
|
||||
await jest.advanceTimersByTimeAsync(1000);
|
||||
@@ -804,19 +680,21 @@ describe("DBQueueWorker", () => {
|
||||
|
||||
it("waits for an in-flight job to finish before flushing", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const gate = deferred<void>();
|
||||
const { worker, mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: slowProducer },
|
||||
mocks: {
|
||||
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
|
||||
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
|
||||
},
|
||||
});
|
||||
|
||||
// Job is parked on produce(); drain must not flush while it is still in flight.
|
||||
// Job is parked in its handler; drain must not flush while it is still in flight.
|
||||
const draining = worker.drain();
|
||||
await jest.advanceTimersByTimeAsync(DRAIN_POLL_MS * 3);
|
||||
expect(mocks.bufferService.shutdown).not.toHaveBeenCalled();
|
||||
|
||||
// Let the job finish; drain observes inFlight == 0, then flushes.
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
gate.resolve();
|
||||
await jest.advanceTimersByTimeAsync(DRAIN_POLL_MS);
|
||||
await draining;
|
||||
expect(mocks.bufferService.shutdown).toHaveBeenCalledTimes(1);
|
||||
@@ -824,10 +702,12 @@ describe("DBQueueWorker", () => {
|
||||
|
||||
it("times out after DRAIN_TIMEOUT_MS, warns, and still flushes", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const stuckProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const gate = deferred<void>();
|
||||
const { worker, mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checkProducer: stuckProducer },
|
||||
mocks: {
|
||||
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) },
|
||||
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
|
||||
},
|
||||
});
|
||||
|
||||
const draining = worker.drain();
|
||||
@@ -837,7 +717,7 @@ describe("DBQueueWorker", () => {
|
||||
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("timed out") }));
|
||||
expect(mocks.bufferService.shutdown).toHaveBeenCalledTimes(1);
|
||||
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } }); // release the stuck job for cleanup
|
||||
gate.resolve(); // release the stuck job for cleanup
|
||||
});
|
||||
});
|
||||
|
||||
@@ -892,15 +772,17 @@ describe("DBQueueWorker", () => {
|
||||
|
||||
it("counts in-flight jobs while they run and releases on completion", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const gate = deferred<{ status: unknown; check: unknown }>();
|
||||
const slowProducer = { produce: jest.fn<any>().mockReturnValue(gate.promise) };
|
||||
const gate = deferred<void>();
|
||||
const { worker } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOneCheck(job) }, checkProducer: slowProducer },
|
||||
mocks: {
|
||||
jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOneCheck(job) },
|
||||
handlers: makeHandlers({ check: jest.fn<any>().mockReturnValue(gate.promise) }),
|
||||
},
|
||||
});
|
||||
|
||||
expect(worker.getHealth().inFlight).toBe(1); // parked on produce()
|
||||
expect(worker.getHealth().inFlight).toBe(1); // parked in its handler
|
||||
|
||||
gate.resolve({ status: { status: "up" }, check: { id: "c1" } });
|
||||
gate.resolve();
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(worker.getHealth().inFlight).toBe(0);
|
||||
});
|
||||
|
||||
+53
-1
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { GeoChecksPipeline } from "../../../src/worker/worker.check-pipeline.ts";
|
||||
import { GeoChecksPipeline } from "../../../src/worker/worker.geo-pipeline.ts";
|
||||
import type { Job } from "../../../src/domain/jobs/job.type.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
@@ -25,12 +26,14 @@ const activeWindow = () => {
|
||||
const createGeoPipeline = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(null) },
|
||||
geoChecksService: { buildGeoCheck: jest.fn().mockResolvedValue(null) },
|
||||
buffer: { addGeoCheckToBuffer: jest.fn() },
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) },
|
||||
...overrides,
|
||||
};
|
||||
const pipeline = new GeoChecksPipeline(
|
||||
defaults.monitorsRepository as any,
|
||||
defaults.maintenanceWindowsRepository as any,
|
||||
defaults.geoChecksService as any,
|
||||
defaults.buffer as any,
|
||||
@@ -42,7 +45,56 @@ const createGeoPipeline = (overrides?: Record<string, any>) => {
|
||||
const geoMonitor = (overrides?: Partial<Monitor>) =>
|
||||
makeMonitor({ geoCheckEnabled: true, type: "http", geoCheckLocations: ["us-east"], ...overrides });
|
||||
|
||||
const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
id: "geo-check:m1",
|
||||
type: "geo-check",
|
||||
refId: "m1",
|
||||
isActive: true,
|
||||
nextScheduledAt: 1000,
|
||||
intervalMs: 300000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
pendingChecks: [],
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("GeoChecksPipeline", () => {
|
||||
describe("handle", () => {
|
||||
it("loads the monitor by refId and runs the pipeline for it", async () => {
|
||||
const geoCheck = { id: "gc-1", monitorId: "m1" };
|
||||
const { pipeline, defaults } = createGeoPipeline({
|
||||
monitorsRepository: { findByIdLean: jest.fn().mockResolvedValue(geoMonitor()) },
|
||||
geoChecksService: { buildGeoCheck: jest.fn().mockResolvedValue(geoCheck) },
|
||||
});
|
||||
|
||||
await pipeline.handle(makeJob());
|
||||
|
||||
expect(defaults.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
|
||||
expect(defaults.buffer.addGeoCheckToBuffer).toHaveBeenCalledWith(geoCheck);
|
||||
});
|
||||
|
||||
it("does nothing for a job without a refId", async () => {
|
||||
const { pipeline, defaults } = createGeoPipeline();
|
||||
|
||||
await pipeline.handle(makeJob({ refId: null }));
|
||||
|
||||
expect(defaults.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
|
||||
expect(defaults.geoChecksService.buildGeoCheck).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does nothing when the monitor no longer exists", async () => {
|
||||
const { pipeline, defaults } = createGeoPipeline();
|
||||
|
||||
await pipeline.handle(makeJob());
|
||||
|
||||
expect(defaults.geoChecksService.buildGeoCheck).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("throws when monitor id is missing", async () => {
|
||||
const { pipeline } = createGeoPipeline();
|
||||
await expect(pipeline.run({} as Monitor)).rejects.toThrow("No monitor id");
|
||||
@@ -7,7 +7,7 @@ import type { IUsersRepository } from "../../../src/domain/users/user.repository
|
||||
import type { INotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { Incident } from "../../../src/domain/incidents/incident.type.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { MonitorStatusPolicy } from "../../../src/worker/worker.monitor-status-policy.ts";
|
||||
import type { StatusChangeResult } from "../../../src/types/network.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const makeStatusChange = (overrides: Partial<StatusChangeResult> & { status?: string }): StatusChangeResult => {
|
||||
const { status = "up", ...rest } = overrides;
|
||||
return {
|
||||
monitor: { id: "m1", status } as any,
|
||||
statusChanged: false,
|
||||
prevStatus: "up",
|
||||
code: 200,
|
||||
timestamp: 0,
|
||||
...rest,
|
||||
} as StatusChangeResult;
|
||||
};
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MonitorStatusPolicy", () => {
|
||||
const policy = new MonitorStatusPolicy();
|
||||
|
||||
it("does nothing when statusChanged is false", () => {
|
||||
const decision = policy.evaluate(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "up" }));
|
||||
expect(decision).toEqual({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("creates incident and notifies when monitor goes down", () => {
|
||||
const decision = policy.evaluate(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 }));
|
||||
expect(decision).toMatchObject({
|
||||
shouldCreateIncident: true,
|
||||
shouldSendNotification: true,
|
||||
incidentReason: "status_down",
|
||||
notificationReason: "status_change",
|
||||
});
|
||||
});
|
||||
|
||||
it("creates incident and notifies when monitor is breached", () => {
|
||||
const decision = policy.evaluate(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" }));
|
||||
expect(decision).toMatchObject({
|
||||
shouldCreateIncident: true,
|
||||
shouldSendNotification: true,
|
||||
incidentReason: "threshold_breach",
|
||||
notificationReason: "threshold_breach",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves incident when monitor recovers from down", () => {
|
||||
const decision = policy.evaluate(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "down" }));
|
||||
expect(decision).toMatchObject({
|
||||
shouldResolveIncident: true,
|
||||
shouldSendNotification: true,
|
||||
shouldCreateIncident: false,
|
||||
notificationReason: "status_change",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves incident when monitor recovers from breached", () => {
|
||||
const decision = policy.evaluate(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" }));
|
||||
expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
|
||||
});
|
||||
|
||||
it("does not create or resolve for unhandled status transitions", () => {
|
||||
const decision = policy.evaluate(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" }));
|
||||
expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false });
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { NotificationMessageBuilder } from "../../../src/domain/notifications/notification.message-builder.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse, HardwareStatusPayload } from "../../../src/types/network.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { Notification } from "../../../src/domain/notifications/notification.type.ts";
|
||||
import type { MonitorStatusResponse } from "../../../src/types/network.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.helper.ts";
|
||||
import type { MonitorActionDecision } from "../../../src/worker/worker.interface.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -0,0 +1,874 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { WorkerPipeline } from "../../../src/worker/worker.pipeline.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
import type { Job } from "../../../src/domain/jobs/job.type.ts";
|
||||
import type { Check } from "../../../src/domain/checks/check.type.ts";
|
||||
import type { StatusChangeResult } from "../../../src/types/network.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
import { NETWORK_ERROR } from "../../../src/types/network.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
|
||||
({
|
||||
id: "m1",
|
||||
teamId: "team",
|
||||
type: "http",
|
||||
interval: 60000,
|
||||
status: "up",
|
||||
...overrides,
|
||||
}) as Monitor;
|
||||
|
||||
const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
id: "check:m1",
|
||||
type: "check",
|
||||
refId: "m1",
|
||||
isActive: true,
|
||||
nextScheduledAt: 1000,
|
||||
intervalMs: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
pendingChecks: [],
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// An evaluate row as ingest leaves it: one pending entry per stored check
|
||||
const makeEvaluateJob = (...checkIds: string[]): Job =>
|
||||
makeJob({
|
||||
id: "evaluate:m1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })),
|
||||
});
|
||||
|
||||
const makeCheck = (overrides?: Partial<Check>): Check =>
|
||||
({
|
||||
id: "check-1",
|
||||
metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" },
|
||||
status: true,
|
||||
statusCode: 200,
|
||||
responseTime: 100,
|
||||
message: "OK",
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
...overrides,
|
||||
}) as Check;
|
||||
|
||||
const makeStatusChange = (overrides: Partial<StatusChangeResult> & { status?: string }): StatusChangeResult => {
|
||||
const { status = "up", ...rest } = overrides;
|
||||
return {
|
||||
monitor: { id: "m1", status } as any,
|
||||
statusChanged: false,
|
||||
prevStatus: "up",
|
||||
code: 200,
|
||||
timestamp: 0,
|
||||
...rest,
|
||||
} as StatusChangeResult;
|
||||
};
|
||||
|
||||
// A maintenance window straddling "now" — active per isWindowActive.
|
||||
const activeWindow = () => {
|
||||
const now = Date.now();
|
||||
return { active: true, start: new Date(now - 1000).toISOString(), end: new Date(now + 1000).toISOString(), repeat: 0 };
|
||||
};
|
||||
|
||||
const createPipeline = (overrides?: Record<string, any>) => {
|
||||
const deps = {
|
||||
logger: createMockLogger(),
|
||||
monitorsRepository: {
|
||||
findByIdLean: jest.fn<any>().mockResolvedValue(makeMonitor()),
|
||||
updateById: jest.fn<any>().mockResolvedValue({}),
|
||||
findDockerTlsKeyById: jest.fn<any>().mockResolvedValue(null),
|
||||
},
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([]) },
|
||||
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([]) },
|
||||
jobsRepository: {
|
||||
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
|
||||
pullEvaluated: jest.fn<any>().mockResolvedValue(true),
|
||||
},
|
||||
checkService: {
|
||||
toCheck: jest.fn<any>().mockReturnValue({ id: "check-1" }),
|
||||
toStatusResponse: jest.fn<any>().mockReturnValue({ monitorId: "m1", status: true, code: 200, message: "OK" }),
|
||||
createChecks: jest.fn<any>().mockResolvedValue([]),
|
||||
},
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
|
||||
proxyResolver: { resolve: jest.fn<any>().mockResolvedValue(undefined) },
|
||||
bufferService: { addToBuffer: jest.fn<any>(), addDockerLogToBuffer: jest.fn<any>() },
|
||||
dockerLogsService: { buildDockerLogs: jest.fn<any>().mockResolvedValue([]) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue(null) },
|
||||
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(makeStatusChange({})) },
|
||||
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
|
||||
...overrides,
|
||||
};
|
||||
const pipeline = new WorkerPipeline(deps as any);
|
||||
return { pipeline, deps };
|
||||
};
|
||||
|
||||
// ── Stage 1: check job ───────────────────────────────────────────────────────
|
||||
|
||||
describe("WorkerPipeline", () => {
|
||||
describe("handleCheck", () => {
|
||||
it("loads the monitor by refId and produces a check for it", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.handleCheck(makeJob());
|
||||
|
||||
expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledWith("m1");
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalled();
|
||||
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith({ id: "check-1" });
|
||||
});
|
||||
|
||||
it("does nothing for a job without a refId", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.handleCheck(makeJob({ refId: null }));
|
||||
|
||||
expect(deps.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
|
||||
expect(deps.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does nothing when the monitor no longer exists", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
monitorsRepository: { findByIdLean: jest.fn<any>().mockResolvedValue(null) },
|
||||
});
|
||||
|
||||
await pipeline.handleCheck(makeJob());
|
||||
|
||||
expect(deps.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("produce", () => {
|
||||
it("throws when monitor id is missing", async () => {
|
||||
const { pipeline } = createPipeline();
|
||||
await expect(pipeline.produce({} as Monitor)).rejects.toThrow("No monitor id");
|
||||
});
|
||||
|
||||
// ── maintenance gate ──────────────────────────────────────────────────
|
||||
|
||||
it("skips the check and flips status to 'maintenance' when in a maintenance window", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
const result = await pipeline.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
|
||||
expect(deps.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(deps.bufferService.addToBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-write status when already in maintenance", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([activeWindow()]) },
|
||||
});
|
||||
|
||||
const result = await pipeline.produce(makeMonitor({ status: "maintenance" }));
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("moves a monitor out of maintenance explicitly once the window is over", async () => {
|
||||
const monitor = makeMonitor({ status: "maintenance" });
|
||||
const { pipeline, deps } = createPipeline({
|
||||
monitorsRepository: {
|
||||
updateById: jest.fn<any>().mockResolvedValue({ ...monitor, status: "initializing" }),
|
||||
findDockerTlsKeyById: jest.fn<any>().mockResolvedValue(null),
|
||||
},
|
||||
});
|
||||
|
||||
await pipeline.produce(monitor);
|
||||
|
||||
// The exit used to be a side effect of evaluating the resulting check, which the degraded-egress
|
||||
// short-circuit skips. Left implicit, a monitor could sit in "maintenance" for a whole egress outage.
|
||||
expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "initializing" });
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not touch the status of a monitor that was not in maintenance", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.produce(makeMonitor({ status: "up" }));
|
||||
|
||||
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("proceeds with the check when the maintenance window is inactive", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([{ ...activeWindow(), active: false }]) },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── proxy resolution ──────────────────────────────────────────────────
|
||||
|
||||
it("passes the resolved proxy url into requestStatus", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
proxyResolver: { resolve: jest.fn<any>().mockResolvedValue("http://proxy.example.com:8080") },
|
||||
});
|
||||
const monitor = makeMonitor();
|
||||
|
||||
await pipeline.produce(monitor);
|
||||
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: "http://proxy.example.com:8080" });
|
||||
});
|
||||
|
||||
it("still produces a check when the resolver returns undefined", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
const monitor = makeMonitor();
|
||||
|
||||
const result = await pipeline.produce(monitor);
|
||||
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined });
|
||||
expect(result).toEqual({ status: expect.objectContaining({ monitorId: "m1" }), check: { id: "check-1" } });
|
||||
});
|
||||
|
||||
// ── docker tls key resolution ─────────────────────────────────────────
|
||||
|
||||
it("fetches the stored key for a docker monitor with a key set and passes it in the context", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
monitorsRepository: { updateById: jest.fn<any>(), findDockerTlsKeyById: jest.fn<any>().mockResolvedValue("v1.abc123.iv.tag.data") },
|
||||
});
|
||||
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
|
||||
|
||||
await pipeline.produce(monitor);
|
||||
|
||||
expect(deps.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: "v1.abc123.iv.tag.data" });
|
||||
});
|
||||
|
||||
it("does not fetch a key for a docker monitor without one set", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKeySet: false });
|
||||
|
||||
await pipeline.produce(monitor);
|
||||
|
||||
expect(deps.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
|
||||
});
|
||||
|
||||
it("does not fetch a key for non-docker monitors even if the flag is set", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.produce(makeMonitor({ type: "http", dockerTlsKeySet: true }));
|
||||
|
||||
expect(deps.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes undefined when the flag is set but the repository has no key", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true });
|
||||
|
||||
await pipeline.produce(monitor);
|
||||
|
||||
expect(deps.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1");
|
||||
expect(deps.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined });
|
||||
});
|
||||
|
||||
// ── acquire / record ──────────────────────────────────────────────────
|
||||
|
||||
it("throws when the network response is null", async () => {
|
||||
const { pipeline } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(null) },
|
||||
});
|
||||
|
||||
await expect(pipeline.produce(makeMonitor())).rejects.toThrow("No network response");
|
||||
});
|
||||
|
||||
it("returns null, warns, and does not buffer when toCheck yields nothing", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(null) },
|
||||
});
|
||||
|
||||
const result = await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(deps.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("No check could be built") }));
|
||||
expect(deps.bufferService.addToBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("buffers the built check and returns the status and check on success", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" };
|
||||
const check = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
const result = await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(result).toEqual({ status, check });
|
||||
});
|
||||
|
||||
it("builds and buffers docker logs for a docker status", async () => {
|
||||
const status = { type: "docker", monitorId: "m1", teamId: "team", status: true, code: 200, message: "OK", payload: {} };
|
||||
const dockerLogs = [{ id: "docker-log-1" }, { id: "docker-log-2" }];
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(status) },
|
||||
dockerLogsService: { buildDockerLogs: jest.fn<any>().mockResolvedValue(dockerLogs) },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor({ type: "docker" }));
|
||||
|
||||
expect(deps.dockerLogsService.buildDockerLogs).toHaveBeenCalledWith(status);
|
||||
expect(deps.bufferService.addDockerLogToBuffer).toHaveBeenNthCalledWith(1, dockerLogs[0]);
|
||||
expect(deps.bufferService.addDockerLogToBuffer).toHaveBeenNthCalledWith(2, dockerLogs[1]);
|
||||
});
|
||||
|
||||
it("does not build docker logs for a non-docker status", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── egress self-check ─────────────────────────────────────────────────
|
||||
|
||||
// A transport failure: nothing came back from the target, so the code is the network-error sentinel.
|
||||
const failingStatus = { monitorId: "m1", status: false, code: NETWORK_ERROR, message: "Timeout" };
|
||||
|
||||
it("does not consult the egress service for a successful check and leaves the field unset", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("does not consult egress when an HTTP response was received", async () => {
|
||||
for (const status of [
|
||||
{ monitorId: "m1", status: false, code: 500, message: "Internal Server Error" },
|
||||
{ monitorId: "m1", status: false, code: 200, message: "Content mismatch" },
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(status) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("consults egress for a timeout or network error", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("never consults egress for a docker daemon on a local socket", async () => {
|
||||
for (const url of ["unix:///var/run/docker.sock", "/var/run/docker.sock"]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, type: "docker" }) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor({ type: "docker", url }));
|
||||
|
||||
// Local IPC never leaves the instance, so it cannot fail through egress.
|
||||
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
|
||||
}
|
||||
});
|
||||
|
||||
it("consults egress for a hardware check and for a docker daemon reached over TCP", async () => {
|
||||
for (const monitor of [
|
||||
makeMonitor({ type: "hardware", url: "http://capture.example.com:59232/api/v1/metrics" }),
|
||||
makeMonitor({ type: "docker", url: "tcp://docker.example.com:2376" }),
|
||||
]) {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, type: monitor.type }) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(monitor);
|
||||
|
||||
// Both requests leave the instance, so our own loss of egress can be the cause.
|
||||
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not consult egress when the provider reports that the peer answered", async () => {
|
||||
// gRPC NOT_SERVING and an authoritative DNS NXDOMAIN both carry NETWORK_ERROR, so only this flag separates
|
||||
// them from a connection failure.
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, peerResponded: true }) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor({ type: "dns" }));
|
||||
|
||||
expect(deps.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
|
||||
it("consults egress when the provider reports that nothing answered, despite an HTTP-shaped code", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue({ ...failingStatus, code: 503, peerResponded: false }) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor({ type: "grpc" }));
|
||||
|
||||
// An explicit report from the provider outranks the HTTP-status heuristic.
|
||||
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
it("flags a failing check as degraded when the egress service reports degraded egress", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
const result = await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check.egressStatus).toBe("degraded");
|
||||
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(expect.objectContaining({ egressStatus: "degraded" }));
|
||||
expect(result?.check.egressStatus).toBe("degraded");
|
||||
});
|
||||
|
||||
// ── status window across a spell ─────────────────────────────────────
|
||||
// Matching the maintenance gate: cleared once as the monitor stops being evaluated, so the results either
|
||||
// side of a spell are never adjacent.
|
||||
|
||||
it("clears the status window on the first degraded check of a spell", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(deps.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { statusWindow: [] });
|
||||
});
|
||||
|
||||
it("does not re-write the status window for later degraded checks in the same spell", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("degraded") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor({ statusWindow: [] }));
|
||||
|
||||
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves the status window alone when the probe found egress fine", async () => {
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor({ statusWindow: [true, true, true, false, false] }));
|
||||
|
||||
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("flags a failing check as ok when the probe found egress fine", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockResolvedValue("ok") },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(check.egressStatus).toBe("ok");
|
||||
});
|
||||
|
||||
it("buffers the check when the egress assessment rejects", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
egressService: { assessAfterFailure: jest.fn<any>().mockRejectedValue(new Error("settings unavailable")) },
|
||||
});
|
||||
|
||||
const result = await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.bufferService.addToBuffer).toHaveBeenCalledWith(check);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
expect(result).toEqual({ status: failingStatus, check });
|
||||
expect(deps.logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("settings unavailable"), service: "WorkerPipeline", method: "assessEgress" })
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves the field unset on a failing check when the egress check is disabled", async () => {
|
||||
const check: Record<string, unknown> = { id: "check-1" };
|
||||
const { pipeline, deps } = createPipeline({
|
||||
networkService: { requestStatus: jest.fn<any>().mockResolvedValue(failingStatus) },
|
||||
checkService: { toCheck: jest.fn<any>().mockReturnValue(check) },
|
||||
});
|
||||
|
||||
await pipeline.produce(makeMonitor());
|
||||
|
||||
expect(deps.egressService.assessAfterFailure).toHaveBeenCalledTimes(1);
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Stage 2: ingestion ───────────────────────────────────────────────────
|
||||
|
||||
describe("ingestChecks", () => {
|
||||
it("stores the batch, then arms the evaluate stage once per distinct monitor", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
const batch = [
|
||||
makeCheck({ id: "c1", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }),
|
||||
makeCheck({ id: "c2", metadata: { monitorId: "mon-1", teamId: "team-1", type: "http" } }),
|
||||
makeCheck({ id: "c3", metadata: { monitorId: "mon-2", teamId: "team-1", type: "http" } }),
|
||||
];
|
||||
|
||||
await pipeline.ingestChecks(batch);
|
||||
|
||||
const createdAt = Date.parse("2026-01-01T00:00:00.000Z");
|
||||
expect(deps.checkService.createChecks).toHaveBeenCalledWith(batch);
|
||||
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledWith(
|
||||
"mon-1",
|
||||
[
|
||||
{ checkId: "c1", createdAt },
|
||||
{ checkId: "c2", createdAt },
|
||||
],
|
||||
expect.any(Number)
|
||||
);
|
||||
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
|
||||
});
|
||||
|
||||
it("retries a failed arm on the next ingest without re-storing the checks", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
deps.jobsRepository.upsertEvaluate.mockRejectedValueOnce(new Error("arm failed"));
|
||||
|
||||
await pipeline.ingestChecks([makeCheck({ id: "c1" })]);
|
||||
expect(deps.checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(deps.logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
|
||||
|
||||
await pipeline.ingestChecks([]); // nothing new, only the retry runs
|
||||
expect(deps.checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith(
|
||||
"mon-1",
|
||||
[{ checkId: "c1", createdAt: expect.any(Number) }],
|
||||
expect.any(Number)
|
||||
);
|
||||
|
||||
await pipeline.ingestChecks([]); // nothing left to retry
|
||||
expect(deps.jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("neither writes nor arms for an empty batch", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.ingestChecks([]);
|
||||
|
||||
expect(deps.checkService.createChecks).not.toHaveBeenCalled();
|
||||
expect(deps.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not arm the evaluate stage when the check write fails", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
deps.checkService.createChecks.mockRejectedValue(new Error("DB write failed"));
|
||||
|
||||
await expect(pipeline.ingestChecks([makeCheck()])).rejects.toThrow("DB write failed");
|
||||
|
||||
expect(deps.jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Stage 3: evaluation ──────────────────────────────────────────────────
|
||||
|
||||
describe("handleEvaluate", () => {
|
||||
it("loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
|
||||
const job = makeEvaluateJob("c1");
|
||||
const { pipeline, deps } = createPipeline({
|
||||
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) },
|
||||
});
|
||||
|
||||
await pipeline.handleEvaluate(job);
|
||||
|
||||
expect(deps.checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks);
|
||||
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(1);
|
||||
expect(deps.dispatcher.dispatch).toHaveBeenCalledTimes(1);
|
||||
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||
// The monitor cursor is gone: nothing writes lastEvaluatedAt
|
||||
expect(deps.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does nothing when the row has no pending checks", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.handleEvaluate(makeEvaluateJob());
|
||||
|
||||
expect(deps.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
|
||||
expect(deps.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("drops every pending id for a deleted monitor", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2");
|
||||
const { pipeline, deps } = createPipeline({
|
||||
monitorsRepository: { findByIdLean: jest.fn<any>().mockResolvedValue(null) },
|
||||
});
|
||||
|
||||
await pipeline.handleEvaluate(job);
|
||||
|
||||
expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled();
|
||||
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
|
||||
});
|
||||
|
||||
it("stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2", "c3");
|
||||
const { pipeline, deps } = createPipeline({
|
||||
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) },
|
||||
jobsRepository: {
|
||||
upsertEvaluate: jest.fn<any>(),
|
||||
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
|
||||
},
|
||||
});
|
||||
|
||||
await pipeline.handleEvaluate(job);
|
||||
|
||||
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2);
|
||||
expect(deps.dispatcher.dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
|
||||
expect(deps.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Lost lease") }));
|
||||
});
|
||||
|
||||
it("pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
|
||||
const job = makeEvaluateJob("c1", "c-gone");
|
||||
const { pipeline, deps } = createPipeline({
|
||||
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }, // c-gone was cleaned up
|
||||
});
|
||||
|
||||
await pipeline.handleEvaluate(job);
|
||||
|
||||
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(1);
|
||||
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||
expect(deps.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]);
|
||||
});
|
||||
|
||||
it("reads the monitor once and threads the post-write monitor across the backlog", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2");
|
||||
const initialMonitor = makeMonitor();
|
||||
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
|
||||
const { pipeline, deps } = createPipeline({
|
||||
monitorsRepository: { findByIdLean: jest.fn<any>().mockResolvedValue(initialMonitor), updateById: jest.fn<any>() },
|
||||
checksRepository: { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }]) },
|
||||
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(makeStatusChange({ monitor: postWriteMonitor })) },
|
||||
});
|
||||
|
||||
await pipeline.handleEvaluate(job);
|
||||
|
||||
// The monitor is read once for the whole backlog, not once per check
|
||||
expect(deps.monitorsRepository.findByIdLean).toHaveBeenCalledTimes(1);
|
||||
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledTimes(2);
|
||||
// First check evaluates against the freshly-read monitor; the second against the post-write monitor from the first
|
||||
expect(deps.statusService.updateMonitorStatus.mock.calls[0][2]).toBe(initialMonitor);
|
||||
expect(deps.statusService.updateMonitorStatus.mock.calls[1][2]).toBe(postWriteMonitor);
|
||||
});
|
||||
});
|
||||
|
||||
describe("evaluateCheck", () => {
|
||||
it("threads status, check, and monitor through to the status service", async () => {
|
||||
const { pipeline, deps } = createPipeline();
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const check = { id: "check-1" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
|
||||
await pipeline.evaluateCheck(status, check, monitor);
|
||||
|
||||
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, monitor);
|
||||
});
|
||||
|
||||
it("returns an evaluation that opens an incident on a down transition", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1" } as any;
|
||||
const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "down" }), statusChanged: true, prevStatus: "up", code: 500 });
|
||||
const { pipeline } = createPipeline({
|
||||
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
|
||||
});
|
||||
|
||||
const result = await pipeline.evaluateCheck(status, check, makeMonitor());
|
||||
|
||||
expect(result).toMatchObject({
|
||||
monitor: statusChange.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange,
|
||||
decision: expect.objectContaining({
|
||||
shouldCreateIncident: true,
|
||||
shouldSendNotification: true,
|
||||
incidentReason: "status_down",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("returns an evaluation that resolves an incident on recovery", async () => {
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const statusChange = makeStatusChange({ monitor: makeMonitor({ status: "up" }), statusChanged: true, prevStatus: "down" });
|
||||
const { pipeline } = createPipeline({
|
||||
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
|
||||
});
|
||||
|
||||
const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor({ status: "down" }));
|
||||
|
||||
expect(result.decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
|
||||
});
|
||||
|
||||
it("produces a no-op decision when status did not change", async () => {
|
||||
const status = { monitorId: "m1", status: true, code: 200, message: "OK" } as any;
|
||||
const { pipeline } = createPipeline();
|
||||
|
||||
const result = await pipeline.evaluateCheck(status, { id: "check-1" } as any, makeMonitor());
|
||||
|
||||
expect(result.decision).toMatchObject({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
});
|
||||
});
|
||||
|
||||
// ── degraded egress ───────────────────────────────────────────────────
|
||||
|
||||
it("short-circuits a check flagged with degraded egress without touching monitor status", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "degraded" } as any;
|
||||
const monitor = makeMonitor({ status: "up" });
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
const result = await pipeline.evaluateCheck(status, check, monitor);
|
||||
|
||||
expect(deps.statusService.updateMonitorStatus).not.toHaveBeenCalled();
|
||||
expect(result.monitor).toBe(monitor);
|
||||
expect(result.statusChange).toMatchObject({ monitor, statusChanged: false, prevStatus: "up", code: 500 });
|
||||
expect(result.statusChange.timestamp).toEqual(expect.any(Number));
|
||||
expect(result.decision).toEqual({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
});
|
||||
expect(deps.logger.debug).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress was degraded") }));
|
||||
});
|
||||
|
||||
it("evaluates a check flagged with egress ok as normal", async () => {
|
||||
const status = { monitorId: "m1", status: false, code: 500, message: "Error" } as any;
|
||||
const check = { id: "check-1", egressStatus: "ok" } as any;
|
||||
const { pipeline, deps } = createPipeline();
|
||||
|
||||
await pipeline.evaluateCheck(status, check, makeMonitor());
|
||||
|
||||
expect(deps.statusService.updateMonitorStatus).toHaveBeenCalledWith(status, check, expect.anything());
|
||||
});
|
||||
});
|
||||
|
||||
// The decision table, driven through evaluateCheck with the status service stubbed to return each transition.
|
||||
|
||||
describe("decision", () => {
|
||||
const decide = async (statusChange: StatusChangeResult) => {
|
||||
const { pipeline } = createPipeline({
|
||||
statusService: { updateMonitorStatus: jest.fn<any>().mockResolvedValue(statusChange) },
|
||||
});
|
||||
const result = await pipeline.evaluateCheck(
|
||||
{ monitorId: "m1", status: true, code: 200, message: "OK" } as any,
|
||||
{ id: "check-1" } as any,
|
||||
makeMonitor()
|
||||
);
|
||||
return result.decision;
|
||||
};
|
||||
|
||||
it("does nothing when statusChanged is false", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "down", statusChanged: false, prevStatus: "up" }));
|
||||
expect(decision).toEqual({
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("creates incident and notifies when monitor goes down", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "down", statusChanged: true, prevStatus: "up", code: 500 }));
|
||||
expect(decision).toMatchObject({
|
||||
shouldCreateIncident: true,
|
||||
shouldSendNotification: true,
|
||||
incidentReason: "status_down",
|
||||
notificationReason: "status_change",
|
||||
});
|
||||
});
|
||||
|
||||
it("creates incident and notifies when monitor is breached", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "breached", statusChanged: true, prevStatus: "up" }));
|
||||
expect(decision).toMatchObject({
|
||||
shouldCreateIncident: true,
|
||||
shouldSendNotification: true,
|
||||
incidentReason: "threshold_breach",
|
||||
notificationReason: "threshold_breach",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves incident when monitor recovers from down", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "down" }));
|
||||
expect(decision).toMatchObject({
|
||||
shouldResolveIncident: true,
|
||||
shouldSendNotification: true,
|
||||
shouldCreateIncident: false,
|
||||
notificationReason: "status_change",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves incident when monitor recovers from breached", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "up", statusChanged: true, prevStatus: "breached" }));
|
||||
expect(decision).toMatchObject({ shouldResolveIncident: true, shouldSendNotification: true });
|
||||
});
|
||||
|
||||
it("does not create or resolve for unhandled status transitions", async () => {
|
||||
const decision = await decide(makeStatusChange({ status: "paused", statusChanged: true, prevStatus: "up" }));
|
||||
expect(decision).toMatchObject({ shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false });
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user