From f43bf9ab3d9a6914730c56d0808a1029c78f25f5 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 25 Jun 2026 11:02:38 -0700 Subject: [PATCH] add observability to worker and health server --- server/src/worker/worker.db-queue.ts | 13 +++++++++ server/src/worker/worker.health-server.ts | 33 ++++++++++++++++++++++- server/src/worker/worker.interface.ts | 2 ++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index e0f168eb1..b7f971fd7 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -332,6 +332,10 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { await this.bufferService.shutdown(); // Flush buffers }; + // ************************ + // Observability + // ************************ + getHealth = () => { return { workerId: this.workerId, @@ -343,4 +347,13 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { inFlight: this.getInFlightCount(), }; }; + + countDueBacklog = () => { + return this.jobsRepository.countDueBacklog(Date.now()); + }; + + countAliveWorkers = async () => { + const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS); + return workers.length; + }; } diff --git a/server/src/worker/worker.health-server.ts b/server/src/worker/worker.health-server.ts index 55727f734..74aa9b786 100644 --- a/server/src/worker/worker.health-server.ts +++ b/server/src/worker/worker.health-server.ts @@ -4,6 +4,8 @@ import { ILogger } from "@/utils/logger.js"; const SERVICE_NAME = "HealthServer"; const LIVENESS_STALE_MS = 30_000; // 30s ~6x POLL_MAX_MS so we don't restart workers +const CONTENT_TYPE_METRICS = "text/plain; version=0.0.4; charset=utf-8"; +const gauge = (name: string, help: string, value: number): string => `# HELP ${name} ${help}\n# TYPE ${name} gauge\n${name} ${value}\n`; export interface IHealthServer { listen(): Promise; @@ -37,8 +39,37 @@ export class HealthServer implements IHealthServer { this.server = http.createServer(this.handle); } - private handle = (req: http.IncomingMessage, res: http.ServerResponse) => { + private metrics = async (): Promise => { + const health = this.worker.getHealth(); + const [dueBacklog, aliveWorkers] = await Promise.all([this.worker.countDueBacklog(), this.worker.countAliveWorkers()]); + return ( + gauge("checkmate_worker_jobs_in_flight", "Jobs currently being processed by this worker", health.inFlight) + + gauge("checkmate_worker_due_backlog", "Due check jobs not currently locked (cluster-wide)", dueBacklog) + + gauge("checkmate_worker_alive_total", "Queue members seen within the TTL window (cluster-wide)", aliveWorkers) + + gauge("checkmate_worker_draining", "1 if this worker is draining, else 0", health.draining ? 1 : 0) + ); + }; + + private handle = async (req: http.IncomingMessage, res: http.ServerResponse) => { const path = (req.url ?? "").split("?")[0]; // Strip query params + + // Metrics + if (path === "/metrics") { + try { + const body = await this.metrics(); + res.writeHead(200, { "Content-Type": CONTENT_TYPE_METRICS }); + res.end(body); + } catch (error: unknown) { + this.logger.warn({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "metrics", + }); + res.writeHead(500).end(); + } + return; + } + if (path !== "/livez" && path !== "/readyz") { res.writeHead(404).end(); return; diff --git a/server/src/worker/worker.interface.ts b/server/src/worker/worker.interface.ts index 6f7c57d07..d418e0263 100644 --- a/server/src/worker/worker.interface.ts +++ b/server/src/worker/worker.interface.ts @@ -85,4 +85,6 @@ export interface IQueueWorker extends IJobScheduler { readonly serviceName: string; init(): Promise; getHealth(): WorkerHealth; + countDueBacklog(): Promise; + countAliveWorkers(): Promise; }