diff --git a/server/src/shutdown.ts b/server/src/shutdown.ts index a25d96c0e..4a80a82ee 100644 --- a/server/src/shutdown.ts +++ b/server/src/shutdown.ts @@ -17,7 +17,7 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu let isShuttingDown = false; - const shutdown = async () => { + const shutdown = async (exitCode: number = 0) => { if (isShuttingDown) { return; } @@ -31,7 +31,7 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu await shutdownTargets.healthServer?.close(); await shutdownTargets.db.disconnect(); logger.info({ message: "Graceful shutdown complete" }); - process.exit(0); + process.exit(exitCode); } catch (error) { const err = error as Error; logger.error({ @@ -43,7 +43,17 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu process.exit(1); } }; - process.on("SIGUSR2", shutdown); - process.on("SIGINT", shutdown); - process.on("SIGTERM", shutdown); + process.on("SIGUSR2", () => shutdown(0)); + process.on("SIGINT", () => shutdown(0)); + process.on("SIGTERM", () => shutdown(0)); + + process.on("unhandledRejection", (reason: unknown) => { + const err = reason instanceof Error ? reason : new Error(String(reason)); + logger.error({ message: `Unhandled promise rejection: ${err.message}`, service: SERVICE_NAME, method: "unhandledRejection", stack: err.stack }); + shutdown(1); + }); + process.on("uncaughtException", (error: Error) => { + logger.error({ message: `Uncaught exception: ${error.message}`, service: SERVICE_NAME, method: "uncaughtException", stack: error.stack }); + shutdown(1); + }); }; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index cdcb2de44..3ea2fddb8 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -195,12 +195,18 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now()); } } catch (error: unknown) { - await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: `runJob:${job.type}`, }); + await this.jobsRepository.recordFailure(job.id, error, Date.now()).catch((recordError: unknown) => { + this.logger.error({ + message: `recordFailure failed for job ${job.id}: ${recordError instanceof Error ? recordError.message : String(recordError)}`, + service: SERVICE_NAME, + method: `runJob:${job.type}`, + }); + }); // Record failure and release lock } finally { clearInterval(renewTimer); } @@ -218,9 +224,18 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now()); for (const job of jobs) { this.inFlight[type]++; // stay within concurrency limits - this.runJob(job).finally(() => { - this.inFlight[type]--; // job done, free the slot - }); + this.runJob(job) + .catch((error: unknown) => { + // defensive, runJob already catches + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: `runJob:${type}`, + }); + }) + .finally(() => { + this.inFlight[type]--; // job done, free the slot + }); } // Back off if no jobs found, reset to base if jobs found this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS; diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index cabf90e31..68df5f02a 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -306,6 +306,83 @@ describe("DBQueueWorker", () => { expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled(); }); + // Regression for the "worker crashes on a Mongo blip" bug: recordFailure is the last write in a + // job's lifecycle. During a Mongo outage it rejects, and on master that rejection escaped runJob → + // unhandledRejection → the whole worker exited (bypassing graceful drain). It must be swallowed. + it("does not propagate when recordFailure itself rejects (transient Mongo failure)", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => unhandled.push(reason); + process.on("unhandledRejection", onUnhandled); + try { + const job = makeJob({ type: "check" }); + const failingProducer = { produce: jest.fn().mockRejectedValue(new Error("network down")) }; + const { mocks, worker } = await start({ + mocks: { + jobsRepository: { + ...createWorker().mocks.jobsRepository, + claimDueBatch: claimOnce(job), + recordFailure: jest.fn().mockRejectedValue(new Error("mongo unavailable")), + }, + checkProducer: failingProducer, + }, + }); + await jest.advanceTimersByTimeAsync(1); // let the recordFailure rejection settle + + // the failing failure-write is caught and logged, not thrown + expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number)); + expect(mocks.logger.error).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining("recordFailure failed for job check:m1") }) + ); + expect(unhandled).toEqual([]); // nothing escaped to the process + expect(worker.getHealth().inFlight).toBe(0); // slot freed despite the double failure + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + + it("swallows the failure when both the success and failure writes reject", async () => { + const job = makeJob({ type: "check", intervalMs: 60000 }); // produce succeeds → recordSuccess path + const { mocks, worker } = await start({ + mocks: { + jobsRepository: { + ...createWorker().mocks.jobsRepository, + claimDueBatch: claimOnce(job), + recordSuccess: jest.fn().mockRejectedValue(new Error("mongo unavailable")), + recordFailure: jest.fn().mockRejectedValue(new Error("mongo still unavailable")), + }, + }, + }); + await jest.advanceTimersByTimeAsync(1); + + // success write is attempted, its rejection falls into the catch, and the failure write is too + expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled(); + expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number)); + expect(mocks.logger.error).toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining("recordFailure failed for job check:m1") }) + ); + expect(worker.getHealth().inFlight).toBe(0); + }); + + it("frees the concurrency slot and keeps polling after a job's completion writes fail", async () => { + const job = makeJob({ type: "check" }); + const { mocks, worker } = await start({ + mocks: { + jobsRepository: { + ...createWorker().mocks.jobsRepository, + claimDueBatch: claimOnce(job), + recordFailure: jest.fn().mockRejectedValue(new Error("mongo unavailable")), + }, + checkProducer: { produce: jest.fn().mockRejectedValue(new Error("network down")) }, + }, + }); + await jest.advanceTimersByTimeAsync(1); + expect(worker.getHealth().inFlight).toBe(0); // slot released via .finally + + const claimsBefore = mocks.jobsRepository.claimDueBatch.mock.calls.length; + await jest.advanceTimersByTimeAsync(POLL_MS); // the loop re-arms and keeps claiming — not wedged + expect(mocks.jobsRepository.claimDueBatch.mock.calls.length).toBeGreaterThan(claimsBefore); + }); + it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => { const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null }); const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });