Merge pull request #3754 from bluewave-labs/fix/worker-crash-on-mongo-error

fix: worker crash on mongo error
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-07-03 10:50:59 -07:00
commit 817d01c857
3 files changed
+111 -9

No files matched your search

+15 -5
View File
@@ -17,7 +17,7 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu
let isShuttingDown = false;
const shutdown = async () => {
const shutdown = async (exitCode: number = 0) => {
if (isShuttingDown) {
return;
}
@@ -31,7 +31,7 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu
await shutdownTargets.healthServer?.close();
await shutdownTargets.db.disconnect();
logger.info({ message: "Graceful shutdown complete" });
process.exit(0);
process.exit(exitCode);
} catch (error) {
const err = error as Error;
logger.error({
@@ -43,7 +43,17 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu
process.exit(1);
}
};
process.on("SIGUSR2", shutdown);
process.on("SIGINT", shutdown);
process.on("SIGTERM", shutdown);
process.on("SIGUSR2", () => shutdown(0));
process.on("SIGINT", () => shutdown(0));
process.on("SIGTERM", () => shutdown(0));
process.on("unhandledRejection", (reason: unknown) => {
const err = reason instanceof Error ? reason : new Error(String(reason));
logger.error({ message: `Unhandled promise rejection: ${err.message}`, service: SERVICE_NAME, method: "unhandledRejection", stack: err.stack });
shutdown(1);
});
process.on("uncaughtException", (error: Error) => {
logger.error({ message: `Uncaught exception: ${error.message}`, service: SERVICE_NAME, method: "uncaughtException", stack: error.stack });
shutdown(1);
});
};
+19 -4
View File
@@ -195,12 +195,18 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now());
}
} catch (error: unknown) {
await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock
this.logger.warn({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: `runJob:${job.type}`,
});
await this.jobsRepository.recordFailure(job.id, error, Date.now()).catch((recordError: unknown) => {
this.logger.error({
message: `recordFailure failed for job ${job.id}: ${recordError instanceof Error ? recordError.message : String(recordError)}`,
service: SERVICE_NAME,
method: `runJob:${job.type}`,
});
}); // Record failure and release lock
} finally {
clearInterval(renewTimer);
}
@@ -218,9 +224,18 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now());
for (const job of jobs) {
this.inFlight[type]++; // stay within concurrency limits
this.runJob(job).finally(() => {
this.inFlight[type]--; // job done, free the slot
});
this.runJob(job)
.catch((error: unknown) => {
// defensive, runJob already catches
this.logger.error({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: `runJob:${type}`,
});
})
.finally(() => {
this.inFlight[type]--; // job done, free the slot
});
}
// Back off if no jobs found, reset to base if jobs found
this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS;
@@ -306,6 +306,83 @@ describe("DBQueueWorker", () => {
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
});
// Regression for the "worker crashes on a Mongo blip" bug: recordFailure is the last write in a
// job's lifecycle. During a Mongo outage it rejects, and on master that rejection escaped runJob →
// unhandledRejection → the whole worker exited (bypassing graceful drain). It must be swallowed.
it("does not propagate when recordFailure itself rejects (transient Mongo failure)", async () => {
const unhandled: unknown[] = [];
const onUnhandled = (reason: unknown) => unhandled.push(reason);
process.on("unhandledRejection", onUnhandled);
try {
const job = makeJob({ type: "check" });
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
const { mocks, worker } = await start({
mocks: {
jobsRepository: {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
},
checkProducer: failingProducer,
},
});
await jest.advanceTimersByTimeAsync(1); // let the recordFailure rejection settle
// the failing failure-write is caught and logged, not thrown
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
expect(mocks.logger.error).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("recordFailure failed for job check:m1") })
);
expect(unhandled).toEqual([]); // nothing escaped to the process
expect(worker.getHealth().inFlight).toBe(0); // slot freed despite the double failure
} finally {
process.off("unhandledRejection", onUnhandled);
}
});
it("swallows the failure when both the success and failure writes reject", async () => {
const job = makeJob({ type: "check", intervalMs: 60000 }); // produce succeeds → recordSuccess path
const { mocks, worker } = await start({
mocks: {
jobsRepository: {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
recordSuccess: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo still unavailable")),
},
},
});
await jest.advanceTimersByTimeAsync(1);
// success write is attempted, its rejection falls into the catch, and the failure write is too
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled();
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
expect(mocks.logger.error).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("recordFailure failed for job check:m1") })
);
expect(worker.getHealth().inFlight).toBe(0);
});
it("frees the concurrency slot and keeps polling after a job's completion writes fail", async () => {
const job = makeJob({ type: "check" });
const { mocks, worker } = await start({
mocks: {
jobsRepository: {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
},
checkProducer: { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) },
},
});
await jest.advanceTimersByTimeAsync(1);
expect(worker.getHealth().inFlight).toBe(0); // slot released via .finally
const claimsBefore = mocks.jobsRepository.claimDueBatch.mock.calls.length;
await jest.advanceTimersByTimeAsync(POLL_MS); // the loop re-arms and keeps claiming — not wedged
expect(mocks.jobsRepository.claimDueBatch.mock.calls.length).toBeGreaterThan(claimsBefore);
});
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });