Merge pull request #3754 from bluewave-labs/fix/worker-crash-on-mongo-error
fix: worker crash on mongo error
This commit is contained in:
3 files changed
+111
-9
No files matched your search
+15
-5
@@ -17,7 +17,7 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu
|
||||
|
||||
let isShuttingDown = false;
|
||||
|
||||
const shutdown = async () => {
|
||||
const shutdown = async (exitCode: number = 0) => {
|
||||
if (isShuttingDown) {
|
||||
return;
|
||||
}
|
||||
@@ -31,7 +31,7 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu
|
||||
await shutdownTargets.healthServer?.close();
|
||||
await shutdownTargets.db.disconnect();
|
||||
logger.info({ message: "Graceful shutdown complete" });
|
||||
process.exit(0);
|
||||
process.exit(exitCode);
|
||||
} catch (error) {
|
||||
const err = error as Error;
|
||||
logger.error({
|
||||
@@ -43,7 +43,17 @@ export const initShutdownListener = (server: Server | null, shutdownTargets: Shu
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
process.on("SIGUSR2", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGUSR2", () => shutdown(0));
|
||||
process.on("SIGINT", () => shutdown(0));
|
||||
process.on("SIGTERM", () => shutdown(0));
|
||||
|
||||
process.on("unhandledRejection", (reason: unknown) => {
|
||||
const err = reason instanceof Error ? reason : new Error(String(reason));
|
||||
logger.error({ message: `Unhandled promise rejection: ${err.message}`, service: SERVICE_NAME, method: "unhandledRejection", stack: err.stack });
|
||||
shutdown(1);
|
||||
});
|
||||
process.on("uncaughtException", (error: Error) => {
|
||||
logger.error({ message: `Uncaught exception: ${error.message}`, service: SERVICE_NAME, method: "uncaughtException", stack: error.stack });
|
||||
shutdown(1);
|
||||
});
|
||||
};
|
||||
@@ -195,12 +195,18 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now());
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock
|
||||
this.logger.warn({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: `runJob:${job.type}`,
|
||||
});
|
||||
await this.jobsRepository.recordFailure(job.id, error, Date.now()).catch((recordError: unknown) => {
|
||||
this.logger.error({
|
||||
message: `recordFailure failed for job ${job.id}: ${recordError instanceof Error ? recordError.message : String(recordError)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: `runJob:${job.type}`,
|
||||
});
|
||||
}); // Record failure and release lock
|
||||
} finally {
|
||||
clearInterval(renewTimer);
|
||||
}
|
||||
@@ -218,9 +224,18 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now());
|
||||
for (const job of jobs) {
|
||||
this.inFlight[type]++; // stay within concurrency limits
|
||||
this.runJob(job).finally(() => {
|
||||
this.inFlight[type]--; // job done, free the slot
|
||||
});
|
||||
this.runJob(job)
|
||||
.catch((error: unknown) => {
|
||||
// defensive, runJob already catches
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: `runJob:${type}`,
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
this.inFlight[type]--; // job done, free the slot
|
||||
});
|
||||
}
|
||||
// Back off if no jobs found, reset to base if jobs found
|
||||
this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS;
|
||||
|
||||
@@ -306,6 +306,83 @@ describe("DBQueueWorker", () => {
|
||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// Regression for the "worker crashes on a Mongo blip" bug: recordFailure is the last write in a
|
||||
// job's lifecycle. During a Mongo outage it rejects, and on master that rejection escaped runJob →
|
||||
// unhandledRejection → the whole worker exited (bypassing graceful drain). It must be swallowed.
|
||||
it("does not propagate when recordFailure itself rejects (transient Mongo failure)", async () => {
|
||||
const unhandled: unknown[] = [];
|
||||
const onUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
try {
|
||||
const job = makeJob({ type: "check" });
|
||||
const failingProducer = { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) };
|
||||
const { mocks, worker } = await start({
|
||||
mocks: {
|
||||
jobsRepository: {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
|
||||
},
|
||||
checkProducer: failingProducer,
|
||||
},
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1); // let the recordFailure rejection settle
|
||||
|
||||
// the failing failure-write is caught and logged, not thrown
|
||||
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
|
||||
expect(mocks.logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("recordFailure failed for job check:m1") })
|
||||
);
|
||||
expect(unhandled).toEqual([]); // nothing escaped to the process
|
||||
expect(worker.getHealth().inFlight).toBe(0); // slot freed despite the double failure
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandled);
|
||||
}
|
||||
});
|
||||
|
||||
it("swallows the failure when both the success and failure writes reject", async () => {
|
||||
const job = makeJob({ type: "check", intervalMs: 60000 }); // produce succeeds → recordSuccess path
|
||||
const { mocks, worker } = await start({
|
||||
mocks: {
|
||||
jobsRepository: {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
recordSuccess: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
|
||||
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo still unavailable")),
|
||||
},
|
||||
},
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
|
||||
// success write is attempted, its rejection falls into the catch, and the failure write is too
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.recordFailure).toHaveBeenCalledWith(job.id, expect.any(Error), expect.any(Number));
|
||||
expect(mocks.logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("recordFailure failed for job check:m1") })
|
||||
);
|
||||
expect(worker.getHealth().inFlight).toBe(0);
|
||||
});
|
||||
|
||||
it("frees the concurrency slot and keeps polling after a job's completion writes fail", async () => {
|
||||
const job = makeJob({ type: "check" });
|
||||
const { mocks, worker } = await start({
|
||||
mocks: {
|
||||
jobsRepository: {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
recordFailure: jest.fn<any>().mockRejectedValue(new Error("mongo unavailable")),
|
||||
},
|
||||
checkProducer: { produce: jest.fn<any>().mockRejectedValue(new Error("network down")) },
|
||||
},
|
||||
});
|
||||
await jest.advanceTimersByTimeAsync(1);
|
||||
expect(worker.getHealth().inFlight).toBe(0); // slot released via .finally
|
||||
|
||||
const claimsBefore = mocks.jobsRepository.claimDueBatch.mock.calls.length;
|
||||
await jest.advanceTimersByTimeAsync(POLL_MS); // the loop re-arms and keeps claiming — not wedged
|
||||
expect(mocks.jobsRepository.claimDueBatch.mock.calls.length).toBeGreaterThan(claimsBefore);
|
||||
});
|
||||
|
||||
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
Reference in new issue
Block a user