From 5a5c0c610d2e6010a18135ec73a05c434761b374 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 31 Oct 2024 15:01:08 +0800 Subject: [PATCH 1/6] replace cleanup method with better shutdown method --- Server/index.js | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Server/index.js b/Server/index.js index b83a3d5ce..52a3784be 100644 --- a/Server/index.js +++ b/Server/index.js @@ -8,7 +8,6 @@ import cors from "cors"; import logger from "./utils/logger.js"; import { verifyJWT } from "./middleware/verifyJWT.js"; import { handleErrors } from "./middleware/handleErrors.js"; -import { errorMessages } from "./utils/messages.js"; import authRouter from "./routes/authRoute.js"; import inviteRouter from "./routes/inviteRoute.js"; import monitorRouter from "./routes/monitorRoute.js"; @@ -46,7 +45,6 @@ import NotificationService from "./service/notificationService.js"; import db from "./db/mongo/MongoDB.js"; const SERVICE_NAME = "Server"; -let cleaningUp = false; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -113,11 +111,11 @@ const startApp = async () => { // Create services await db.connect(); - app.listen(PORT, () => { + const server = app.listen(PORT, () => { logger.info({ message: `server started on port:${PORT}` }); }); - const settingsService = new SettingsService(AppSettings); + const settingsService = new SettingsService(AppSettings); await settingsService.loadSettings(); const emailService = new EmailService( settingsService, @@ -142,30 +140,35 @@ const startApp = async () => { Worker ); - const cleanup = async () => { - if (cleaningUp) { - logger.warn({ message: "Already cleaning up" }); - return; - } - cleaningUp = true; + const shutdown = async () => { + logger.info({ message: "Attempting graceful shutdown" }); + setTimeout(() => { + logger.error({ + message: "Could not shut down in time, forcing shutdown", + service: SERVICE_NAME, + method: "shutdown", + }); + process.exit(1); + }, 10000); try { - logger.info({ message: "shutting down gracefully" }); + server.close(); await jobQueue.obliterate(); await db.disconnect(); - logger.info({ message: "shut down gracefully" }); + logger.info({ message: "Graceful shutdown complete" }); + process.exit(0); } catch (error) { logger.error({ message: error.message, service: SERVICE_NAME, - method: "cleanup", + method: "shutdown", stack: error.stack, }); } - process.exit(0); }; - process.on("SIGUSR2", cleanup); - process.on("SIGINT", cleanup); - process.on("SIGTERM", cleanup); + + process.on("SIGUSR2", shutdown); + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); }; startApp().catch((error) => { From 0f30d58125ddbe658a40bd94bdf3d20ef827b8b6 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 31 Oct 2024 15:59:51 +0800 Subject: [PATCH 2/6] Added isShuttigDown var to handle case of multiple kill/interupt signals --- Server/index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Server/index.js b/Server/index.js index 52a3784be..3e083df50 100644 --- a/Server/index.js +++ b/Server/index.js @@ -45,6 +45,7 @@ import NotificationService from "./service/notificationService.js"; import db from "./db/mongo/MongoDB.js"; const SERVICE_NAME = "Server"; +let isShuttingDown = false; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -141,6 +142,10 @@ const startApp = async () => { ); const shutdown = async () => { + if (isShuttingDown) { + return; + } + isShuttingDown = true; logger.info({ message: "Attempting graceful shutdown" }); setTimeout(() => { logger.error({ @@ -149,7 +154,7 @@ const startApp = async () => { method: "shutdown", }); process.exit(1); - }, 10000); + }, 2000); try { server.close(); await jobQueue.obliterate(); From 120296763bf46e3723eeb1136059fe5be777fcbe Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 31 Oct 2024 16:00:29 +0800 Subject: [PATCH 3/6] Attempt to stop active jobs --- Server/service/jobQueue.js | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/Server/service/jobQueue.js b/Server/service/jobQueue.js index 6d46052f1..3793a256a 100644 --- a/Server/service/jobQueue.js +++ b/Server/service/jobQueue.js @@ -36,10 +36,10 @@ class JobQueue { host: redisHost, port: redisPort, }; - this.connection = connection; this.queue = new Queue(QUEUE_NAME, { connection, }); + this.connection = connection; this.workers = []; this.db = null; this.networkService = null; @@ -406,9 +406,6 @@ class JobQueue { delayed: await this.queue.getDelayedCount(), repeatableJobs: (await this.queue.getRepeatableJobs()).length, }; - this.logger.info({ - message: metrics, - }); return metrics; } catch (error) { this.logger.error({ @@ -424,17 +421,33 @@ class JobQueue { * @async * @returns {Promise} - Returns true if obliteration is successful */ + async obliterate() { try { - let metrics = await this.getMetrics(); - this.logger.info({ message: metrics }); + this.logger.info({ message: "Attempting to obliterate job queue..." }); await this.queue.pause(); const jobs = await this.getJobs(); + // Stop currently active jobs + const redisClient = await this.queue.client; + const activeJobs = await this.queue.getJobs(["active"]); + if (activeJobs.length !== 0) { + this.logger.info({ message: "Attempting to stop active jobs..." }); + } + await Promise.all( + activeJobs.map(async (job) => { + await redisClient.del(`${this.queue.toKey(job.id)}:lock`); + await job.remove(); + }) + ); + + // Remove all repeatable jobs for (const job of jobs) { await this.queue.removeRepeatableByKey(job.key); await this.queue.remove(job.id); } + + // Close workers await Promise.all( this.workers.map(async (worker) => { await worker.close(); @@ -442,9 +455,13 @@ class JobQueue { ); await this.queue.obliterate(); - metrics = await this.getMetrics(); - this.logger.info({ message: metrics }); - this.logger.info({ message: successMessages.JOB_QUEUE_OBLITERATE }); + const metrics = await this.getMetrics(); + this.logger.info({ + message: successMessages.JOB_QUEUE_OBLITERATE, + service: SERVICE_NAME, + method: "obliterate", + details: metrics, + }); return true; } catch (error) { error.service === undefined ? (error.service = SERVICE_NAME) : null; From 3fc79d47ba4e690ca7e771891e0bdf4d8203f48f Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 31 Oct 2024 16:01:04 +0800 Subject: [PATCH 4/6] Stringify objects in the details field of logger config --- Server/utils/logger.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Server/utils/logger.js b/Server/utils/logger.js index 60d509a6a..e263085a7 100644 --- a/Server/utils/logger.js +++ b/Server/utils/logger.js @@ -7,6 +7,10 @@ class Logger { if (message instanceof Object) { message = JSON.stringify(message, null, 2); } + + if (details instanceof Object) { + details = JSON.stringify(details, null, 2); + } let msg = `${timestamp} ${level}:`; service && (msg += ` [${service}]`); method && (msg += `(${method})`); From c83e2cb6f2fe042448ec7f27e3c16d6d2ae5f340 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 31 Oct 2024 17:01:27 +0800 Subject: [PATCH 5/6] Add module var for shutdown timeout --- Server/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Server/index.js b/Server/index.js index 3e083df50..e12d83a18 100644 --- a/Server/index.js +++ b/Server/index.js @@ -44,6 +44,7 @@ import NotificationService from "./service/notificationService.js"; import db from "./db/mongo/MongoDB.js"; const SERVICE_NAME = "Server"; +const SHUTDOWN_TIMEOUT = 0; let isShuttingDown = false; const __filename = fileURLToPath(import.meta.url); @@ -154,7 +155,7 @@ const startApp = async () => { method: "shutdown", }); process.exit(1); - }, 2000); + }, SHUTDOWN_TIMEOUT); try { server.close(); await jobQueue.obliterate(); From ce221afa9fe0e8247455633668df8845d66d9908 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 31 Oct 2024 17:02:02 +0800 Subject: [PATCH 6/6] remove active job code, does not accomplish intended purpose --- Server/service/jobQueue.js | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Server/service/jobQueue.js b/Server/service/jobQueue.js index 3793a256a..aaab725e5 100644 --- a/Server/service/jobQueue.js +++ b/Server/service/jobQueue.js @@ -428,19 +428,6 @@ class JobQueue { await this.queue.pause(); const jobs = await this.getJobs(); - // Stop currently active jobs - const redisClient = await this.queue.client; - const activeJobs = await this.queue.getJobs(["active"]); - if (activeJobs.length !== 0) { - this.logger.info({ message: "Attempting to stop active jobs..." }); - } - await Promise.all( - activeJobs.map(async (job) => { - await redisClient.del(`${this.queue.toKey(job.id)}:lock`); - await job.remove(); - }) - ); - // Remove all repeatable jobs for (const job of jobs) { await this.queue.removeRepeatableByKey(job.key);