Merge pull request #1101 from bluewave-labs/feat/be/graceful-shutdown

Feat/be/graceful shutdown

Server/index.js

@@ -8,7 +8,6 @@ import cors from "cors";
 import logger from "./utils/logger.js";
 import { verifyJWT } from "./middleware/verifyJWT.js";
 import { handleErrors } from "./middleware/handleErrors.js";
-import { errorMessages } from "./utils/messages.js";
 import authRouter from "./routes/authRoute.js";
 import inviteRouter from "./routes/inviteRoute.js";
 import monitorRouter from "./routes/monitorRoute.js";
@@ -45,8 +44,9 @@ import NotificationService from "./service/notificationService.js";
 
 import db from "./db/mongo/MongoDB.js";
 const SERVICE_NAME = "Server";
+const SHUTDOWN_TIMEOUT = 0;
 
-let cleaningUp = false;
+let isShuttingDown = false;
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
@@ -113,11 +113,11 @@ const startApp = async () => {
 
 	// Create services
 	await db.connect();
-	app.listen(PORT, () => {
+	const server = app.listen(PORT, () => {
 		logger.info({ message: `server started on port:${PORT}` });
 	});
-	const settingsService = new SettingsService(AppSettings);
 
+	const settingsService = new SettingsService(AppSettings);
 	await settingsService.loadSettings();
 	const emailService = new EmailService(
 		settingsService,
@@ -142,30 +142,39 @@ const startApp = async () => {
 		Worker
 	);
 
-	const cleanup = async () => {
-		if (cleaningUp) {
-			logger.warn({ message: "Already cleaning up" });
+	const shutdown = async () => {
+		if (isShuttingDown) {
 			return;
 		}
-		cleaningUp = true;
+		isShuttingDown = true;
+		logger.info({ message: "Attempting graceful shutdown" });
+		setTimeout(() => {
+			logger.error({
+				message: "Could not shut down in time, forcing shutdown",
+				service: SERVICE_NAME,
+				method: "shutdown",
+			});
+			process.exit(1);
+		}, SHUTDOWN_TIMEOUT);
 		try {
-			logger.info({ message: "shutting down gracefully" });
+			server.close();
 			await jobQueue.obliterate();
 			await db.disconnect();
-			logger.info({ message: "shut down gracefully" });
+			logger.info({ message: "Graceful shutdown complete" });
+			process.exit(0);
 		} catch (error) {
 			logger.error({
 				message: error.message,
 				service: SERVICE_NAME,
-				method: "cleanup",
+				method: "shutdown",
 				stack: error.stack,
 			});
 		}
-		process.exit(0);
 	};
-	process.on("SIGUSR2", cleanup);
-	process.on("SIGINT", cleanup);
-	process.on("SIGTERM", cleanup);
+
+	process.on("SIGUSR2", shutdown);
+	process.on("SIGINT", shutdown);
+	process.on("SIGTERM", shutdown);
 };
 
 startApp().catch((error) => {

Server/service/jobQueue.js

@@ -36,10 +36,10 @@ class JobQueue {
 			host: redisHost,
 			port: redisPort,
 		};
-		this.connection = connection;
 		this.queue = new Queue(QUEUE_NAME, {
 			connection,
 		});
+		this.connection = connection;
 		this.workers = [];
 		this.db = null;
 		this.networkService = null;
@@ -406,9 +406,6 @@ class JobQueue {
 				delayed: await this.queue.getDelayedCount(),
 				repeatableJobs: (await this.queue.getRepeatableJobs()).length,
 			};
-			this.logger.info({
-				message: metrics,
-			});
 			return metrics;
 		} catch (error) {
 			this.logger.error({
@@ -424,17 +421,20 @@ class JobQueue {
 	 * @async
 	 * @returns {Promise<boolean>} - Returns true if obliteration is successful
 	 */
+
 	async obliterate() {
 		try {
-			let metrics = await this.getMetrics();
-			this.logger.info({ message: metrics });
+			this.logger.info({ message: "Attempting to obliterate job queue..." });
 			await this.queue.pause();
 			const jobs = await this.getJobs();
 
+			// Remove all repeatable jobs
 			for (const job of jobs) {
 				await this.queue.removeRepeatableByKey(job.key);
 				await this.queue.remove(job.id);
 			}
+
+			// Close workers
 			await Promise.all(
 				this.workers.map(async (worker) => {
 					await worker.close();
@@ -442,9 +442,13 @@ class JobQueue {
 			);
 
 			await this.queue.obliterate();
-			metrics = await this.getMetrics();
-			this.logger.info({ message: metrics });
-			this.logger.info({ message: successMessages.JOB_QUEUE_OBLITERATE });
+			const metrics = await this.getMetrics();
+			this.logger.info({
+				message: successMessages.JOB_QUEUE_OBLITERATE,
+				service: SERVICE_NAME,
+				method: "obliterate",
+				details: metrics,
+			});
 			return true;
 		} catch (error) {
 			error.service === undefined ? (error.service = SERVICE_NAME) : null;

Server/utils/logger.js

@@ -7,6 +7,10 @@ class Logger {
 				if (message instanceof Object) {
 					message = JSON.stringify(message, null, 2);
 				}
+
+				if (details instanceof Object) {
+					details = JSON.stringify(details, null, 2);
+				}
 				let msg = `${timestamp} ${level}:`;
 				service && (msg += ` [${service}]`);
 				method && (msg += `(${method})`);