diff --git a/Server/index.js b/Server/index.js index b83a3d5ce..e12d83a18 100644 --- a/Server/index.js +++ b/Server/index.js @@ -8,7 +8,6 @@ import cors from "cors"; import logger from "./utils/logger.js"; import { verifyJWT } from "./middleware/verifyJWT.js"; import { handleErrors } from "./middleware/handleErrors.js"; -import { errorMessages } from "./utils/messages.js"; import authRouter from "./routes/authRoute.js"; import inviteRouter from "./routes/inviteRoute.js"; import monitorRouter from "./routes/monitorRoute.js"; @@ -45,8 +44,9 @@ import NotificationService from "./service/notificationService.js"; import db from "./db/mongo/MongoDB.js"; const SERVICE_NAME = "Server"; +const SHUTDOWN_TIMEOUT = 0; -let cleaningUp = false; +let isShuttingDown = false; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -113,11 +113,11 @@ const startApp = async () => { // Create services await db.connect(); - app.listen(PORT, () => { + const server = app.listen(PORT, () => { logger.info({ message: `server started on port:${PORT}` }); }); - const settingsService = new SettingsService(AppSettings); + const settingsService = new SettingsService(AppSettings); await settingsService.loadSettings(); const emailService = new EmailService( settingsService, @@ -142,30 +142,39 @@ const startApp = async () => { Worker ); - const cleanup = async () => { - if (cleaningUp) { - logger.warn({ message: "Already cleaning up" }); + const shutdown = async () => { + if (isShuttingDown) { return; } - cleaningUp = true; + isShuttingDown = true; + logger.info({ message: "Attempting graceful shutdown" }); + setTimeout(() => { + logger.error({ + message: "Could not shut down in time, forcing shutdown", + service: SERVICE_NAME, + method: "shutdown", + }); + process.exit(1); + }, SHUTDOWN_TIMEOUT); try { - logger.info({ message: "shutting down gracefully" }); + server.close(); await jobQueue.obliterate(); await db.disconnect(); - logger.info({ message: "shut down gracefully" }); + logger.info({ message: "Graceful shutdown complete" }); + process.exit(0); } catch (error) { logger.error({ message: error.message, service: SERVICE_NAME, - method: "cleanup", + method: "shutdown", stack: error.stack, }); } - process.exit(0); }; - process.on("SIGUSR2", cleanup); - process.on("SIGINT", cleanup); - process.on("SIGTERM", cleanup); + + process.on("SIGUSR2", shutdown); + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); }; startApp().catch((error) => { diff --git a/Server/service/jobQueue.js b/Server/service/jobQueue.js index 6d46052f1..aaab725e5 100644 --- a/Server/service/jobQueue.js +++ b/Server/service/jobQueue.js @@ -36,10 +36,10 @@ class JobQueue { host: redisHost, port: redisPort, }; - this.connection = connection; this.queue = new Queue(QUEUE_NAME, { connection, }); + this.connection = connection; this.workers = []; this.db = null; this.networkService = null; @@ -406,9 +406,6 @@ class JobQueue { delayed: await this.queue.getDelayedCount(), repeatableJobs: (await this.queue.getRepeatableJobs()).length, }; - this.logger.info({ - message: metrics, - }); return metrics; } catch (error) { this.logger.error({ @@ -424,17 +421,20 @@ class JobQueue { * @async * @returns {Promise} - Returns true if obliteration is successful */ + async obliterate() { try { - let metrics = await this.getMetrics(); - this.logger.info({ message: metrics }); + this.logger.info({ message: "Attempting to obliterate job queue..." }); await this.queue.pause(); const jobs = await this.getJobs(); + // Remove all repeatable jobs for (const job of jobs) { await this.queue.removeRepeatableByKey(job.key); await this.queue.remove(job.id); } + + // Close workers await Promise.all( this.workers.map(async (worker) => { await worker.close(); @@ -442,9 +442,13 @@ class JobQueue { ); await this.queue.obliterate(); - metrics = await this.getMetrics(); - this.logger.info({ message: metrics }); - this.logger.info({ message: successMessages.JOB_QUEUE_OBLITERATE }); + const metrics = await this.getMetrics(); + this.logger.info({ + message: successMessages.JOB_QUEUE_OBLITERATE, + service: SERVICE_NAME, + method: "obliterate", + details: metrics, + }); return true; } catch (error) { error.service === undefined ? (error.service = SERVICE_NAME) : null; diff --git a/Server/utils/logger.js b/Server/utils/logger.js index 60d509a6a..e263085a7 100644 --- a/Server/utils/logger.js +++ b/Server/utils/logger.js @@ -7,6 +7,10 @@ class Logger { if (message instanceof Object) { message = JSON.stringify(message, null, 2); } + + if (details instanceof Object) { + details = JSON.stringify(details, null, 2); + } let msg = `${timestamp} ${level}:`; service && (msg += ` [${service}]`); method && (msg += `(${method})`);