diff --git a/Server/controllers/queueController.js b/Server/controllers/queueController.js new file mode 100644 index 000000000..813fa9808 --- /dev/null +++ b/Server/controllers/queueController.js @@ -0,0 +1,42 @@ +const express = require("express"); +const logger = require("../utils/logger"); +const SERVICE_NAME = "JobQueue"; + +const getJobs = async (req, res, next) => { + try { + const jobs = await req.jobQueue.getJobs(); + return res.status(200).json({ jobs }); + } catch (error) { + error.service = SERVICE_NAME; + next(error); + return; + } +}; + +const addJob = async (req, res, next) => { + try { + await req.jobQueue.addJob(Math.random().toString(36).substring(7)); + return res.send("Added job"); + } catch (error) { + error.service = SERVICE_NAME; + next(error); + return; + } +}; + +const obliterateQueue = async (req, res, next) => { + try { + const obliterated = await req.jobQueue.obliterate(); + return res.status(200).send("Obliterated jobs"); + } catch (error) { + error.service = SERVICE_NAME; + next(error); + return; + } +}; + +module.exports = { + getJobs, + addJob, + obliterateQueue, +}; diff --git a/Server/index.js b/Server/index.js index d194606f6..571a95657 100644 --- a/Server/index.js +++ b/Server/index.js @@ -10,84 +10,97 @@ require("dotenv").config(); const logger = require("./utils/logger"); const { verifyJWT } = require("./middleware/verifyJWT"); const { handleErrors } = require("./middleware/handleErrors"); +const queueRouter = require("./routes/queueRoute"); +const JobQueue = require("./service/jobQueue"); -// const { sendEmail } = require('./utils/sendEmail') +// Need to wrap server setup in a function to handle async nature of JobQueue +const startApp = async () => { + // const { sendEmail } = require('./utils/sendEmail') -// ************************** -// Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out. -// Simply comment out the FakeDB and uncomment the MongoDB or vice versa. -// We can easily swap between any type of data source as long as the methods are implemented -// -// FakeDB -// const db = require("./db/FakeDb"); -// -// MongoDB -// const db = require("./db/MongoDB"); -// -// ************************** + // ************************** + // Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out. + // Simply comment out the FakeDB and uncomment the MongoDB or vice versa. + // We can easily swap between any type of data source as long as the methods are implemented + // + // FakeDB + // const db = require("./db/FakeDb"); + // + // MongoDB + // const db = require("./db/MongoDB"); + // + // ************************** + const DB_TYPE = { + MongoDB: () => require("./db/MongoDB"), + FakedDB: () => require("./db/FakeDb"), + }; -const DB_TYPE = { - MongoDB: () => require("./db/MongoDB"), - FakedDB: () => require("./db/FakeDb"), + const db = DB_TYPE[process.env.DB_TYPE] + ? DB_TYPE[process.env.DB_TYPE]() + : require("./db/FakeDb"); + + const jobQueue = await JobQueue.createJobQueue(); + /** + * NOTES + * Email Service will be added + * Logger Service will be added (Winston or similar) + */ + + const app = express(); + + // middlewares + app.use( + cors() + //We will add configuration later + ); + app.use(express.json()); + app.use(helmet()); + + // ************************** + // Make DB accessible anywhere we have a Request object + // By adding the DB to the request object, we can access it in any route + // Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it + // Same applies for JobQueue + // ************************** + app.use((req, res, next) => { + req.db = db; + req.jobQueue = jobQueue; + next(); + }); + + //routes + app.use("/api/v1/auth", authRouter); + app.use("/api/v1/monitors", monitorRouter); + app.use("/api/v1/checks", verifyJWT, checkRouter); + app.use("/api/v1/alerts", verifyJWT, alertRouter); + //Temporary route for testing, remove later + app.use("/api/v1/job", queueRouter); + + // Testing email service + // app.use('/sendEmail', async (req, res) => { + // const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', '

Testing Bluewavelabs

'); + // console.log(response); + // }) + + //health check + app.use("/api/v1/healthy", (req, res) => { + try { + logger.info("Checking Health of the server."); + return res.status(200).json({ message: "Healthy" }); + } catch (error) { + logger.error(error.message); + return res.status(500).json({ message: error.message }); + } + }); + + /** + * Error handler middleware + * Should be called last + */ + app.use(handleErrors); + + connectDbAndRunServer(app, db); }; -const db = DB_TYPE[process.env.DB_TYPE] - ? DB_TYPE[process.env.DB_TYPE]() - : require("./db/FakeDb"); - -/** - * NOTES - * Email Service will be added - * Logger Service will be added (Winston or similar) - */ - -const app = express(); - -// middlewares -app.use( - cors() - //We will add configuration later -); -app.use(express.json()); -app.use(helmet()); - -// ************************** -// Make DB accessible anywhere we have a Request object -// By adding the DB to the request object, we can access it in any route -// Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it -// ************************** -app.use((req, res, next) => { - req.db = db; - next(); +startApp().catch((error) => { + console.log(error); }); - -//routes -app.use("/api/v1/auth", authRouter); -app.use("/api/v1/monitors", monitorRouter); -app.use("/api/v1/checks", verifyJWT, checkRouter); -app.use("/api/v1/alerts", verifyJWT, alertRouter); - -// Testing email service -// app.use('/sendEmail', async (req, res) => { -// const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', '

Testing Bluewavelabs

'); -// console.log(response); -// }) - -//health check -app.use("/api/v1/healthy", (req, res) => { - try { - logger.info("Checking Health of the server."); - return res.status(200).json({ message: "Healthy" }); - } catch (error) { - logger.error(error.message); - return res.status(500).json({ message: error.message }); - } -}); - -/** - * Error handler middleware - * Should be called last - */ -app.use(handleErrors); - -connectDbAndRunServer(app, db); diff --git a/Server/package-lock.json b/Server/package-lock.json index 9ee830ebd..55b7858b3 100644 --- a/Server/package-lock.json +++ b/Server/package-lock.json @@ -10,6 +10,7 @@ "license": "ISC", "dependencies": { "bcrypt": "^5.1.1", + "bullmq": "5.7.15", "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.19.2", @@ -55,6 +56,12 @@ "@hapi/hoek": "^9.0.0" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", + "license": "MIT" + }, "node_modules/@mapbox/node-pre-gyp": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", @@ -96,6 +103,84 @@ "sparse-bitfield": "^3.0.3" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@sideway/address": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", @@ -316,6 +401,21 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" }, + "node_modules/bullmq": { + "version": "5.7.15", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.15.tgz", + "integrity": "sha512-XR5sTA8BPUY67sS37sMKGCDvSLaVpMq7aaQG8FGSKOUnPoJMRf17n1TibVWP3+yK0xKLdK5Y7PY9D874Fpeqpg==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.6.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.10.1", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -374,6 +474,15 @@ "node": ">=10" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", @@ -476,6 +585,18 @@ "node": ">= 0.10" } }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -505,6 +626,15 @@ "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==" }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -992,6 +1122,53 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "license": "MIT", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "license": "MIT" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1145,11 +1322,23 @@ "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -1212,6 +1401,15 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/mailersend": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/mailersend/-/mailersend-2.2.0.tgz", @@ -1478,6 +1676,37 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/msgpackr": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz", + "integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.7" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -1486,6 +1715,12 @@ "node": ">= 0.6" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, "node_modules/node-addon-api": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz", @@ -1529,6 +1764,18 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/nodemon": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.0.tgz", @@ -1778,6 +2025,27 @@ "node": ">=8.10.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -1969,6 +2237,12 @@ "node": "*" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -2093,6 +2367,12 @@ "node": ">= 14.0.0" } }, + "node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "license": "0BSD" + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -2137,6 +2417,19 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/Server/package.json b/Server/package.json index 60777f7a5..d35461ebd 100644 --- a/Server/package.json +++ b/Server/package.json @@ -12,6 +12,7 @@ "license": "ISC", "dependencies": { "bcrypt": "^5.1.1", + "bullmq": "5.7.15", "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.19.2", diff --git a/Server/routes/queueRoute.js b/Server/routes/queueRoute.js new file mode 100644 index 000000000..79fa9b689 --- /dev/null +++ b/Server/routes/queueRoute.js @@ -0,0 +1,13 @@ +const router = require("express").Router(); +const queueController = require("../controllers/queueController"); + +// Get Jobs +router.get("/", queueController.getJobs); + +// Add Job +router.post("/", queueController.addJob); + +// Obliterate Queue +router.post("/obliterate", queueController.obliterateQueue); + +module.exports = router; diff --git a/Server/service/jobQueue.js b/Server/service/jobQueue.js new file mode 100644 index 000000000..84241f443 --- /dev/null +++ b/Server/service/jobQueue.js @@ -0,0 +1,195 @@ +const { Queue, Worker, Job } = require("bullmq"); +const QUEUE_NAME = "monitors"; +const connection = { + host: process.env.REDIS_HOST || "127.0.0.1", + port: process.env.REDIS_PORT || 6379, +}; +const JOBS_PER_WORKER = 5; +const logger = require("../utils/logger"); + +class JobQueue { + /** + * Constructs a new JobQueue + * @constructor + * @throws {Error} + */ + constructor() { + console.log(process.env.REDIS_PORT); + this.queue = new Queue(QUEUE_NAME, { + connection, + }); + this.workers = []; + } + + /** + * Static factory method to create a JobQueue + * @static + * @async + * @returns {Promise} - Returns a new JobQueue + * + */ + static async createJobQueue() { + const queue = new JobQueue(); + try { + const workerStats = await queue.getWorkerStats(); + await queue.scaleWorkers(workerStats); + return queue; + } catch (error) { + throw error; + } + } + + /** + * Creates a worker for the queue + * Operations are carried out in the async callback + * @returns {Worker} The newly created worker + */ + createWorker() { + const worker = new Worker( + QUEUE_NAME, + async (job) => { + // TODO Ping a monitor + console.log(`${job.name} completed, workers: ${this.workers.length}`); + }, + { + connection, + } + ); + return worker; + } + + /** + * @typedef {Object} WorkerStats + * @property {Array} jobs - Array of jobs in the Queue + * @property {number} - workerLoad - The number of jobs per worker + * + */ + + /** + * Gets stats related to the workers + * This is used for scaling workers right now + * In the future we will likely want to scale based on server performance metrics + * CPU Usage & memory usage, if too high, scale down workers. + * When to scale up? If jobs are taking too long to complete? + * @async + * @returns {Promise} - Returns the worker stats + */ + async getWorkerStats() { + try { + const jobs = await this.queue.getRepeatableJobs(); + const load = jobs.length / this.workers.length; + return { jobs, load }; + } catch (error) { + throw error; + } + } + + /** + * Scale Workers + * This function scales workers based on the load per worker + * If the load is higher than the JOBS_PER_WORKER threshold, we add more workers + * If the load is lower than the JOBS_PER_WORKER threshold, we release workers + * This approach ignores server performance, which we should add in the future + * + + * @async + * @param {WorkerStats} workerStats - The payload for the job. + * @returns {Promise} + */ + async scaleWorkers(workerStats) { + if (this.workers.length === 0) { + // There are no workers, need to add one + const worker = this.createWorker(); + this.workers.push(worker); + return true; + } + + if (workerStats.load > JOBS_PER_WORKER) { + // Find out how many more jobs we have than current workers can handle + const excessJobs = + workerStats.jobs.length - this.workers.length * JOBS_PER_WORKER; + + // Divide by jobs/worker to find out how many workers to add + const workersToAdd = Math.ceil(excessJobs / JOBS_PER_WORKER); + for (let i = 0; i < workersToAdd; i++) { + const worker = this.createWorker(); + this.workers.push(worker); + } + return true; + } + + if (workerStats.load < JOBS_PER_WORKER) { + // Find out how much excess capacity we have + const workerCapacity = this.workers.length * JOBS_PER_WORKER; + const excessCapacity = workerCapacity - workerStats.jobs.length; + // Calculate how many workers to remove + const workersToRemove = Math.floor(excessCapacity / JOBS_PER_WORKER); + for (let i = 0; i < workersToRemove; i++) { + const worker = this.workers.pop(); + try { + await worker.close(); + } catch (error) { + // Catch the error instead of throwing it + console.error("Error closing worker", error); + } + } + return true; + } + return false; + } + + /** + * Gets all jobs in the queue. + * + * @async + * @returns {Promise>} + * @throws {Error} - Throws error if getting jobs fails + */ + async getJobs() { + try { + const jobs = await this.queue.getRepeatableJobs(); + console.log("jobs", jobs); + return jobs; + } catch (error) { + throw error; + } + } + + /** + * Adds a job to the queue and scales workers based on worker stats. + * + * @async + * @param {string} jobName - The name of the job to be added. + * @param {Monitor} payload - The payload for the job. + * @throws {Error} - Will throw an error if the job cannot be added or workers don't scale + */ + async addJob(jobName, payload) { + try { + await this.queue.add(jobName, payload, { + repeat: { + every: 1000, + limit: 100, + }, + }); + const workerStats = await this.getWorkerStats(); + await this.scaleWorkers(workerStats); + } catch (error) { + throw error; + } + } + + /** + * @async + * @returns {Promise} - Returns true if obliteration is successful + */ + async obliterate() { + try { + await this.queue.obliterate(); + return true; + } catch (error) { + throw error; + } + } +} + +module.exports = JobQueue;