diff --git a/Server/controllers/queueController.js b/Server/controllers/queueController.js
new file mode 100644
index 000000000..813fa9808
--- /dev/null
+++ b/Server/controllers/queueController.js
@@ -0,0 +1,42 @@
+const express = require("express");
+const logger = require("../utils/logger");
+const SERVICE_NAME = "JobQueue";
+
+const getJobs = async (req, res, next) => {
+ try {
+ const jobs = await req.jobQueue.getJobs();
+ return res.status(200).json({ jobs });
+ } catch (error) {
+ error.service = SERVICE_NAME;
+ next(error);
+ return;
+ }
+};
+
+const addJob = async (req, res, next) => {
+ try {
+ await req.jobQueue.addJob(Math.random().toString(36).substring(7));
+ return res.send("Added job");
+ } catch (error) {
+ error.service = SERVICE_NAME;
+ next(error);
+ return;
+ }
+};
+
+const obliterateQueue = async (req, res, next) => {
+ try {
+ const obliterated = await req.jobQueue.obliterate();
+ return res.status(200).send("Obliterated jobs");
+ } catch (error) {
+ error.service = SERVICE_NAME;
+ next(error);
+ return;
+ }
+};
+
+module.exports = {
+ getJobs,
+ addJob,
+ obliterateQueue,
+};
diff --git a/Server/index.js b/Server/index.js
index d194606f6..571a95657 100644
--- a/Server/index.js
+++ b/Server/index.js
@@ -10,84 +10,97 @@ require("dotenv").config();
const logger = require("./utils/logger");
const { verifyJWT } = require("./middleware/verifyJWT");
const { handleErrors } = require("./middleware/handleErrors");
+const queueRouter = require("./routes/queueRoute");
+const JobQueue = require("./service/jobQueue");
-// const { sendEmail } = require('./utils/sendEmail')
+// Need to wrap server setup in a function to handle async nature of JobQueue
+const startApp = async () => {
+ // const { sendEmail } = require('./utils/sendEmail')
-// **************************
-// Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out.
-// Simply comment out the FakeDB and uncomment the MongoDB or vice versa.
-// We can easily swap between any type of data source as long as the methods are implemented
-//
-// FakeDB
-// const db = require("./db/FakeDb");
-//
-// MongoDB
-// const db = require("./db/MongoDB");
-//
-// **************************
+ // **************************
+ // Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out.
+ // Simply comment out the FakeDB and uncomment the MongoDB or vice versa.
+ // We can easily swap between any type of data source as long as the methods are implemented
+ //
+ // FakeDB
+ // const db = require("./db/FakeDb");
+ //
+ // MongoDB
+ // const db = require("./db/MongoDB");
+ //
+ // **************************
+ const DB_TYPE = {
+ MongoDB: () => require("./db/MongoDB"),
+ FakedDB: () => require("./db/FakeDb"),
+ };
-const DB_TYPE = {
- MongoDB: () => require("./db/MongoDB"),
- FakedDB: () => require("./db/FakeDb"),
+ const db = DB_TYPE[process.env.DB_TYPE]
+ ? DB_TYPE[process.env.DB_TYPE]()
+ : require("./db/FakeDb");
+
+ const jobQueue = await JobQueue.createJobQueue();
+ /**
+ * NOTES
+ * Email Service will be added
+ * Logger Service will be added (Winston or similar)
+ */
+
+ const app = express();
+
+ // middlewares
+ app.use(
+ cors()
+ //We will add configuration later
+ );
+ app.use(express.json());
+ app.use(helmet());
+
+ // **************************
+ // Make DB accessible anywhere we have a Request object
+ // By adding the DB to the request object, we can access it in any route
+ // Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it
+ // Same applies for JobQueue
+ // **************************
+ app.use((req, res, next) => {
+ req.db = db;
+ req.jobQueue = jobQueue;
+ next();
+ });
+
+ //routes
+ app.use("/api/v1/auth", authRouter);
+ app.use("/api/v1/monitors", monitorRouter);
+ app.use("/api/v1/checks", verifyJWT, checkRouter);
+ app.use("/api/v1/alerts", verifyJWT, alertRouter);
+ //Temporary route for testing, remove later
+ app.use("/api/v1/job", queueRouter);
+
+ // Testing email service
+ // app.use('/sendEmail', async (req, res) => {
+ // const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', '
Testing Bluewavelabs
');
+ // console.log(response);
+ // })
+
+ //health check
+ app.use("/api/v1/healthy", (req, res) => {
+ try {
+ logger.info("Checking Health of the server.");
+ return res.status(200).json({ message: "Healthy" });
+ } catch (error) {
+ logger.error(error.message);
+ return res.status(500).json({ message: error.message });
+ }
+ });
+
+ /**
+ * Error handler middleware
+ * Should be called last
+ */
+ app.use(handleErrors);
+
+ connectDbAndRunServer(app, db);
};
-const db = DB_TYPE[process.env.DB_TYPE]
- ? DB_TYPE[process.env.DB_TYPE]()
- : require("./db/FakeDb");
-
-/**
- * NOTES
- * Email Service will be added
- * Logger Service will be added (Winston or similar)
- */
-
-const app = express();
-
-// middlewares
-app.use(
- cors()
- //We will add configuration later
-);
-app.use(express.json());
-app.use(helmet());
-
-// **************************
-// Make DB accessible anywhere we have a Request object
-// By adding the DB to the request object, we can access it in any route
-// Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it
-// **************************
-app.use((req, res, next) => {
- req.db = db;
- next();
+startApp().catch((error) => {
+ console.log(error);
});
-
-//routes
-app.use("/api/v1/auth", authRouter);
-app.use("/api/v1/monitors", monitorRouter);
-app.use("/api/v1/checks", verifyJWT, checkRouter);
-app.use("/api/v1/alerts", verifyJWT, alertRouter);
-
-// Testing email service
-// app.use('/sendEmail', async (req, res) => {
-// const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', 'Testing Bluewavelabs
');
-// console.log(response);
-// })
-
-//health check
-app.use("/api/v1/healthy", (req, res) => {
- try {
- logger.info("Checking Health of the server.");
- return res.status(200).json({ message: "Healthy" });
- } catch (error) {
- logger.error(error.message);
- return res.status(500).json({ message: error.message });
- }
-});
-
-/**
- * Error handler middleware
- * Should be called last
- */
-app.use(handleErrors);
-
-connectDbAndRunServer(app, db);
diff --git a/Server/package-lock.json b/Server/package-lock.json
index 9ee830ebd..55b7858b3 100644
--- a/Server/package-lock.json
+++ b/Server/package-lock.json
@@ -10,6 +10,7 @@
"license": "ISC",
"dependencies": {
"bcrypt": "^5.1.1",
+ "bullmq": "5.7.15",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
"express": "^4.19.2",
@@ -55,6 +56,12 @@
"@hapi/hoek": "^9.0.0"
}
},
+ "node_modules/@ioredis/commands": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
+ "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
+ "license": "MIT"
+ },
"node_modules/@mapbox/node-pre-gyp": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
@@ -96,6 +103,84 @@
"sparse-bitfield": "^3.0.3"
}
},
+ "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz",
+ "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz",
+ "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz",
+ "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==",
+ "cpu": [
+ "arm"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz",
+ "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==",
+ "cpu": [
+ "arm64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz",
+ "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz",
+ "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
+ "cpu": [
+ "x64"
+ ],
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ]
+ },
"node_modules/@sideway/address": {
"version": "4.1.5",
"resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz",
@@ -316,6 +401,21 @@
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
},
+ "node_modules/bullmq": {
+ "version": "5.7.15",
+ "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.15.tgz",
+ "integrity": "sha512-XR5sTA8BPUY67sS37sMKGCDvSLaVpMq7aaQG8FGSKOUnPoJMRf17n1TibVWP3+yK0xKLdK5Y7PY9D874Fpeqpg==",
+ "license": "MIT",
+ "dependencies": {
+ "cron-parser": "^4.6.0",
+ "ioredis": "^5.4.1",
+ "msgpackr": "^1.10.1",
+ "node-abort-controller": "^3.1.1",
+ "semver": "^7.5.4",
+ "tslib": "^2.0.0",
+ "uuid": "^9.0.0"
+ }
+ },
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -374,6 +474,15 @@
"node": ">=10"
}
},
+ "node_modules/cluster-key-slot": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+ "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/color": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
@@ -476,6 +585,18 @@
"node": ">= 0.10"
}
},
+ "node_modules/cron-parser": {
+ "version": "4.9.0",
+ "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
+ "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
+ "license": "MIT",
+ "dependencies": {
+ "luxon": "^3.2.1"
+ },
+ "engines": {
+ "node": ">=12.0.0"
+ }
+ },
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -505,6 +626,15 @@
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ=="
},
+ "node_modules/denque": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+ "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.10"
+ }
+ },
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -992,6 +1122,53 @@
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
+ "node_modules/ioredis": {
+ "version": "5.4.1",
+ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz",
+ "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==",
+ "license": "MIT",
+ "dependencies": {
+ "@ioredis/commands": "^1.1.1",
+ "cluster-key-slot": "^1.1.0",
+ "debug": "^4.3.4",
+ "denque": "^2.1.0",
+ "lodash.defaults": "^4.2.0",
+ "lodash.isarguments": "^3.1.0",
+ "redis-errors": "^1.2.0",
+ "redis-parser": "^3.0.0",
+ "standard-as-callback": "^2.1.0"
+ },
+ "engines": {
+ "node": ">=12.22.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/ioredis"
+ }
+ },
+ "node_modules/ioredis/node_modules/debug": {
+ "version": "4.3.5",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
+ "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
+ "license": "MIT",
+ "dependencies": {
+ "ms": "2.1.2"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/ioredis/node_modules/ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "license": "MIT"
+ },
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -1145,11 +1322,23 @@
"resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz",
"integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A=="
},
+ "node_modules/lodash.defaults": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
+ "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
+ "license": "MIT"
+ },
"node_modules/lodash.includes": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w=="
},
+ "node_modules/lodash.isarguments": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+ "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
+ "license": "MIT"
+ },
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -1212,6 +1401,15 @@
"node": ">=10"
}
},
+ "node_modules/luxon": {
+ "version": "3.4.4",
+ "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
+ "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/mailersend": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/mailersend/-/mailersend-2.2.0.tgz",
@@ -1478,6 +1676,37 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
+ "node_modules/msgpackr": {
+ "version": "1.10.2",
+ "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz",
+ "integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==",
+ "license": "MIT",
+ "optionalDependencies": {
+ "msgpackr-extract": "^3.0.2"
+ }
+ },
+ "node_modules/msgpackr-extract": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz",
+ "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==",
+ "hasInstallScript": true,
+ "license": "MIT",
+ "optional": true,
+ "dependencies": {
+ "node-gyp-build-optional-packages": "5.0.7"
+ },
+ "bin": {
+ "download-msgpackr-prebuilds": "bin/download-prebuilds.js"
+ },
+ "optionalDependencies": {
+ "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2",
+ "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2",
+ "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2",
+ "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2",
+ "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2",
+ "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2"
+ }
+ },
"node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -1486,6 +1715,12 @@
"node": ">= 0.6"
}
},
+ "node_modules/node-abort-controller": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz",
+ "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==",
+ "license": "MIT"
+ },
"node_modules/node-addon-api": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
@@ -1529,6 +1764,18 @@
"webidl-conversions": "^3.0.0"
}
},
+ "node_modules/node-gyp-build-optional-packages": {
+ "version": "5.0.7",
+ "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz",
+ "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==",
+ "license": "MIT",
+ "optional": true,
+ "bin": {
+ "node-gyp-build-optional-packages": "bin.js",
+ "node-gyp-build-optional-packages-optional": "optional.js",
+ "node-gyp-build-optional-packages-test": "build-test.js"
+ }
+ },
"node_modules/nodemon": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.0.tgz",
@@ -1778,6 +2025,27 @@
"node": ">=8.10.0"
}
},
+ "node_modules/redis-errors": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+ "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/redis-parser": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+ "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
+ "license": "MIT",
+ "dependencies": {
+ "redis-errors": "^1.0.0"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
"node_modules/rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -1969,6 +2237,12 @@
"node": "*"
}
},
+ "node_modules/standard-as-callback": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+ "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
+ "license": "MIT"
+ },
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -2093,6 +2367,12 @@
"node": ">= 14.0.0"
}
},
+ "node_modules/tslib": {
+ "version": "2.6.3",
+ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+ "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
+ "license": "0BSD"
+ },
"node_modules/type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@@ -2137,6 +2417,19 @@
"node": ">= 0.4.0"
}
},
+ "node_modules/uuid": {
+ "version": "9.0.1",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
+ "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
+ "funding": [
+ "https://github.com/sponsors/broofa",
+ "https://github.com/sponsors/ctavan"
+ ],
+ "license": "MIT",
+ "bin": {
+ "uuid": "dist/bin/uuid"
+ }
+ },
"node_modules/vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
diff --git a/Server/package.json b/Server/package.json
index 60777f7a5..d35461ebd 100644
--- a/Server/package.json
+++ b/Server/package.json
@@ -12,6 +12,7 @@
"license": "ISC",
"dependencies": {
"bcrypt": "^5.1.1",
+ "bullmq": "5.7.15",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
"express": "^4.19.2",
diff --git a/Server/routes/queueRoute.js b/Server/routes/queueRoute.js
new file mode 100644
index 000000000..79fa9b689
--- /dev/null
+++ b/Server/routes/queueRoute.js
@@ -0,0 +1,13 @@
+const router = require("express").Router();
+const queueController = require("../controllers/queueController");
+
+// Get Jobs
+router.get("/", queueController.getJobs);
+
+// Add Job
+router.post("/", queueController.addJob);
+
+// Obliterate Queue
+router.post("/obliterate", queueController.obliterateQueue);
+
+module.exports = router;
diff --git a/Server/service/jobQueue.js b/Server/service/jobQueue.js
new file mode 100644
index 000000000..84241f443
--- /dev/null
+++ b/Server/service/jobQueue.js
@@ -0,0 +1,195 @@
+const { Queue, Worker, Job } = require("bullmq");
+const QUEUE_NAME = "monitors";
+const connection = {
+ host: process.env.REDIS_HOST || "127.0.0.1",
+ port: process.env.REDIS_PORT || 6379,
+};
+const JOBS_PER_WORKER = 5;
+const logger = require("../utils/logger");
+
+class JobQueue {
+ /**
+ * Constructs a new JobQueue
+ * @constructor
+ * @throws {Error}
+ */
+ constructor() {
+ console.log(process.env.REDIS_PORT);
+ this.queue = new Queue(QUEUE_NAME, {
+ connection,
+ });
+ this.workers = [];
+ }
+
+ /**
+ * Static factory method to create a JobQueue
+ * @static
+ * @async
+ * @returns {Promise} - Returns a new JobQueue
+ *
+ */
+ static async createJobQueue() {
+ const queue = new JobQueue();
+ try {
+ const workerStats = await queue.getWorkerStats();
+ await queue.scaleWorkers(workerStats);
+ return queue;
+ } catch (error) {
+ throw error;
+ }
+ }
+
+ /**
+ * Creates a worker for the queue
+ * Operations are carried out in the async callback
+ * @returns {Worker} The newly created worker
+ */
+ createWorker() {
+ const worker = new Worker(
+ QUEUE_NAME,
+ async (job) => {
+ // TODO Ping a monitor
+ console.log(`${job.name} completed, workers: ${this.workers.length}`);
+ },
+ {
+ connection,
+ }
+ );
+ return worker;
+ }
+
+ /**
+ * @typedef {Object} WorkerStats
+ * @property {Array} jobs - Array of jobs in the Queue
+ * @property {number} - workerLoad - The number of jobs per worker
+ *
+ */
+
+ /**
+ * Gets stats related to the workers
+ * This is used for scaling workers right now
+ * In the future we will likely want to scale based on server performance metrics
+ * CPU Usage & memory usage, if too high, scale down workers.
+ * When to scale up? If jobs are taking too long to complete?
+ * @async
+ * @returns {Promise} - Returns the worker stats
+ */
+ async getWorkerStats() {
+ try {
+ const jobs = await this.queue.getRepeatableJobs();
+ const load = jobs.length / this.workers.length;
+ return { jobs, load };
+ } catch (error) {
+ throw error;
+ }
+ }
+
+ /**
+ * Scale Workers
+ * This function scales workers based on the load per worker
+ * If the load is higher than the JOBS_PER_WORKER threshold, we add more workers
+ * If the load is lower than the JOBS_PER_WORKER threshold, we release workers
+ * This approach ignores server performance, which we should add in the future
+ *
+
+ * @async
+ * @param {WorkerStats} workerStats - The payload for the job.
+ * @returns {Promise}
+ */
+ async scaleWorkers(workerStats) {
+ if (this.workers.length === 0) {
+ // There are no workers, need to add one
+ const worker = this.createWorker();
+ this.workers.push(worker);
+ return true;
+ }
+
+ if (workerStats.load > JOBS_PER_WORKER) {
+ // Find out how many more jobs we have than current workers can handle
+ const excessJobs =
+ workerStats.jobs.length - this.workers.length * JOBS_PER_WORKER;
+
+ // Divide by jobs/worker to find out how many workers to add
+ const workersToAdd = Math.ceil(excessJobs / JOBS_PER_WORKER);
+ for (let i = 0; i < workersToAdd; i++) {
+ const worker = this.createWorker();
+ this.workers.push(worker);
+ }
+ return true;
+ }
+
+ if (workerStats.load < JOBS_PER_WORKER) {
+ // Find out how much excess capacity we have
+ const workerCapacity = this.workers.length * JOBS_PER_WORKER;
+ const excessCapacity = workerCapacity - workerStats.jobs.length;
+ // Calculate how many workers to remove
+ const workersToRemove = Math.floor(excessCapacity / JOBS_PER_WORKER);
+ for (let i = 0; i < workersToRemove; i++) {
+ const worker = this.workers.pop();
+ try {
+ await worker.close();
+ } catch (error) {
+ // Catch the error instead of throwing it
+ console.error("Error closing worker", error);
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Gets all jobs in the queue.
+ *
+ * @async
+ * @returns {Promise>}
+ * @throws {Error} - Throws error if getting jobs fails
+ */
+ async getJobs() {
+ try {
+ const jobs = await this.queue.getRepeatableJobs();
+ console.log("jobs", jobs);
+ return jobs;
+ } catch (error) {
+ throw error;
+ }
+ }
+
+ /**
+ * Adds a job to the queue and scales workers based on worker stats.
+ *
+ * @async
+ * @param {string} jobName - The name of the job to be added.
+ * @param {Monitor} payload - The payload for the job.
+ * @throws {Error} - Will throw an error if the job cannot be added or workers don't scale
+ */
+ async addJob(jobName, payload) {
+ try {
+ await this.queue.add(jobName, payload, {
+ repeat: {
+ every: 1000,
+ limit: 100,
+ },
+ });
+ const workerStats = await this.getWorkerStats();
+ await this.scaleWorkers(workerStats);
+ } catch (error) {
+ throw error;
+ }
+ }
+
+ /**
+ * @async
+ * @returns {Promise} - Returns true if obliteration is successful
+ */
+ async obliterate() {
+ try {
+ await this.queue.obliterate();
+ return true;
+ } catch (error) {
+ throw error;
+ }
+ }
+}
+
+module.exports = JobQueue;