Added queue service, Added controller for testing purposes

This commit is contained in:
Alex Holliday committed 2024-06-05 12:29:01 -07:00
1 parent 86130ac57e
commit 88b0b0f97c
6 files changed
+632 -75

No files matched your search

+42
View File
@@ -0,0 +1,42 @@
const express = require("express");
const logger = require("../utils/logger");
const SERVICE_NAME = "JobQueue";
const getJobs = async (req, res, next) => {
try {
const jobs = await req.jobQueue.getJobs();
return res.status(200).json({ jobs });
} catch (error) {
error.service = SERVICE_NAME;
next(error);
return;
}
};
const addJob = async (req, res, next) => {
try {
await req.jobQueue.addJob(Math.random().toString(36).substring(7));
return res.send("Added job");
} catch (error) {
error.service = SERVICE_NAME;
next(error);
return;
}
};
const obliterateQueue = async (req, res, next) => {
try {
const obliterated = await req.jobQueue.obliterate();
return res.status(200).send("Obliterated jobs");
} catch (error) {
error.service = SERVICE_NAME;
next(error);
return;
}
};
module.exports = {
getJobs,
addJob,
obliterateQueue,
};
+88 -75
View File
@@ -10,84 +10,97 @@ require("dotenv").config();
const logger = require("./utils/logger");
const { verifyJWT } = require("./middleware/verifyJWT");
const { handleErrors } = require("./middleware/handleErrors");
const queueRouter = require("./routes/queueRoute");
const JobQueue = require("./service/jobQueue");
// const { sendEmail } = require('./utils/sendEmail')
// Need to wrap server setup in a function to handle async nature of JobQueue
const startApp = async () => {
// const { sendEmail } = require('./utils/sendEmail')
// **************************
// Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out.
// Simply comment out the FakeDB and uncomment the MongoDB or vice versa.
// We can easily swap between any type of data source as long as the methods are implemented
//
// FakeDB
// const db = require("./db/FakeDb");
//
// MongoDB
// const db = require("./db/MongoDB");
//
// **************************
// **************************
// Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out.
// Simply comment out the FakeDB and uncomment the MongoDB or vice versa.
// We can easily swap between any type of data source as long as the methods are implemented
//
// FakeDB
// const db = require("./db/FakeDb");
//
// MongoDB
// const db = require("./db/MongoDB");
//
// **************************
const DB_TYPE = {
MongoDB: () => require("./db/MongoDB"),
FakedDB: () => require("./db/FakeDb"),
};
const DB_TYPE = {
MongoDB: () => require("./db/MongoDB"),
FakedDB: () => require("./db/FakeDb"),
const db = DB_TYPE[process.env.DB_TYPE]
? DB_TYPE[process.env.DB_TYPE]()
: require("./db/FakeDb");
const jobQueue = await JobQueue.createJobQueue();
/**
* NOTES
* Email Service will be added
* Logger Service will be added (Winston or similar)
*/
const app = express();
// middlewares
app.use(
cors()
//We will add configuration later
);
app.use(express.json());
app.use(helmet());
// **************************
// Make DB accessible anywhere we have a Request object
// By adding the DB to the request object, we can access it in any route
// Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it
// Same applies for JobQueue
// **************************
app.use((req, res, next) => {
req.db = db;
req.jobQueue = jobQueue;
next();
});
//routes
app.use("/api/v1/auth", authRouter);
app.use("/api/v1/monitors", monitorRouter);
app.use("/api/v1/checks", verifyJWT, checkRouter);
app.use("/api/v1/alerts", verifyJWT, alertRouter);
//Temporary route for testing, remove later
app.use("/api/v1/job", queueRouter);
// Testing email service
// app.use('/sendEmail', async (req, res) => {
// const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', '<h1>Testing Bluewavelabs</h1>');
// console.log(response);
// })
//health check
app.use("/api/v1/healthy", (req, res) => {
try {
logger.info("Checking Health of the server.");
return res.status(200).json({ message: "Healthy" });
} catch (error) {
logger.error(error.message);
return res.status(500).json({ message: error.message });
}
});
/**
* Error handler middleware
* Should be called last
*/
app.use(handleErrors);
connectDbAndRunServer(app, db);
};
const db = DB_TYPE[process.env.DB_TYPE]
? DB_TYPE[process.env.DB_TYPE]()
: require("./db/FakeDb");
/**
* NOTES
* Email Service will be added
* Logger Service will be added (Winston or similar)
*/
const app = express();
// middlewares
app.use(
cors()
//We will add configuration later
);
app.use(express.json());
app.use(helmet());
// **************************
// Make DB accessible anywhere we have a Request object
// By adding the DB to the request object, we can access it in any route
// Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it
// **************************
app.use((req, res, next) => {
req.db = db;
next();
startApp().catch((error) => {
console.log(error);
});
//routes
app.use("/api/v1/auth", authRouter);
app.use("/api/v1/monitors", monitorRouter);
app.use("/api/v1/checks", verifyJWT, checkRouter);
app.use("/api/v1/alerts", verifyJWT, alertRouter);
// Testing email service
// app.use('/sendEmail', async (req, res) => {
// const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', '<h1>Testing Bluewavelabs</h1>');
// console.log(response);
// })
//health check
app.use("/api/v1/healthy", (req, res) => {
try {
logger.info("Checking Health of the server.");
return res.status(200).json({ message: "Healthy" });
} catch (error) {
logger.error(error.message);
return res.status(500).json({ message: error.message });
}
});
/**
* Error handler middleware
* Should be called last
*/
app.use(handleErrors);
connectDbAndRunServer(app, db);
+293
View File
@@ -10,6 +10,7 @@
"license": "ISC",
"dependencies": {
"bcrypt": "^5.1.1",
"bullmq": "5.7.15",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
"express": "^4.19.2",
@@ -55,6 +56,12 @@
"@hapi/hoek": "^9.0.0"
}
},
"node_modules/@ioredis/commands": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
"license": "MIT"
},
"node_modules/@mapbox/node-pre-gyp": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
@@ -96,6 +103,84 @@
"sparse-bitfield": "^3.0.3"
}
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz",
"integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz",
"integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz",
"integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz",
"integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz",
"integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz",
"integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
]
},
"node_modules/@sideway/address": {
"version": "4.1.5",
"resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz",
@@ -316,6 +401,21 @@
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
},
"node_modules/bullmq": {
"version": "5.7.15",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.15.tgz",
"integrity": "sha512-XR5sTA8BPUY67sS37sMKGCDvSLaVpMq7aaQG8FGSKOUnPoJMRf17n1TibVWP3+yK0xKLdK5Y7PY9D874Fpeqpg==",
"license": "MIT",
"dependencies": {
"cron-parser": "^4.6.0",
"ioredis": "^5.4.1",
"msgpackr": "^1.10.1",
"node-abort-controller": "^3.1.1",
"semver": "^7.5.4",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -374,6 +474,15 @@
"node": ">=10"
}
},
"node_modules/cluster-key-slot": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/color": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
@@ -476,6 +585,18 @@
"node": ">= 0.10"
}
},
"node_modules/cron-parser": {
"version": "4.9.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
"integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
"license": "MIT",
"dependencies": {
"luxon": "^3.2.1"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -505,6 +626,15 @@
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ=="
},
"node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -992,6 +1122,53 @@
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"node_modules/ioredis": {
"version": "5.4.1",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz",
"integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==",
"license": "MIT",
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/ioredis/node_modules/debug": {
"version": "4.3.5",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
"integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
"license": "MIT",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/ioredis/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"license": "MIT"
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -1145,11 +1322,23 @@
"resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz",
"integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A=="
},
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
"license": "MIT"
},
"node_modules/lodash.includes": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w=="
},
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
"license": "MIT"
},
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -1212,6 +1401,15 @@
"node": ">=10"
}
},
"node_modules/luxon": {
"version": "3.4.4",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
"integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==",
"license": "MIT",
"engines": {
"node": ">=12"
}
},
"node_modules/mailersend": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/mailersend/-/mailersend-2.2.0.tgz",
@@ -1478,6 +1676,37 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"node_modules/msgpackr": {
"version": "1.10.2",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz",
"integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==",
"license": "MIT",
"optionalDependencies": {
"msgpackr-extract": "^3.0.2"
}
},
"node_modules/msgpackr-extract": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz",
"integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==",
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"dependencies": {
"node-gyp-build-optional-packages": "5.0.7"
},
"bin": {
"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
},
"optionalDependencies": {
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2",
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2"
}
},
"node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -1486,6 +1715,12 @@
"node": ">= 0.6"
}
},
"node_modules/node-abort-controller": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz",
"integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==",
"license": "MIT"
},
"node_modules/node-addon-api": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
@@ -1529,6 +1764,18 @@
"webidl-conversions": "^3.0.0"
}
},
"node_modules/node-gyp-build-optional-packages": {
"version": "5.0.7",
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz",
"integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==",
"license": "MIT",
"optional": true,
"bin": {
"node-gyp-build-optional-packages": "bin.js",
"node-gyp-build-optional-packages-optional": "optional.js",
"node-gyp-build-optional-packages-test": "build-test.js"
}
},
"node_modules/nodemon": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.0.tgz",
@@ -1778,6 +2025,27 @@
"node": ">=8.10.0"
}
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"license": "MIT",
"dependencies": {
"redis-errors": "^1.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -1969,6 +2237,12 @@
"node": "*"
}
},
"node_modules/standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
"license": "MIT"
},
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -2093,6 +2367,12 @@
"node": ">= 14.0.0"
}
},
"node_modules/tslib": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
"integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
"license": "0BSD"
},
"node_modules/type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@@ -2137,6 +2417,19 @@
"node": ">= 0.4.0"
}
},
"node_modules/uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
+1
View File
@@ -12,6 +12,7 @@
"license": "ISC",
"dependencies": {
"bcrypt": "^5.1.1",
"bullmq": "5.7.15",
"cors": "^2.8.5",
"dotenv": "^16.4.5",
"express": "^4.19.2",
+13
View File
@@ -0,0 +1,13 @@
const router = require("express").Router();
const queueController = require("../controllers/queueController");
// Get Jobs
router.get("/", queueController.getJobs);
// Add Job
router.post("/", queueController.addJob);
// Obliterate Queue
router.post("/obliterate", queueController.obliterateQueue);
module.exports = router;
+195
View File
@@ -0,0 +1,195 @@
const { Queue, Worker, Job } = require("bullmq");
const QUEUE_NAME = "monitors";
const connection = {
host: process.env.REDIS_HOST || "127.0.0.1",
port: process.env.REDIS_PORT || 6379,
};
const JOBS_PER_WORKER = 5;
const logger = require("../utils/logger");
class JobQueue {
/**
* Constructs a new JobQueue
* @constructor
* @throws {Error}
*/
constructor() {
console.log(process.env.REDIS_PORT);
this.queue = new Queue(QUEUE_NAME, {
connection,
});
this.workers = [];
}
/**
* Static factory method to create a JobQueue
* @static
* @async
* @returns {Promise<JobQueue>} - Returns a new JobQueue
*
*/
static async createJobQueue() {
const queue = new JobQueue();
try {
const workerStats = await queue.getWorkerStats();
await queue.scaleWorkers(workerStats);
return queue;
} catch (error) {
throw error;
}
}
/**
* Creates a worker for the queue
* Operations are carried out in the async callback
* @returns {Worker} The newly created worker
*/
createWorker() {
const worker = new Worker(
QUEUE_NAME,
async (job) => {
// TODO Ping a monitor
console.log(`${job.name} completed, workers: ${this.workers.length}`);
},
{
connection,
}
);
return worker;
}
/**
* @typedef {Object} WorkerStats
* @property {Array<Job>} jobs - Array of jobs in the Queue
* @property {number} - workerLoad - The number of jobs per worker
*
*/
/**
* Gets stats related to the workers
* This is used for scaling workers right now
* In the future we will likely want to scale based on server performance metrics
* CPU Usage & memory usage, if too high, scale down workers.
* When to scale up? If jobs are taking too long to complete?
* @async
* @returns {Promise<WorkerStats>} - Returns the worker stats
*/
async getWorkerStats() {
try {
const jobs = await this.queue.getRepeatableJobs();
const load = jobs.length / this.workers.length;
return { jobs, load };
} catch (error) {
throw error;
}
}
/**
* Scale Workers
* This function scales workers based on the load per worker
* If the load is higher than the JOBS_PER_WORKER threshold, we add more workers
* If the load is lower than the JOBS_PER_WORKER threshold, we release workers
* This approach ignores server performance, which we should add in the future
*
* @async
* @param {WorkerStats} workerStats - The payload for the job.
* @returns {Promise<boolean>}
*/
async scaleWorkers(workerStats) {
if (this.workers.length === 0) {
// There are no workers, need to add one
const worker = this.createWorker();
this.workers.push(worker);
return true;
}
if (workerStats.load > JOBS_PER_WORKER) {
// Find out how many more jobs we have than current workers can handle
const excessJobs =
workerStats.jobs.length - this.workers.length * JOBS_PER_WORKER;
// Divide by jobs/worker to find out how many workers to add
const workersToAdd = Math.ceil(excessJobs / JOBS_PER_WORKER);
for (let i = 0; i < workersToAdd; i++) {
const worker = this.createWorker();
this.workers.push(worker);
}
return true;
}
if (workerStats.load < JOBS_PER_WORKER) {
// Find out how much excess capacity we have
const workerCapacity = this.workers.length * JOBS_PER_WORKER;
const excessCapacity = workerCapacity - workerStats.jobs.length;
// Calculate how many workers to remove
const workersToRemove = Math.floor(excessCapacity / JOBS_PER_WORKER);
for (let i = 0; i < workersToRemove; i++) {
const worker = this.workers.pop();
try {
await worker.close();
} catch (error) {
// Catch the error instead of throwing it
console.error("Error closing worker", error);
}
}
return true;
}
return false;
}
/**
* Gets all jobs in the queue.
*
* @async
* @returns {Promise<Array<Job>>}
* @throws {Error} - Throws error if getting jobs fails
*/
async getJobs() {
try {
const jobs = await this.queue.getRepeatableJobs();
console.log("jobs", jobs);
return jobs;
} catch (error) {
throw error;
}
}
/**
* Adds a job to the queue and scales workers based on worker stats.
*
* @async
* @param {string} jobName - The name of the job to be added.
* @param {Monitor} payload - The payload for the job.
* @throws {Error} - Will throw an error if the job cannot be added or workers don't scale
*/
async addJob(jobName, payload) {
try {
await this.queue.add(jobName, payload, {
repeat: {
every: 1000,
limit: 100,
},
});
const workerStats = await this.getWorkerStats();
await this.scaleWorkers(workerStats);
} catch (error) {
throw error;
}
}
/**
* @async
* @returns {Promise<boolean>} - Returns true if obliteration is successful
*/
async obliterate() {
try {
await this.queue.obliterate();
return true;
} catch (error) {
throw error;
}
}
}
module.exports = JobQueue;