Added queue service, Added controller for testing purposes
This commit is contained in:
1 parent
86130ac57e
commit
88b0b0f97c
6 files changed
+632
-75
No files matched your search
@@ -0,0 +1,42 @@
|
||||
const express = require("express");
|
||||
const logger = require("../utils/logger");
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
|
||||
const getJobs = async (req, res, next) => {
|
||||
try {
|
||||
const jobs = await req.jobQueue.getJobs();
|
||||
return res.status(200).json({ jobs });
|
||||
} catch (error) {
|
||||
error.service = SERVICE_NAME;
|
||||
next(error);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const addJob = async (req, res, next) => {
|
||||
try {
|
||||
await req.jobQueue.addJob(Math.random().toString(36).substring(7));
|
||||
return res.send("Added job");
|
||||
} catch (error) {
|
||||
error.service = SERVICE_NAME;
|
||||
next(error);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const obliterateQueue = async (req, res, next) => {
|
||||
try {
|
||||
const obliterated = await req.jobQueue.obliterate();
|
||||
return res.status(200).send("Obliterated jobs");
|
||||
} catch (error) {
|
||||
error.service = SERVICE_NAME;
|
||||
next(error);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getJobs,
|
||||
addJob,
|
||||
obliterateQueue,
|
||||
};
|
||||
+88
-75
@@ -10,84 +10,97 @@ require("dotenv").config();
|
||||
const logger = require("./utils/logger");
|
||||
const { verifyJWT } = require("./middleware/verifyJWT");
|
||||
const { handleErrors } = require("./middleware/handleErrors");
|
||||
const queueRouter = require("./routes/queueRoute");
|
||||
const JobQueue = require("./service/jobQueue");
|
||||
|
||||
// const { sendEmail } = require('./utils/sendEmail')
|
||||
// Need to wrap server setup in a function to handle async nature of JobQueue
|
||||
const startApp = async () => {
|
||||
// const { sendEmail } = require('./utils/sendEmail')
|
||||
|
||||
// **************************
|
||||
// Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out.
|
||||
// Simply comment out the FakeDB and uncomment the MongoDB or vice versa.
|
||||
// We can easily swap between any type of data source as long as the methods are implemented
|
||||
//
|
||||
// FakeDB
|
||||
// const db = require("./db/FakeDb");
|
||||
//
|
||||
// MongoDB
|
||||
// const db = require("./db/MongoDB");
|
||||
//
|
||||
// **************************
|
||||
// **************************
|
||||
// Here is where we can swap out DBs easily. Spin up a mongoDB instance and try it out.
|
||||
// Simply comment out the FakeDB and uncomment the MongoDB or vice versa.
|
||||
// We can easily swap between any type of data source as long as the methods are implemented
|
||||
//
|
||||
// FakeDB
|
||||
// const db = require("./db/FakeDb");
|
||||
//
|
||||
// MongoDB
|
||||
// const db = require("./db/MongoDB");
|
||||
//
|
||||
// **************************
|
||||
const DB_TYPE = {
|
||||
MongoDB: () => require("./db/MongoDB"),
|
||||
FakedDB: () => require("./db/FakeDb"),
|
||||
};
|
||||
|
||||
const DB_TYPE = {
|
||||
MongoDB: () => require("./db/MongoDB"),
|
||||
FakedDB: () => require("./db/FakeDb"),
|
||||
const db = DB_TYPE[process.env.DB_TYPE]
|
||||
? DB_TYPE[process.env.DB_TYPE]()
|
||||
: require("./db/FakeDb");
|
||||
|
||||
const jobQueue = await JobQueue.createJobQueue();
|
||||
/**
|
||||
* NOTES
|
||||
* Email Service will be added
|
||||
* Logger Service will be added (Winston or similar)
|
||||
*/
|
||||
|
||||
const app = express();
|
||||
|
||||
// middlewares
|
||||
app.use(
|
||||
cors()
|
||||
//We will add configuration later
|
||||
);
|
||||
app.use(express.json());
|
||||
app.use(helmet());
|
||||
|
||||
// **************************
|
||||
// Make DB accessible anywhere we have a Request object
|
||||
// By adding the DB to the request object, we can access it in any route
|
||||
// Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it
|
||||
// Same applies for JobQueue
|
||||
// **************************
|
||||
app.use((req, res, next) => {
|
||||
req.db = db;
|
||||
req.jobQueue = jobQueue;
|
||||
next();
|
||||
});
|
||||
|
||||
//routes
|
||||
app.use("/api/v1/auth", authRouter);
|
||||
app.use("/api/v1/monitors", monitorRouter);
|
||||
app.use("/api/v1/checks", verifyJWT, checkRouter);
|
||||
app.use("/api/v1/alerts", verifyJWT, alertRouter);
|
||||
//Temporary route for testing, remove later
|
||||
app.use("/api/v1/job", queueRouter);
|
||||
|
||||
// Testing email service
|
||||
// app.use('/sendEmail', async (req, res) => {
|
||||
// const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', '<h1>Testing Bluewavelabs</h1>');
|
||||
// console.log(response);
|
||||
// })
|
||||
|
||||
//health check
|
||||
app.use("/api/v1/healthy", (req, res) => {
|
||||
try {
|
||||
logger.info("Checking Health of the server.");
|
||||
return res.status(200).json({ message: "Healthy" });
|
||||
} catch (error) {
|
||||
logger.error(error.message);
|
||||
return res.status(500).json({ message: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Error handler middleware
|
||||
* Should be called last
|
||||
*/
|
||||
app.use(handleErrors);
|
||||
|
||||
connectDbAndRunServer(app, db);
|
||||
};
|
||||
|
||||
const db = DB_TYPE[process.env.DB_TYPE]
|
||||
? DB_TYPE[process.env.DB_TYPE]()
|
||||
: require("./db/FakeDb");
|
||||
|
||||
/**
|
||||
* NOTES
|
||||
* Email Service will be added
|
||||
* Logger Service will be added (Winston or similar)
|
||||
*/
|
||||
|
||||
const app = express();
|
||||
|
||||
// middlewares
|
||||
app.use(
|
||||
cors()
|
||||
//We will add configuration later
|
||||
);
|
||||
app.use(express.json());
|
||||
app.use(helmet());
|
||||
|
||||
// **************************
|
||||
// Make DB accessible anywhere we have a Request object
|
||||
// By adding the DB to the request object, we can access it in any route
|
||||
// Thus we do not need to import it in every route file, and we can easily swap out DBs as there is only one place to change it
|
||||
// **************************
|
||||
app.use((req, res, next) => {
|
||||
req.db = db;
|
||||
next();
|
||||
startApp().catch((error) => {
|
||||
console.log(error);
|
||||
});
|
||||
|
||||
//routes
|
||||
app.use("/api/v1/auth", authRouter);
|
||||
app.use("/api/v1/monitors", monitorRouter);
|
||||
app.use("/api/v1/checks", verifyJWT, checkRouter);
|
||||
app.use("/api/v1/alerts", verifyJWT, alertRouter);
|
||||
|
||||
// Testing email service
|
||||
// app.use('/sendEmail', async (req, res) => {
|
||||
// const response = sendEmail(['veysel.boybay@bluewavelabs.ca'], 'Testing email service', '<h1>Testing Bluewavelabs</h1>');
|
||||
// console.log(response);
|
||||
// })
|
||||
|
||||
//health check
|
||||
app.use("/api/v1/healthy", (req, res) => {
|
||||
try {
|
||||
logger.info("Checking Health of the server.");
|
||||
return res.status(200).json({ message: "Healthy" });
|
||||
} catch (error) {
|
||||
logger.error(error.message);
|
||||
return res.status(500).json({ message: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Error handler middleware
|
||||
* Should be called last
|
||||
*/
|
||||
app.use(handleErrors);
|
||||
|
||||
connectDbAndRunServer(app, db);
|
||||
Generated
+293
@@ -10,6 +10,7 @@
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"bcrypt": "^5.1.1",
|
||||
"bullmq": "5.7.15",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.19.2",
|
||||
@@ -55,6 +56,12 @@
|
||||
"@hapi/hoek": "^9.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@ioredis/commands": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
|
||||
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@mapbox/node-pre-gyp": {
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
|
||||
@@ -96,6 +103,84 @@
|
||||
"sparse-bitfield": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz",
|
||||
"integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz",
|
||||
"integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz",
|
||||
"integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz",
|
||||
"integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz",
|
||||
"integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz",
|
||||
"integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@sideway/address": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz",
|
||||
@@ -316,6 +401,21 @@
|
||||
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
|
||||
"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
|
||||
},
|
||||
"node_modules/bullmq": {
|
||||
"version": "5.7.15",
|
||||
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.15.tgz",
|
||||
"integrity": "sha512-XR5sTA8BPUY67sS37sMKGCDvSLaVpMq7aaQG8FGSKOUnPoJMRf17n1TibVWP3+yK0xKLdK5Y7PY9D874Fpeqpg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cron-parser": "^4.6.0",
|
||||
"ioredis": "^5.4.1",
|
||||
"msgpackr": "^1.10.1",
|
||||
"node-abort-controller": "^3.1.1",
|
||||
"semver": "^7.5.4",
|
||||
"tslib": "^2.0.0",
|
||||
"uuid": "^9.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bytes": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
|
||||
@@ -374,6 +474,15 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/cluster-key-slot": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
|
||||
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/color": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz",
|
||||
@@ -476,6 +585,18 @@
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/cron-parser": {
|
||||
"version": "4.9.0",
|
||||
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
|
||||
"integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"luxon": "^3.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||
@@ -505,6 +626,15 @@
|
||||
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
|
||||
"integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ=="
|
||||
},
|
||||
"node_modules/denque": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
|
||||
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/depd": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
|
||||
@@ -992,6 +1122,53 @@
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
},
|
||||
"node_modules/ioredis": {
|
||||
"version": "5.4.1",
|
||||
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz",
|
||||
"integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@ioredis/commands": "^1.1.1",
|
||||
"cluster-key-slot": "^1.1.0",
|
||||
"debug": "^4.3.4",
|
||||
"denque": "^2.1.0",
|
||||
"lodash.defaults": "^4.2.0",
|
||||
"lodash.isarguments": "^3.1.0",
|
||||
"redis-errors": "^1.2.0",
|
||||
"redis-parser": "^3.0.0",
|
||||
"standard-as-callback": "^2.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.22.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/ioredis"
|
||||
}
|
||||
},
|
||||
"node_modules/ioredis/node_modules/debug": {
|
||||
"version": "4.3.5",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz",
|
||||
"integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "2.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"supports-color": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/ioredis/node_modules/ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ipaddr.js": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
|
||||
@@ -1145,11 +1322,23 @@
|
||||
"resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz",
|
||||
"integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A=="
|
||||
},
|
||||
"node_modules/lodash.defaults": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
|
||||
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.includes": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
|
||||
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w=="
|
||||
},
|
||||
"node_modules/lodash.isarguments": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
|
||||
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isboolean": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
|
||||
@@ -1212,6 +1401,15 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/luxon": {
|
||||
"version": "3.4.4",
|
||||
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
|
||||
"integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/mailersend": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mailersend/-/mailersend-2.2.0.tgz",
|
||||
@@ -1478,6 +1676,37 @@
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
|
||||
},
|
||||
"node_modules/msgpackr": {
|
||||
"version": "1.10.2",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz",
|
||||
"integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==",
|
||||
"license": "MIT",
|
||||
"optionalDependencies": {
|
||||
"msgpackr-extract": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/msgpackr-extract": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz",
|
||||
"integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"node-gyp-build-optional-packages": "5.0.7"
|
||||
},
|
||||
"bin": {
|
||||
"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2",
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2",
|
||||
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/negotiator": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
|
||||
@@ -1486,6 +1715,12 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/node-abort-controller": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz",
|
||||
"integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-addon-api": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
|
||||
@@ -1529,6 +1764,18 @@
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp-build-optional-packages": {
|
||||
"version": "5.0.7",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz",
|
||||
"integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"bin": {
|
||||
"node-gyp-build-optional-packages": "bin.js",
|
||||
"node-gyp-build-optional-packages-optional": "optional.js",
|
||||
"node-gyp-build-optional-packages-test": "build-test.js"
|
||||
}
|
||||
},
|
||||
"node_modules/nodemon": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.0.tgz",
|
||||
@@ -1778,6 +2025,27 @@
|
||||
"node": ">=8.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-errors": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
|
||||
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-parser": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
|
||||
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"redis-errors": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||
@@ -1969,6 +2237,12 @@
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/standard-as-callback": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
|
||||
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/statuses": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
|
||||
@@ -2093,6 +2367,12 @@
|
||||
"node": ">= 14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.6.3",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
|
||||
"integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/type-is": {
|
||||
"version": "1.6.18",
|
||||
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
|
||||
@@ -2137,6 +2417,19 @@
|
||||
"node": ">= 0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "9.0.1",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
|
||||
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/vary": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"bcrypt": "^5.1.1",
|
||||
"bullmq": "5.7.15",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.19.2",
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
const router = require("express").Router();
|
||||
const queueController = require("../controllers/queueController");
|
||||
|
||||
// Get Jobs
|
||||
router.get("/", queueController.getJobs);
|
||||
|
||||
// Add Job
|
||||
router.post("/", queueController.addJob);
|
||||
|
||||
// Obliterate Queue
|
||||
router.post("/obliterate", queueController.obliterateQueue);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,195 @@
|
||||
const { Queue, Worker, Job } = require("bullmq");
|
||||
const QUEUE_NAME = "monitors";
|
||||
const connection = {
|
||||
host: process.env.REDIS_HOST || "127.0.0.1",
|
||||
port: process.env.REDIS_PORT || 6379,
|
||||
};
|
||||
const JOBS_PER_WORKER = 5;
|
||||
const logger = require("../utils/logger");
|
||||
|
||||
class JobQueue {
|
||||
/**
|
||||
* Constructs a new JobQueue
|
||||
* @constructor
|
||||
* @throws {Error}
|
||||
*/
|
||||
constructor() {
|
||||
console.log(process.env.REDIS_PORT);
|
||||
this.queue = new Queue(QUEUE_NAME, {
|
||||
connection,
|
||||
});
|
||||
this.workers = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Static factory method to create a JobQueue
|
||||
* @static
|
||||
* @async
|
||||
* @returns {Promise<JobQueue>} - Returns a new JobQueue
|
||||
*
|
||||
*/
|
||||
static async createJobQueue() {
|
||||
const queue = new JobQueue();
|
||||
try {
|
||||
const workerStats = await queue.getWorkerStats();
|
||||
await queue.scaleWorkers(workerStats);
|
||||
return queue;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a worker for the queue
|
||||
* Operations are carried out in the async callback
|
||||
* @returns {Worker} The newly created worker
|
||||
*/
|
||||
createWorker() {
|
||||
const worker = new Worker(
|
||||
QUEUE_NAME,
|
||||
async (job) => {
|
||||
// TODO Ping a monitor
|
||||
console.log(`${job.name} completed, workers: ${this.workers.length}`);
|
||||
},
|
||||
{
|
||||
connection,
|
||||
}
|
||||
);
|
||||
return worker;
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} WorkerStats
|
||||
* @property {Array<Job>} jobs - Array of jobs in the Queue
|
||||
* @property {number} - workerLoad - The number of jobs per worker
|
||||
*
|
||||
*/
|
||||
|
||||
/**
|
||||
* Gets stats related to the workers
|
||||
* This is used for scaling workers right now
|
||||
* In the future we will likely want to scale based on server performance metrics
|
||||
* CPU Usage & memory usage, if too high, scale down workers.
|
||||
* When to scale up? If jobs are taking too long to complete?
|
||||
* @async
|
||||
* @returns {Promise<WorkerStats>} - Returns the worker stats
|
||||
*/
|
||||
async getWorkerStats() {
|
||||
try {
|
||||
const jobs = await this.queue.getRepeatableJobs();
|
||||
const load = jobs.length / this.workers.length;
|
||||
return { jobs, load };
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale Workers
|
||||
* This function scales workers based on the load per worker
|
||||
* If the load is higher than the JOBS_PER_WORKER threshold, we add more workers
|
||||
* If the load is lower than the JOBS_PER_WORKER threshold, we release workers
|
||||
* This approach ignores server performance, which we should add in the future
|
||||
*
|
||||
|
||||
* @async
|
||||
* @param {WorkerStats} workerStats - The payload for the job.
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async scaleWorkers(workerStats) {
|
||||
if (this.workers.length === 0) {
|
||||
// There are no workers, need to add one
|
||||
const worker = this.createWorker();
|
||||
this.workers.push(worker);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (workerStats.load > JOBS_PER_WORKER) {
|
||||
// Find out how many more jobs we have than current workers can handle
|
||||
const excessJobs =
|
||||
workerStats.jobs.length - this.workers.length * JOBS_PER_WORKER;
|
||||
|
||||
// Divide by jobs/worker to find out how many workers to add
|
||||
const workersToAdd = Math.ceil(excessJobs / JOBS_PER_WORKER);
|
||||
for (let i = 0; i < workersToAdd; i++) {
|
||||
const worker = this.createWorker();
|
||||
this.workers.push(worker);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (workerStats.load < JOBS_PER_WORKER) {
|
||||
// Find out how much excess capacity we have
|
||||
const workerCapacity = this.workers.length * JOBS_PER_WORKER;
|
||||
const excessCapacity = workerCapacity - workerStats.jobs.length;
|
||||
// Calculate how many workers to remove
|
||||
const workersToRemove = Math.floor(excessCapacity / JOBS_PER_WORKER);
|
||||
for (let i = 0; i < workersToRemove; i++) {
|
||||
const worker = this.workers.pop();
|
||||
try {
|
||||
await worker.close();
|
||||
} catch (error) {
|
||||
// Catch the error instead of throwing it
|
||||
console.error("Error closing worker", error);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all jobs in the queue.
|
||||
*
|
||||
* @async
|
||||
* @returns {Promise<Array<Job>>}
|
||||
* @throws {Error} - Throws error if getting jobs fails
|
||||
*/
|
||||
async getJobs() {
|
||||
try {
|
||||
const jobs = await this.queue.getRepeatableJobs();
|
||||
console.log("jobs", jobs);
|
||||
return jobs;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a job to the queue and scales workers based on worker stats.
|
||||
*
|
||||
* @async
|
||||
* @param {string} jobName - The name of the job to be added.
|
||||
* @param {Monitor} payload - The payload for the job.
|
||||
* @throws {Error} - Will throw an error if the job cannot be added or workers don't scale
|
||||
*/
|
||||
async addJob(jobName, payload) {
|
||||
try {
|
||||
await this.queue.add(jobName, payload, {
|
||||
repeat: {
|
||||
every: 1000,
|
||||
limit: 100,
|
||||
},
|
||||
});
|
||||
const workerStats = await this.getWorkerStats();
|
||||
await this.scaleWorkers(workerStats);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @async
|
||||
* @returns {Promise<boolean>} - Returns true if obliteration is successful
|
||||
*/
|
||||
async obliterate() {
|
||||
try {
|
||||
await this.queue.obliterate();
|
||||
return true;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = JobQueue;
|
||||
Reference in new issue
Block a user