refactor service dir structure

This commit is contained in:
Alex Holliday committed 2025-07-23 13:03:57 -07:00
1 parent 0b4ec46f79
commit 06fa79e09d
28 files changed
+251 -42

No files matched your search

@@ -0,0 +1,317 @@
const QUEUE_NAMES = ["uptime", "pagespeed", "hardware"];
const SERVICE_NAME = "JobQueue";
const HEALTH_CHECK_INTERVAL = 10 * 60 * 1000; // 10 minutes
const QUEUE_LOOKUP = {
hardware: "hardware",
http: "uptime",
ping: "uptime",
port: "uptime",
docker: "uptime",
pagespeed: "pagespeed",
};
const getSchedulerId = (monitor) => `scheduler:${monitor.type}:${monitor._id}`;
class JobQueue {
static SERVICE_NAME = SERVICE_NAME;
constructor({ db, jobQueueHelper, logger, stringService }) {
this.db = db;
this.jobQueueHelper = jobQueueHelper;
this.stringService = stringService;
this.logger = logger;
this.queues = {};
this.workers = [];
}
static async create({ db, jobQueueHelper, logger, stringService }) {
const instance = new JobQueue({ db, jobQueueHelper, logger, stringService });
await instance.init();
return instance;
}
async init() {
try {
await this.initQueues();
await this.initWorkers();
const monitors = await this.db.getAllMonitors();
await Promise.all(
monitors
.filter((monitor) => monitor.isActive)
.map(async (monitor) => {
try {
await this.addJob(monitor._id, monitor);
} catch (error) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "initJobQueue",
stack: error.stack,
});
}
})
);
this.healthCheckInterval = setInterval(async () => {
try {
const queueHealthChecks = await this.checkQueueHealth();
const queueIsStuck = queueHealthChecks.some((healthCheck) => healthCheck.stuck);
if (queueIsStuck) {
this.logger.warn({
message: "Queue is stuck",
service: SERVICE_NAME,
method: "periodicHealthCheck",
details: queueHealthChecks,
});
await this.flushQueues();
}
} catch (error) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "periodicHealthCheck",
stack: error.stack,
});
}
}, HEALTH_CHECK_INTERVAL);
} catch (error) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "initJobQueue",
stack: error.stack,
});
}
}
async initQueues() {
const readyPromises = [];
for (const queueName of QUEUE_NAMES) {
const q = this.jobQueueHelper.createQueue(queueName);
this.queues[queueName] = q;
readyPromises.push(q.waitUntilReady());
}
await Promise.all(readyPromises);
this.logger.info({
message: "Queues ready",
service: SERVICE_NAME,
method: "initQueues",
});
}
async initWorkers() {
const workerReadyPromises = [];
for (const queueName of QUEUE_NAMES) {
const worker = this.jobQueueHelper.createWorker(queueName, this.queues[queueName]);
this.workers.push(worker);
workerReadyPromises.push(worker.waitUntilReady());
}
await Promise.all(workerReadyPromises);
this.logger.info({
message: "Workers ready",
service: SERVICE_NAME,
method: "initWorkers",
});
}
pauseJob = async (monitor) => {
this.deleteJob(monitor);
};
resumeJob = async (monitor) => {
this.addJob(monitor._id, monitor);
};
async addJob(jobName, monitor) {
this.logger.info({
message: `Adding job ${monitor?.url ?? "No URL"}`,
service: SERVICE_NAME,
method: "addJob",
});
const queueName = QUEUE_LOOKUP[monitor.type];
const queue = this.queues[queueName];
if (typeof queue === "undefined") {
throw new Error(`Queue for ${monitor.type} not found`);
}
const jobTemplate = {
name: jobName,
data: monitor,
opts: {
attempts: 1,
backoff: {
type: "exponential",
delay: 1000,
},
removeOnComplete: true,
removeOnFail: false,
timeout: 1 * 60 * 1000,
},
};
const schedulerId = getSchedulerId(monitor);
await queue.upsertJobScheduler(schedulerId, { every: monitor?.interval ?? 60000 }, jobTemplate);
}
async deleteJob(monitor) {
try {
const queue = this.queues[QUEUE_LOOKUP[monitor.type]];
const schedulerId = getSchedulerId(monitor);
const wasDeleted = await queue.removeJobScheduler(schedulerId);
if (wasDeleted === true) {
this.logger.info({
message: this.stringService.jobQueueDeleteJob,
service: SERVICE_NAME,
method: "deleteJob",
details: `Deleted job ${monitor._id}`,
});
return true;
} else {
this.logger.error({
message: this.stringService.jobQueueDeleteJob,
service: SERVICE_NAME,
method: "deleteJob",
details: `Failed to delete job ${monitor._id}`,
});
return false;
}
} catch (error) {
error.service === undefined ? (error.service = SERVICE_NAME) : null;
error.method === undefined ? (error.method = "deleteJob") : null;
throw error;
}
}
async updateJob(monitor) {
await this.deleteJob(monitor);
await this.addJob(monitor._id, monitor);
}
async getJobs() {
try {
let stats = {};
await Promise.all(
QUEUE_NAMES.map(async (name) => {
const queue = this.queues[name];
const jobs = await queue.getJobs();
const ret = await Promise.all(
jobs.map(async (job) => {
const state = await job.getState();
return { url: job.data.url, state, progress: job.progress };
})
);
stats[name] = { jobs: ret };
})
);
return stats;
} catch (error) {
error.service === undefined ? (error.service = SERVICE_NAME) : null;
error.method === undefined ? (error.method = "getJobStats") : null;
throw error;
}
}
async getMetrics() {
try {
let metrics = {};
await Promise.all(
QUEUE_NAMES.map(async (name) => {
const queue = this.queues[name];
const [waiting, active, failed, delayed, repeatableJobs] = await Promise.all([
queue.getWaitingCount(),
queue.getActiveCount(),
queue.getFailedCount(),
queue.getDelayedCount(),
queue.getRepeatableJobs(),
]);
metrics[name] = {
waiting,
active,
failed,
delayed,
repeatableJobs: repeatableJobs.length,
};
})
);
return metrics;
} catch (error) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "getMetrics",
stack: error.stack,
});
}
}
async checkQueueHealth() {
const res = [];
for (const queueName of QUEUE_NAMES) {
const q = this.queues[queueName];
await q.waitUntilReady();
const lastJobProcessedTime = q.lastJobProcessedTime;
const currentTime = Date.now();
const timeDiff = currentTime - lastJobProcessedTime;
// Check for jobs
const jobCounts = await q.getJobCounts();
const hasJobs = Object.values(jobCounts).some((count) => count > 0);
res.push({
queueName,
timeSinceLastJob: timeDiff,
stuck: hasJobs && timeDiff > 10000,
jobCounts,
});
}
return res;
}
async flushQueues() {
try {
this.logger.warn({
message: "Flushing queues",
method: "flushQueues",
service: SERVICE_NAME,
});
for (const worker of this.workers) {
await worker.close();
}
this.workers = [];
for (const queue of Object.values(this.queues)) {
await queue.obliterate();
}
this.queue = {};
await this.init();
return true;
} catch (error) {
this.logger.warn({
message: `${error.message} - Flushing redis manually`,
service: SERVICE_NAME,
method: "flushQueues",
});
return await this.jobQueueHelper.flushRedis();
}
}
async shutdown() {
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
this.healthCheckInterval = null;
}
for (const worker of this.workers) {
await worker.close();
}
for (const queue of Object.values(this.queues)) {
await queue.obliterate();
}
}
}
export default JobQueue;
@@ -0,0 +1,302 @@
const SERVICE_NAME = "JobQueueHelper";
class JobQueueHelper {
constructor({ redisService, Queue, Worker, logger, db, networkService, statusService, notificationService }) {
this.db = db;
this.redisService = redisService;
this.Queue = Queue;
this.Worker = Worker;
this.logger = logger;
this.networkService = networkService;
this.statusService = statusService;
this.notificationService = notificationService;
}
createQueue(queueName) {
const connection = this.redisService.getNewConnection();
const q = new this.Queue(queueName, {
connection,
});
q.lastJobProcessedTime = Date.now();
q.on("cleaned", (jobs, type) => {
this.logger.debug({
message: `Queue ${queueName} is cleaned with jobs: ${jobs} and type: ${type}`,
service: SERVICE_NAME,
method: "createQueue:cleaned",
});
});
q.on("error", (err) => {
this.logger.error({
message: `Queue ${queueName} is error with msg: ${err}`,
service: SERVICE_NAME,
method: "createQueue:error",
});
});
q.on("ioredis:close", () => {
this.logger.debug({
message: `Queue ${queueName} is ioredis:close`,
service: SERVICE_NAME,
method: "createQueue:ioredis:close",
});
});
q.on("paused", () => {
this.logger.debug({
message: `Queue ${queueName} is paused`,
service: SERVICE_NAME,
method: "createQueue:paused",
});
});
q.on("progress", (job, progress) => {
this.logger.debug({
message: `Queue ${queueName} is progress with msg: ${progress}`,
service: SERVICE_NAME,
method: "createQueue:progress",
});
});
q.on("removed", (job) => {
this.logger.debug({
message: `Queue ${queueName} is removed with msg: ${job}`,
service: SERVICE_NAME,
method: "createQueue:removed",
});
});
q.on("resumed", () => {
this.logger.debug({
message: `Queue ${queueName} is resumed`,
service: SERVICE_NAME,
method: "createQueue:resumed",
});
});
q.on("waiting", () => {
this.logger.debug({
message: `Queue ${queueName} is waiting`,
service: SERVICE_NAME,
method: "createQueue:waiting",
});
});
return q;
}
createWorker(queueName, queue) {
const connection = this.redisService.getNewConnection({
maxRetriesPerRequest: null,
});
const worker = new this.Worker(queueName, this.createJobHandler(queue), {
connection,
concurrency: 50,
});
worker.on("active", (job) => {
this.logger.debug({
message: `Worker ${queueName} is active`,
service: SERVICE_NAME,
method: "createWorker:active",
});
});
worker.on("closed", () => {
this.logger.debug({
message: `Worker ${queueName} is closed`,
service: SERVICE_NAME,
method: "createWorker:closed",
});
});
worker.on("closing", (msg) => {
this.logger.debug({
message: `Worker ${queueName} is closing with msg: ${msg}`,
service: SERVICE_NAME,
method: "createWorker:closing",
});
});
worker.on("completed", (job) => {
this.logger.debug({
message: `Worker ${queueName} is completed`,
service: SERVICE_NAME,
method: "createWorker:completed",
});
});
worker.on("drained", () => {
this.logger.debug({
message: `Worker ${queueName} is drained`,
service: SERVICE_NAME,
method: "createWorker:drained",
});
});
worker.on("error", (failedReason) => {
this.logger.error({
message: `Worker ${queueName} is error with msg: ${failedReason}`,
service: SERVICE_NAME,
method: "createWorker:error",
});
});
worker.on("failed", (job, error, prev) => {
this.logger.error({
message: `Worker ${queueName} is failed with msg: ${error.message}`,
service: error?.service ?? SERVICE_NAME,
method: error?.method ?? "createWorker:failed",
stack: error?.stack,
});
});
worker.on("ioredis:close", () => {
this.logger.debug({
message: `Worker ${queueName} is ioredis:close`,
service: SERVICE_NAME,
method: "createWorker:ioredis:close",
});
});
worker.on("paused", () => {
this.logger.debug({
message: `Worker ${queueName} is paused`,
service: SERVICE_NAME,
method: "createWorker:paused",
});
});
worker.on("progress", (job, progress) => {
this.logger.debug({
message: `Worker ${queueName} is progress with msg: ${progress}`,
service: SERVICE_NAME,
method: "createWorker:progress",
});
});
worker.on("ready", () => {
this.logger.debug({
message: `Worker ${queueName} is ready`,
service: SERVICE_NAME,
method: "createWorker:ready",
});
});
worker.on("resumed", () => {
this.logger.debug({
message: `Worker ${queueName} is resumed`,
service: SERVICE_NAME,
method: "createWorker:resumed",
});
});
worker.on("stalled", () => {
this.logger.warn({
message: `Worker ${queueName} is stalled`,
service: SERVICE_NAME,
method: "createWorker:stalled",
});
});
return worker;
}
async isInMaintenanceWindow(monitorId) {
const maintenanceWindows = await this.db.getMaintenanceWindowsByMonitorId(monitorId);
// Check for active maintenance window:
const maintenanceWindowIsActive = maintenanceWindows.reduce((acc, window) => {
if (window.active) {
const start = new Date(window.start);
const end = new Date(window.end);
const now = new Date();
const repeatInterval = window.repeat || 0;
// If start is < now and end > now, we're in maintenance
if (start <= now && end >= now) return true;
// If maintenance window was set in the past with a repeat,
// we need to advance start and end to see if we are in range
while (start < now && repeatInterval !== 0) {
start.setTime(start.getTime() + repeatInterval);
end.setTime(end.getTime() + repeatInterval);
if (start <= now && end >= now) {
return true;
}
}
return false;
}
return acc;
}, false);
return maintenanceWindowIsActive;
}
createJobHandler(q) {
return async (job) => {
try {
// Update the last job processed time for this queue
q.lastJobProcessedTime = Date.now();
// Get all maintenance windows for this monitor
await job.updateProgress(0);
const monitorId = job.data._id;
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId);
// If a maintenance window is active, we're done
if (maintenanceWindowActive) {
await job.updateProgress(100);
this.logger.info({
message: `Monitor ${monitorId} is in maintenance window`,
service: SERVICE_NAME,
method: "createWorker",
});
return false;
}
// Get the current status
await job.updateProgress(30);
const monitor = job.data;
const networkResponse = await this.networkService.getStatus(monitor);
// If the network response is not found, we're done
if (!networkResponse) {
await job.updateProgress(100);
return false;
}
// Handle status change
await job.updateProgress(60);
const { monitor: updatedMonitor, statusChanged, prevStatus } = await this.statusService.updateStatus(networkResponse);
// Handle notifications
await job.updateProgress(80);
this.notificationService
.handleNotifications({
...networkResponse,
monitor: updatedMonitor,
prevStatus,
statusChanged,
})
.catch((error) => {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "createJobHandler",
details: `Error sending notifications for job ${job.id}: ${error.message}`,
stack: error.stack,
});
});
await job.updateProgress(100);
return true;
} catch (error) {
await job.updateProgress(100);
throw error;
}
};
}
async flushRedis() {
try {
const connection = this.redisService.getNewConnection();
const flushResult = await connection.flushall();
return flushResult;
} catch (error) {
this.logger.warn({
message: error.message,
service: SERVICE_NAME,
method: "flushRedis",
});
return false;
}
}
}
export default JobQueueHelper;