initial Pulse implementation

This commit is contained in:
Alex Holliday committed 2025-06-06 14:05:59 +08:00
1 parent 1f9fbf7115
commit 9a22d4a971
12 files changed
+455 -72

No files matched your search

+8 -1
View File
@@ -51,7 +51,6 @@ class JobQueue {
}
})
);
this.healthCheckInterval = setInterval(async () => {
try {
const queueHealthChecks = await this.checkQueueHealth();
@@ -116,6 +115,14 @@ class JobQueue {
});
}
pauseJob = async (monitor) => {
this.deleteJob(monitor);
};
resumeJob = async (monitor) => {
this.addJob(monitor._id, monitor);
};
async addJob(jobName, monitor) {
this.logger.info({
message: `Adding job ${monitor?.url ?? "No URL"}`,
+8 -4
View File
@@ -254,7 +254,8 @@ class JobQueueHelper {
// Get the current status
await job.updateProgress(30);
const networkResponse = await this.networkService.getStatus(job);
const monitor = job.data;
const networkResponse = await this.networkService.getStatus(monitor);
if (
job.data.type === "distributed_http" ||
job.data.type === "distributed_test"
@@ -271,14 +272,17 @@ class JobQueueHelper {
// Handle status change
await job.updateProgress(60);
const { monitor, statusChanged, prevStatus } =
await this.statusService.updateStatus(networkResponse);
const {
monitor: updatedMonitor,
statusChanged,
prevStatus,
} = await this.statusService.updateStatus(networkResponse);
// Handle notifications
await job.updateProgress(80);
this.notificationService
.handleNotifications({
...networkResponse,
monitor,
monitor: updatedMonitor,
prevStatus,
statusChanged,
})
+171
View File
@@ -0,0 +1,171 @@
import { Pulse } from "@pulsecron/pulse";
import { ObjectId } from "mongodb";
const SERVICE_NAME = "PulseQueue";
class PulseQueue {
static SERVICE_NAME = SERVICE_NAME;
constructor({ appSettings, db, pulseQueueHelper, logger }) {
this.db = db;
this.appSettings = appSettings;
this.pulseQueueHelper = pulseQueueHelper;
this.logger = logger;
this.init();
}
// ****************************************
// Core methods
// ****************************************
init = async () => {
try {
const mongoConnectionString =
this.appSettings.dbConnectionString || "mongodb://localhost:27017/uptime_db";
this.pulse = new Pulse({ db: { address: mongoConnectionString } });
await this.pulse.start();
this.pulse.define("monitor-job", this.pulseQueueHelper.getMonitorJob(), {});
const monitors = await this.db.getAllMonitors();
for (const monitor of monitors) {
await this.addJob(monitor._id, monitor);
}
return true;
} catch (error) {
this.logger.error({
message: "Failed to initialize PulseQueue",
service: SERVICE_NAME,
method: "init",
details: error,
});
return false;
}
};
addJob = async (monitorId, monitor) => {
this.logger.debug({
message: `Adding job ${monitor?.url ?? "No URL"}`,
service: SERVICE_NAME,
method: "addJob",
});
const intervalInSeconds = monitor.interval / 1000;
const job = this.pulse.create("monitor-job", {
monitor,
});
job.unique({ "data.monitor._id": monitor._id });
job.attrs.jobId = monitorId.toString();
job.repeatEvery(`${intervalInSeconds} seconds`);
await job.save();
};
deleteJob = async (monitor) => {
this.logger.debug({
message: `Deleting job ${monitor?.url ?? "No URL"}`,
service: SERVICE_NAME,
method: "deleteJob",
});
const result = await this.pulse.cancel({
"data.monitor._id": monitor._id,
});
console.log(result);
};
pauseJob = async (monitor) => {
const result = await this.pulse.disable({
"data.monitor._id": monitor._id,
});
if (result.length < 1) {
throw new Error("Failed to pause monitor");
}
this.logger.debug({
message: `Paused monitor ${monitor._id}`,
service: SERVICE_NAME,
method: "pauseJob",
});
};
resumeJob = async (monitor) => {
const result = await this.pulse.enable({
"data.monitor._id": monitor._id,
});
if (result.length < 1) {
throw new Error("Failed to resume monitor");
}
this.logger.debug({
message: `Resumed monitor ${monitor._id}`,
service: SERVICE_NAME,
method: "resumeJob",
});
};
shutdown = async () => {
this.logger.info({
message: "Shutting down JobQueue",
service: SERVICE_NAME,
method: "shutdown",
});
await this.pulse.stop();
};
// ****************************************
// Diagnostic methods
// ****************************************
getMetrics = async () => {
const jobs = await this.pulse.jobs();
const metrics = jobs.reduce(
(acc, job) => {
acc.jobs++;
if (job.attrs.failCount > 0 && job.attrs.failedAt >= job.attrs.lastFinishedAt) {
acc.failingJobs++;
}
if (job.attrs.lockedAt) {
acc.activeJobs++;
}
if (job.attrs.failCount > 0) {
acc.jobsWithFailures.push({
monitorId: job.attrs.data.monitor._id,
monitorUrl: job.attrs.data.monitor.url,
failCount: job.attrs.failCount,
failReason: job.attrs.failReason,
});
}
return acc;
},
{ jobs: 0, activeJobs: 0, failingJobs: 0, jobsWithFailures: [] }
);
return metrics;
};
getJobs = async () => {
const jobs = await this.pulse.jobs();
return jobs.map((job) => {
return {
monitorId: job.attrs.data.monitor._id,
monitorUrl: job.attrs.data.monitor.url,
lockedAt: job.attrs.lockedAt,
runCount: job.attrs.runCount || 0,
failCount: job.attrs.failCount || 0,
failReason: job.attrs.failReason,
};
});
};
flushQueues = async () => {
const cancelRes = await this.pulse.cancel();
await this.pulse.stop();
const initRes = await this.init();
return {
flushedJobs: cancelRes,
initSuccess: initRes,
};
};
obliterate = async () => {
await this.flushQueues();
};
}
export default PulseQueue;
@@ -0,0 +1,106 @@
const SERVICE_NAME = "PulseQueueHelper";
class PulseQueueHelper {
constructor({ db, logger, networkService, statusService, notificationService }) {
this.db = db;
this.logger = logger;
this.networkService = networkService;
this.statusService = statusService;
this.notificationService = notificationService;
}
getMonitorJob = () => {
return async (job) => {
try {
const monitor = job.attrs.data.monitor;
const monitorId = job.attrs.data.monitor._id;
if (!monitorId) {
throw new Error("No monitor id");
}
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId);
if (maintenanceWindowActive) {
this.logger.info({
message: `Monitor ${monitorId} is in maintenance window`,
service: SERVICE_NAME,
method: "getMonitorJob",
});
return;
}
const networkResponse = await this.networkService.getStatus(monitor);
if (monitor.type === "distributed_http" || monitor.type === "distributed_test") {
await job.updateProgress(100);
return true;
}
if (!networkResponse) {
throw new Error("No network response");
}
const {
monitor: updatedMonitor,
statusChanged,
prevStatus,
} = await this.statusService.updateStatus(networkResponse);
this.notificationService
.handleNotifications({
...networkResponse,
monitor: updatedMonitor,
prevStatus,
statusChanged,
})
.catch((error) => {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "createJobHandler",
details: `Error sending notifications for job ${job.id}: ${error.message}`,
stack: error.stack,
});
});
} catch (error) {
this.logger.warn({
message: error.message,
service: error.service || SERVICE_NAME,
method: error.method || "getMonitorJob",
stack: error.stack,
});
throw error;
}
};
};
async isInMaintenanceWindow(monitorId) {
const maintenanceWindows = await this.db.getMaintenanceWindowsByMonitorId(monitorId);
// Check for active maintenance window:
const maintenanceWindowIsActive = maintenanceWindows.reduce((acc, window) => {
if (window.active) {
const start = new Date(window.start);
const end = new Date(window.end);
const now = new Date();
const repeatInterval = window.repeat || 0;
// If start is < now and end > now, we're in maintenance
if (start <= now && end >= now) return true;
// If maintenance window was set in the past with a repeat,
// we need to advance start and end to see if we are in range
while (start < now && repeatInterval !== 0) {
start.setTime(start.getTime() + repeatInterval);
end.setTime(end.getTime() + repeatInterval);
if (start <= now && end >= now) {
return true;
}
}
return false;
}
return acc;
}, false);
return maintenanceWindowIsActive;
}
}
export default PulseQueueHelper;
+28 -29
View File
@@ -125,7 +125,7 @@ class NetworkService {
* @property {number} code - The response code (200 if successful, error code otherwise).
* @property {string} message - The message indicating the result of the HTTP request.
*/
async requestHttp(job) {
async requestHttp(monitor) {
try {
const {
url,
@@ -138,7 +138,7 @@ class NetworkService {
jsonPath,
matchMethod,
expectedValue,
} = job.data;
} = monitor;
const config = {};
secret !== undefined && (config.headers = { Authorization: `Bearer ${secret}` });
@@ -248,10 +248,10 @@ class NetworkService {
* @property {number} code - The response code (200 if successful, error code otherwise).
* @property {string} message - The message indicating the result of the PageSpeed request.
*/
async requestPagespeed(job) {
async requestPagespeed(monitor) {
try {
const url = job.data.url;
const updatedJob = { ...job };
const url = monitor.url;
const updatedMonitor = { ...monitor };
let pagespeedUrl = `https://pagespeedonline.googleapis.com/pagespeedonline/v5/runPagespeed?url=${url}&category=seo&category=accessibility&category=best-practices&category=performance`;
const dbSettings = await this.settingsService.getDBSettings();
@@ -266,8 +266,8 @@ class NetworkService {
});
return;
}
updatedJob.data.url = pagespeedUrl;
return await this.requestHttp(updatedJob);
updatedMonitor.url = pagespeedUrl;
return await this.requestHttp(updatedMonitor);
} catch (error) {
error.service = this.SERVICE_NAME;
error.method = "requestPagespeed";
@@ -292,9 +292,9 @@ class NetworkService {
* @property {number} code - The response code (200 if successful, error code otherwise).
* @property {string} message - The message indicating the result of the hardware status request.
*/
async requestHardware(job) {
async requestHardware(monitor) {
try {
return await this.requestHttp(job);
return await this.requestHttp(monitor);
} catch (error) {
error.service = this.SERVICE_NAME;
error.method = "requestHardware";
@@ -318,7 +318,7 @@ class NetworkService {
* @property {number} code - The response code (200 if successful, error code otherwise).
* @property {string} message - The message indicating the result of the Docker inspection.
*/
async requestDocker(job) {
async requestDocker(monitor) {
try {
const docker = new this.Docker({
socketPath: "/var/run/docker.sock",
@@ -326,19 +326,19 @@ class NetworkService {
});
const containers = await docker.listContainers({ all: true });
const containerExists = containers.some((c) => c.Id.startsWith(job.data.url));
const containerExists = containers.some((c) => c.Id.startsWith(monitor.url));
if (!containerExists) {
throw new Error(this.stringService.dockerNotFound);
}
const container = docker.getContainer(job.data.url);
const container = docker.getContainer(monitor.url);
const { response, responseTime, error } = await this.timeRequest(() =>
container.inspect()
);
const dockerResponse = {
monitorId: job.data._id,
type: job.data.type,
monitorId: monitor._id,
type: monitor.type,
responseTime,
};
@@ -360,9 +360,9 @@ class NetworkService {
}
}
async requestPort(job) {
async requestPort(monitor) {
try {
const { url, port } = job.data;
const { url, port } = monitor;
const { response, responseTime, error } = await this.timeRequest(async () => {
return new Promise((resolve, reject) => {
const socket = this.net.createConnection(
@@ -391,8 +391,8 @@ class NetworkService {
});
const portResponse = {
monitorId: job.data._id,
type: job.data.type,
monitorId: monitor._id,
type: monitor.type,
responseTime,
};
@@ -414,9 +414,8 @@ class NetworkService {
}
}
async requestDistributedHttp(job) {
async requestDistributedHttp(monitor) {
try {
const monitor = job.data;
const CALLBACK_URL = process.env.CALLBACK_URL;
const response = await this.axios.post(
@@ -503,23 +502,23 @@ class NetworkService {
* @returns {Promise<Object>} The response object from the appropriate request method.
* @throws {Error} Throws an error if the job type is unsupported.
*/
async getStatus(job) {
const type = job?.data?.type ?? "unknown";
async getStatus(monitor) {
const type = monitor.type ?? "unknown";
switch (type) {
case this.TYPE_PING:
return await this.requestPing(job);
return await this.requestPing(monitor);
case this.TYPE_HTTP:
return await this.requestHttp(job);
return await this.requestHttp(monitor);
case this.TYPE_PAGESPEED:
return await this.requestPagespeed(job);
return await this.requestPagespeed(monitor);
case this.TYPE_HARDWARE:
return await this.requestHardware(job);
return await this.requestHardware(monitor);
case this.TYPE_DOCKER:
return await this.requestDocker(job);
return await this.requestDocker(monitor);
case this.TYPE_PORT:
return await this.requestPort(job);
return await this.requestPort(monitor);
case this.TYPE_DISTRIBUTED_HTTP:
return await this.requestDistributedHttp(job);
return await this.requestDistributedHttp(monitor);
case this.TYPE_DISTRIBUTED_TEST:
return;
default:
+11 -3
View File
@@ -228,6 +228,12 @@ class StatusService {
if (type === "pagespeed") {
if (typeof payload === "undefined") {
this.logger.warn({
message: "Failed to build check",
service: this.SERVICE_NAME,
method: "buildCheck",
details: "empty payload",
});
return undefined;
}
const categories = payload?.lighthouseResult?.categories ?? {};
@@ -287,9 +293,11 @@ class StatusService {
} catch (error) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "insertCheck",
details: `Error inserting check for monitor: ${networkResponse?.monitorId}`,
service: error.service || this.SERVICE_NAME,
method: error.method || "insertCheck",
details:
error.details ||
`Error inserting check for monitor: ${networkResponse?.monitorId}`,
stack: error.stack,
});
}