initial Pulse implementation
This commit is contained in:
1 parent
1f9fbf7115
commit
9a22d4a971
12 files changed
+455
-72
No files matched your search
@@ -51,7 +51,6 @@ class JobQueue {
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
this.healthCheckInterval = setInterval(async () => {
|
||||
try {
|
||||
const queueHealthChecks = await this.checkQueueHealth();
|
||||
@@ -116,6 +115,14 @@ class JobQueue {
|
||||
});
|
||||
}
|
||||
|
||||
pauseJob = async (monitor) => {
|
||||
this.deleteJob(monitor);
|
||||
};
|
||||
|
||||
resumeJob = async (monitor) => {
|
||||
this.addJob(monitor._id, monitor);
|
||||
};
|
||||
|
||||
async addJob(jobName, monitor) {
|
||||
this.logger.info({
|
||||
message: `Adding job ${monitor?.url ?? "No URL"}`,
|
||||
|
||||
@@ -254,7 +254,8 @@ class JobQueueHelper {
|
||||
|
||||
// Get the current status
|
||||
await job.updateProgress(30);
|
||||
const networkResponse = await this.networkService.getStatus(job);
|
||||
const monitor = job.data;
|
||||
const networkResponse = await this.networkService.getStatus(monitor);
|
||||
if (
|
||||
job.data.type === "distributed_http" ||
|
||||
job.data.type === "distributed_test"
|
||||
@@ -271,14 +272,17 @@ class JobQueueHelper {
|
||||
|
||||
// Handle status change
|
||||
await job.updateProgress(60);
|
||||
const { monitor, statusChanged, prevStatus } =
|
||||
await this.statusService.updateStatus(networkResponse);
|
||||
const {
|
||||
monitor: updatedMonitor,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
} = await this.statusService.updateStatus(networkResponse);
|
||||
// Handle notifications
|
||||
await job.updateProgress(80);
|
||||
this.notificationService
|
||||
.handleNotifications({
|
||||
...networkResponse,
|
||||
monitor,
|
||||
monitor: updatedMonitor,
|
||||
prevStatus,
|
||||
statusChanged,
|
||||
})
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
import { Pulse } from "@pulsecron/pulse";
|
||||
import { ObjectId } from "mongodb";
|
||||
|
||||
const SERVICE_NAME = "PulseQueue";
|
||||
class PulseQueue {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor({ appSettings, db, pulseQueueHelper, logger }) {
|
||||
this.db = db;
|
||||
this.appSettings = appSettings;
|
||||
this.pulseQueueHelper = pulseQueueHelper;
|
||||
this.logger = logger;
|
||||
this.init();
|
||||
}
|
||||
|
||||
// ****************************************
|
||||
// Core methods
|
||||
// ****************************************
|
||||
init = async () => {
|
||||
try {
|
||||
const mongoConnectionString =
|
||||
this.appSettings.dbConnectionString || "mongodb://localhost:27017/uptime_db";
|
||||
this.pulse = new Pulse({ db: { address: mongoConnectionString } });
|
||||
await this.pulse.start();
|
||||
this.pulse.define("monitor-job", this.pulseQueueHelper.getMonitorJob(), {});
|
||||
|
||||
const monitors = await this.db.getAllMonitors();
|
||||
for (const monitor of monitors) {
|
||||
await this.addJob(monitor._id, monitor);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: "Failed to initialize PulseQueue",
|
||||
service: SERVICE_NAME,
|
||||
method: "init",
|
||||
details: error,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
addJob = async (monitorId, monitor) => {
|
||||
this.logger.debug({
|
||||
message: `Adding job ${monitor?.url ?? "No URL"}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "addJob",
|
||||
});
|
||||
const intervalInSeconds = monitor.interval / 1000;
|
||||
const job = this.pulse.create("monitor-job", {
|
||||
monitor,
|
||||
});
|
||||
job.unique({ "data.monitor._id": monitor._id });
|
||||
job.attrs.jobId = monitorId.toString();
|
||||
job.repeatEvery(`${intervalInSeconds} seconds`);
|
||||
await job.save();
|
||||
};
|
||||
|
||||
deleteJob = async (monitor) => {
|
||||
this.logger.debug({
|
||||
message: `Deleting job ${monitor?.url ?? "No URL"}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "deleteJob",
|
||||
});
|
||||
const result = await this.pulse.cancel({
|
||||
"data.monitor._id": monitor._id,
|
||||
});
|
||||
console.log(result);
|
||||
};
|
||||
|
||||
pauseJob = async (monitor) => {
|
||||
const result = await this.pulse.disable({
|
||||
"data.monitor._id": monitor._id,
|
||||
});
|
||||
|
||||
if (result.length < 1) {
|
||||
throw new Error("Failed to pause monitor");
|
||||
}
|
||||
|
||||
this.logger.debug({
|
||||
message: `Paused monitor ${monitor._id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "pauseJob",
|
||||
});
|
||||
};
|
||||
|
||||
resumeJob = async (monitor) => {
|
||||
const result = await this.pulse.enable({
|
||||
"data.monitor._id": monitor._id,
|
||||
});
|
||||
|
||||
if (result.length < 1) {
|
||||
throw new Error("Failed to resume monitor");
|
||||
}
|
||||
|
||||
this.logger.debug({
|
||||
message: `Resumed monitor ${monitor._id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "resumeJob",
|
||||
});
|
||||
};
|
||||
|
||||
shutdown = async () => {
|
||||
this.logger.info({
|
||||
message: "Shutting down JobQueue",
|
||||
service: SERVICE_NAME,
|
||||
method: "shutdown",
|
||||
});
|
||||
await this.pulse.stop();
|
||||
};
|
||||
|
||||
// ****************************************
|
||||
// Diagnostic methods
|
||||
// ****************************************
|
||||
|
||||
getMetrics = async () => {
|
||||
const jobs = await this.pulse.jobs();
|
||||
const metrics = jobs.reduce(
|
||||
(acc, job) => {
|
||||
acc.jobs++;
|
||||
if (job.attrs.failCount > 0 && job.attrs.failedAt >= job.attrs.lastFinishedAt) {
|
||||
acc.failingJobs++;
|
||||
}
|
||||
if (job.attrs.lockedAt) {
|
||||
acc.activeJobs++;
|
||||
}
|
||||
if (job.attrs.failCount > 0) {
|
||||
acc.jobsWithFailures.push({
|
||||
monitorId: job.attrs.data.monitor._id,
|
||||
monitorUrl: job.attrs.data.monitor.url,
|
||||
failCount: job.attrs.failCount,
|
||||
failReason: job.attrs.failReason,
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{ jobs: 0, activeJobs: 0, failingJobs: 0, jobsWithFailures: [] }
|
||||
);
|
||||
return metrics;
|
||||
};
|
||||
|
||||
getJobs = async () => {
|
||||
const jobs = await this.pulse.jobs();
|
||||
return jobs.map((job) => {
|
||||
return {
|
||||
monitorId: job.attrs.data.monitor._id,
|
||||
monitorUrl: job.attrs.data.monitor.url,
|
||||
lockedAt: job.attrs.lockedAt,
|
||||
runCount: job.attrs.runCount || 0,
|
||||
failCount: job.attrs.failCount || 0,
|
||||
failReason: job.attrs.failReason,
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
flushQueues = async () => {
|
||||
const cancelRes = await this.pulse.cancel();
|
||||
await this.pulse.stop();
|
||||
const initRes = await this.init();
|
||||
return {
|
||||
flushedJobs: cancelRes,
|
||||
initSuccess: initRes,
|
||||
};
|
||||
};
|
||||
|
||||
obliterate = async () => {
|
||||
await this.flushQueues();
|
||||
};
|
||||
}
|
||||
|
||||
export default PulseQueue;
|
||||
@@ -0,0 +1,106 @@
|
||||
const SERVICE_NAME = "PulseQueueHelper";
|
||||
|
||||
class PulseQueueHelper {
|
||||
constructor({ db, logger, networkService, statusService, notificationService }) {
|
||||
this.db = db;
|
||||
this.logger = logger;
|
||||
this.networkService = networkService;
|
||||
this.statusService = statusService;
|
||||
this.notificationService = notificationService;
|
||||
}
|
||||
|
||||
getMonitorJob = () => {
|
||||
return async (job) => {
|
||||
try {
|
||||
const monitor = job.attrs.data.monitor;
|
||||
const monitorId = job.attrs.data.monitor._id;
|
||||
if (!monitorId) {
|
||||
throw new Error("No monitor id");
|
||||
}
|
||||
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.logger.info({
|
||||
message: `Monitor ${monitorId} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "getMonitorJob",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const networkResponse = await this.networkService.getStatus(monitor);
|
||||
if (monitor.type === "distributed_http" || monitor.type === "distributed_test") {
|
||||
await job.updateProgress(100);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!networkResponse) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
const {
|
||||
monitor: updatedMonitor,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
} = await this.statusService.updateStatus(networkResponse);
|
||||
|
||||
this.notificationService
|
||||
.handleNotifications({
|
||||
...networkResponse,
|
||||
monitor: updatedMonitor,
|
||||
prevStatus,
|
||||
statusChanged,
|
||||
})
|
||||
.catch((error) => {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "createJobHandler",
|
||||
details: `Error sending notifications for job ${job.id}: ${error.message}`,
|
||||
stack: error.stack,
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.warn({
|
||||
message: error.message,
|
||||
service: error.service || SERVICE_NAME,
|
||||
method: error.method || "getMonitorJob",
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
async isInMaintenanceWindow(monitorId) {
|
||||
const maintenanceWindows = await this.db.getMaintenanceWindowsByMonitorId(monitorId);
|
||||
// Check for active maintenance window:
|
||||
const maintenanceWindowIsActive = maintenanceWindows.reduce((acc, window) => {
|
||||
if (window.active) {
|
||||
const start = new Date(window.start);
|
||||
const end = new Date(window.end);
|
||||
const now = new Date();
|
||||
const repeatInterval = window.repeat || 0;
|
||||
|
||||
// If start is < now and end > now, we're in maintenance
|
||||
if (start <= now && end >= now) return true;
|
||||
|
||||
// If maintenance window was set in the past with a repeat,
|
||||
// we need to advance start and end to see if we are in range
|
||||
|
||||
while (start < now && repeatInterval !== 0) {
|
||||
start.setTime(start.getTime() + repeatInterval);
|
||||
end.setTime(end.getTime() + repeatInterval);
|
||||
if (start <= now && end >= now) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return acc;
|
||||
}, false);
|
||||
return maintenanceWindowIsActive;
|
||||
}
|
||||
}
|
||||
|
||||
export default PulseQueueHelper;
|
||||
@@ -125,7 +125,7 @@ class NetworkService {
|
||||
* @property {number} code - The response code (200 if successful, error code otherwise).
|
||||
* @property {string} message - The message indicating the result of the HTTP request.
|
||||
*/
|
||||
async requestHttp(job) {
|
||||
async requestHttp(monitor) {
|
||||
try {
|
||||
const {
|
||||
url,
|
||||
@@ -138,7 +138,7 @@ class NetworkService {
|
||||
jsonPath,
|
||||
matchMethod,
|
||||
expectedValue,
|
||||
} = job.data;
|
||||
} = monitor;
|
||||
const config = {};
|
||||
|
||||
secret !== undefined && (config.headers = { Authorization: `Bearer ${secret}` });
|
||||
@@ -248,10 +248,10 @@ class NetworkService {
|
||||
* @property {number} code - The response code (200 if successful, error code otherwise).
|
||||
* @property {string} message - The message indicating the result of the PageSpeed request.
|
||||
*/
|
||||
async requestPagespeed(job) {
|
||||
async requestPagespeed(monitor) {
|
||||
try {
|
||||
const url = job.data.url;
|
||||
const updatedJob = { ...job };
|
||||
const url = monitor.url;
|
||||
const updatedMonitor = { ...monitor };
|
||||
let pagespeedUrl = `https://pagespeedonline.googleapis.com/pagespeedonline/v5/runPagespeed?url=${url}&category=seo&category=accessibility&category=best-practices&category=performance`;
|
||||
|
||||
const dbSettings = await this.settingsService.getDBSettings();
|
||||
@@ -266,8 +266,8 @@ class NetworkService {
|
||||
});
|
||||
return;
|
||||
}
|
||||
updatedJob.data.url = pagespeedUrl;
|
||||
return await this.requestHttp(updatedJob);
|
||||
updatedMonitor.url = pagespeedUrl;
|
||||
return await this.requestHttp(updatedMonitor);
|
||||
} catch (error) {
|
||||
error.service = this.SERVICE_NAME;
|
||||
error.method = "requestPagespeed";
|
||||
@@ -292,9 +292,9 @@ class NetworkService {
|
||||
* @property {number} code - The response code (200 if successful, error code otherwise).
|
||||
* @property {string} message - The message indicating the result of the hardware status request.
|
||||
*/
|
||||
async requestHardware(job) {
|
||||
async requestHardware(monitor) {
|
||||
try {
|
||||
return await this.requestHttp(job);
|
||||
return await this.requestHttp(monitor);
|
||||
} catch (error) {
|
||||
error.service = this.SERVICE_NAME;
|
||||
error.method = "requestHardware";
|
||||
@@ -318,7 +318,7 @@ class NetworkService {
|
||||
* @property {number} code - The response code (200 if successful, error code otherwise).
|
||||
* @property {string} message - The message indicating the result of the Docker inspection.
|
||||
*/
|
||||
async requestDocker(job) {
|
||||
async requestDocker(monitor) {
|
||||
try {
|
||||
const docker = new this.Docker({
|
||||
socketPath: "/var/run/docker.sock",
|
||||
@@ -326,19 +326,19 @@ class NetworkService {
|
||||
});
|
||||
|
||||
const containers = await docker.listContainers({ all: true });
|
||||
const containerExists = containers.some((c) => c.Id.startsWith(job.data.url));
|
||||
const containerExists = containers.some((c) => c.Id.startsWith(monitor.url));
|
||||
if (!containerExists) {
|
||||
throw new Error(this.stringService.dockerNotFound);
|
||||
}
|
||||
const container = docker.getContainer(job.data.url);
|
||||
const container = docker.getContainer(monitor.url);
|
||||
|
||||
const { response, responseTime, error } = await this.timeRequest(() =>
|
||||
container.inspect()
|
||||
);
|
||||
|
||||
const dockerResponse = {
|
||||
monitorId: job.data._id,
|
||||
type: job.data.type,
|
||||
monitorId: monitor._id,
|
||||
type: monitor.type,
|
||||
responseTime,
|
||||
};
|
||||
|
||||
@@ -360,9 +360,9 @@ class NetworkService {
|
||||
}
|
||||
}
|
||||
|
||||
async requestPort(job) {
|
||||
async requestPort(monitor) {
|
||||
try {
|
||||
const { url, port } = job.data;
|
||||
const { url, port } = monitor;
|
||||
const { response, responseTime, error } = await this.timeRequest(async () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = this.net.createConnection(
|
||||
@@ -391,8 +391,8 @@ class NetworkService {
|
||||
});
|
||||
|
||||
const portResponse = {
|
||||
monitorId: job.data._id,
|
||||
type: job.data.type,
|
||||
monitorId: monitor._id,
|
||||
type: monitor.type,
|
||||
responseTime,
|
||||
};
|
||||
|
||||
@@ -414,9 +414,8 @@ class NetworkService {
|
||||
}
|
||||
}
|
||||
|
||||
async requestDistributedHttp(job) {
|
||||
async requestDistributedHttp(monitor) {
|
||||
try {
|
||||
const monitor = job.data;
|
||||
const CALLBACK_URL = process.env.CALLBACK_URL;
|
||||
|
||||
const response = await this.axios.post(
|
||||
@@ -503,23 +502,23 @@ class NetworkService {
|
||||
* @returns {Promise<Object>} The response object from the appropriate request method.
|
||||
* @throws {Error} Throws an error if the job type is unsupported.
|
||||
*/
|
||||
async getStatus(job) {
|
||||
const type = job?.data?.type ?? "unknown";
|
||||
async getStatus(monitor) {
|
||||
const type = monitor.type ?? "unknown";
|
||||
switch (type) {
|
||||
case this.TYPE_PING:
|
||||
return await this.requestPing(job);
|
||||
return await this.requestPing(monitor);
|
||||
case this.TYPE_HTTP:
|
||||
return await this.requestHttp(job);
|
||||
return await this.requestHttp(monitor);
|
||||
case this.TYPE_PAGESPEED:
|
||||
return await this.requestPagespeed(job);
|
||||
return await this.requestPagespeed(monitor);
|
||||
case this.TYPE_HARDWARE:
|
||||
return await this.requestHardware(job);
|
||||
return await this.requestHardware(monitor);
|
||||
case this.TYPE_DOCKER:
|
||||
return await this.requestDocker(job);
|
||||
return await this.requestDocker(monitor);
|
||||
case this.TYPE_PORT:
|
||||
return await this.requestPort(job);
|
||||
return await this.requestPort(monitor);
|
||||
case this.TYPE_DISTRIBUTED_HTTP:
|
||||
return await this.requestDistributedHttp(job);
|
||||
return await this.requestDistributedHttp(monitor);
|
||||
case this.TYPE_DISTRIBUTED_TEST:
|
||||
return;
|
||||
default:
|
||||
|
||||
@@ -228,6 +228,12 @@ class StatusService {
|
||||
|
||||
if (type === "pagespeed") {
|
||||
if (typeof payload === "undefined") {
|
||||
this.logger.warn({
|
||||
message: "Failed to build check",
|
||||
service: this.SERVICE_NAME,
|
||||
method: "buildCheck",
|
||||
details: "empty payload",
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
const categories = payload?.lighthouseResult?.categories ?? {};
|
||||
@@ -287,9 +293,11 @@ class StatusService {
|
||||
} catch (error) {
|
||||
this.logger.error({
|
||||
message: error.message,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "insertCheck",
|
||||
details: `Error inserting check for monitor: ${networkResponse?.monitorId}`,
|
||||
service: error.service || this.SERVICE_NAME,
|
||||
method: error.method || "insertCheck",
|
||||
details:
|
||||
error.details ||
|
||||
`Error inserting check for monitor: ${networkResponse?.monitorId}`,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user