resolve conflicts

This commit is contained in:
Alex Holliday committed 2026-02-26 17:32:11 +00:00
commit dfeba1e4fb
48 files changed
+2921 -217

No files matched your search

@@ -1,5 +1,7 @@
import { IMonitorsRepository } from "@/repositories/index.js";
import { ILogger } from "@/utils/logger.js";
import Scheduler from "super-simple-scheduler";
import { ISuperSimpleQueueHelper } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
const SERVICE_NAME = "JobQueue";
type QueueJobFailure = {
@@ -54,22 +56,12 @@ export interface ISuperSimpleQueue {
class SuperSimpleQueue implements ISuperSimpleQueue {
static SERVICE_NAME = SERVICE_NAME;
private logger: any;
private helper: any;
private logger: ILogger;
private helper: ISuperSimpleQueueHelper;
private monitorsRepository: IMonitorsRepository;
private readonly scheduler: Scheduler;
constructor({
logger,
helper,
monitorsRepository,
scheduler,
}: {
logger: any;
helper: any;
monitorsRepository: IMonitorsRepository;
scheduler: Scheduler;
}) {
constructor(logger: ILogger, helper: ISuperSimpleQueueHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
@@ -80,14 +72,14 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
return SuperSimpleQueue.SERVICE_NAME;
}
static async create({ logger, helper, monitorsRepository }: { logger: any; helper: any; monitorsRepository: IMonitorsRepository }) {
static async create(logger: ILogger, helper: ISuperSimpleQueueHelper, monitorsRepository: IMonitorsRepository) {
const scheduler = new Scheduler({
// storeType: "mongo",
// storeType: "redis",
logLevel: "debug",
// dbUri: envSettings.dbConnectionString,
});
const instance = new SuperSimpleQueue({ logger, helper, monitorsRepository, scheduler });
const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
await instance.init();
return instance;
}
@@ -97,6 +89,7 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getMonitorJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getGeoCheckJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
@@ -112,7 +105,7 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
return true;
} catch (error) {
} catch (error: any) {
this.logger.error({
message: "Failed to initialize SuperSimpleQueue",
service: SERVICE_NAME,
@@ -131,17 +124,30 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
active: monitor.isActive,
data: monitor,
});
// Add geo check job if enabled for HTTP monitors
if (monitor.geoCheckEnabled && monitor.type === "http") {
this.scheduler.addJob({
id: `${monitorId}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
}
};
deleteJob = async (monitor: any) => {
this.scheduler.removeJob(monitor.id);
this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: any) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
throw new Error("Failed to pause monitor");
}
await this.scheduler.pauseJob(`${monitor.id}-geo`);
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
@@ -154,6 +160,9 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
if (result === false) {
throw new Error("Failed to resume monitor");
}
await this.scheduler.resumeJob(`${monitor.id}-geo`);
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
@@ -163,6 +172,29 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
updateJob = async (monitor: any) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
// Handle geo check job lifecycle
const geoJobId = `${monitor.id}-geo`;
if (monitor.geoCheckEnabled && monitor.type === "http") {
// Check if geo job exists
const existingGeoJob = await this.scheduler.getJob(geoJobId);
if (existingGeoJob) {
// Update existing geo job
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
} else {
// Create new geo job
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
}
} else {
// Remove geo job if disabled or monitor type changed
this.scheduler.removeJob(geoJobId);
}
};
shutdown = async () => {
@@ -4,6 +4,7 @@ import { AppError } from "@/utils/AppError.js";
import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js";
import type { StatusChangeResult, MonitorStatusResponse, HardwareStatusPayload, MonitorStatus } from "@/types/index.js";
import IncidentService from "@/service/business/incidentService.js";
import type { IGeoChecksService } from "@/service/business/geoChecksService.js";
import {
IMaintenanceWindowsRepository,
IMonitorsRepository,
@@ -11,7 +12,18 @@ import {
IMonitorStatsRepository,
IChecksRepository,
IIncidentsRepository,
IGeoChecksRepository,
} from "@/repositories/index.js";
import { ILogger } from "@/utils/logger.js";
import { IBufferService } from "../bufferService.js";
export interface ISuperSimpleQueueHelper {
readonly serviceName: string;
getMonitorJob(): (monitor: Monitor) => Promise<void>;
getCleanupOrphanedJob(): () => Promise<void>;
getGeoCheckJob(): (monitor: Monitor) => Promise<void>;
isInMaintenanceWindow(monitorId: string, teamId: string): Promise<boolean>;
}
export interface MonitorActionDecision {
shouldCreateIncident: boolean;
@@ -27,7 +39,7 @@ export interface MonitorActionDecision {
};
}
class SuperSimpleQueueHelper {
class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
static SERVICE_NAME = SERVICE_NAME;
private logger: any;
@@ -35,7 +47,7 @@ class SuperSimpleQueueHelper {
private statusService: IStatusService;
private notificationsService: INotificationsService;
private checkService: any;
private buffer: any;
private buffer: IBufferService;
private incidentService: IncidentService;
private maintenanceWindowsRepository: IMaintenanceWindowsRepository;
private monitorsRepository: IMonitorsRepository;
@@ -43,36 +55,26 @@ class SuperSimpleQueueHelper {
private monitorStatsRepository: IMonitorStatsRepository;
private checksRepository: IChecksRepository;
private incidentsRepository: IIncidentsRepository;
private geoChecksService: IGeoChecksService;
private geoChecksRepository: IGeoChecksRepository;
constructor({
logger,
networkService,
statusService,
notificationsService,
checkService,
buffer,
incidentService,
maintenanceWindowsRepository,
monitorsRepository,
teamsRepository,
monitorStatsRepository,
checksRepository,
incidentsRepository,
}: {
logger: any;
networkService: INetworkService;
statusService: IStatusService;
notificationsService: INotificationsService;
checkService: any;
buffer: any;
incidentService: IncidentService;
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
monitorsRepository: IMonitorsRepository;
teamsRepository: ITeamsRepository;
monitorStatsRepository: IMonitorStatsRepository;
checksRepository: IChecksRepository;
incidentsRepository: IIncidentsRepository;
}) {
constructor(
logger: ILogger,
networkService: INetworkService,
statusService: IStatusService,
notificationsService: INotificationsService,
checkService: any,
buffer: IBufferService,
incidentService: IncidentService,
maintenanceWindowsRepository: IMaintenanceWindowsRepository,
monitorsRepository: IMonitorsRepository,
teamsRepository: ITeamsRepository,
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository,
incidentsRepository: IIncidentsRepository,
geoChecksService: IGeoChecksService,
geoChecksRepository: IGeoChecksRepository
) {
this.logger = logger;
this.networkService = networkService;
this.statusService = statusService;
@@ -86,6 +88,8 @@ class SuperSimpleQueueHelper {
this.monitorStatsRepository = monitorStatsRepository;
this.checksRepository = checksRepository;
this.incidentsRepository = incidentsRepository;
this.geoChecksService = geoChecksService;
this.geoChecksRepository = geoChecksRepository;
}
get serviceName() {
@@ -240,6 +244,16 @@ class SuperSimpleQueueHelper {
});
}
// Remove orphaned geo checks
const deletedGeoChecksCount = await this.geoChecksRepository.deleteByMonitorIdsNotIn(allMonitorIds);
if (deletedGeoChecksCount > 0) {
this.logger.info({
message: `Deleted ${deletedGeoChecksCount} orphaned geo checks`,
service: SERVICE_NAME,
method: "getCleanupOrphanedJob",
});
}
this.logger.info({
message: "Cleanup of orphaned data completed",
service: SERVICE_NAME,
@@ -257,6 +271,81 @@ class SuperSimpleQueueHelper {
};
};
getGeoCheckJob = () => {
return async (monitor: Monitor) => {
try {
const monitorId = monitor.id;
const teamId = monitor.teamId;
// Step 1: Validate monitor eligibility
if (!monitorId) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getGeoCheckJob" });
}
if (monitor.type !== "http") {
this.logger.debug({
message: `Monitor ${monitorId} is not HTTP type, skipping geo check`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
if (!monitor.geoCheckEnabled) {
return;
}
if (!monitor.geoCheckLocations || monitor.geoCheckLocations.length === 0) {
this.logger.warn({
message: `No geo check locations configured for monitor ${monitorId}`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
// Step 2: Check for maintenance window
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId, teamId);
if (maintenanceWindowActive) {
this.logger.debug({
message: `Monitor ${monitorId} is in maintenance window, skipping geo check`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
// Step 3: Build geo check (handles API calls and polling)
const geoCheck = await this.geoChecksService.buildGeoCheck(monitor);
if (!geoCheck) {
this.logger.warn({
message: `No geo check could be built for monitor ${monitorId}`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
// Step 4: Add geo check to buffer
this.buffer.addGeoCheckToBuffer(geoCheck);
this.logger.debug({
message: `Geo check job executed for monitor ${monitorId}`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
} catch (error: any) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "getGeoCheckJob",
stack: error.stack,
});
// Don't throw - geo check failures shouldn't crash the job scheduler
}
};
};
async isInMaintenanceWindow(monitorId: string, teamId: string) {
const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
// Check for active maintenance window:
@@ -1,29 +1,38 @@
import type { Check } from "@/types/index.js";
import type { GeoCheck } from "@/types/index.js";
import type { IGeoChecksService } from "../business/geoChecksService.js";
import type { ILogger } from "@/utils/logger.js";
import type { ISettingsService } from "@/service/system/settingsService.js";
const SERVICE_NAME = "BufferService";
export interface IBufferService {
addToBuffer(check: Check): void;
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
removeCheckFromBuffer(check: Check): boolean;
scheduleNextFlush(): void;
flushBuffer(): Promise<void>;
flushGeoBuffer(): Promise<void>;
}
class BufferService implements IBufferService {
static SERVICE_NAME = SERVICE_NAME;
private BUFFER_TIMEOUT: number;
private logger: any;
private logger: ILogger;
private SERVICE_NAME: string;
private buffer: any[];
private geoBuffer: any[];
private bufferTimer: NodeJS.Timeout | null = null;
private checksService: any;
private geoChecksService: IGeoChecksService;
constructor({ logger, checkService, settingsService }: { logger: any; checkService: any; settingsService: any }) {
constructor(logger: ILogger, checkService: any, geoChecksService: IGeoChecksService, settingsService: ISettingsService) {
this.BUFFER_TIMEOUT = settingsService.getSettings().nodeEnv === "development" ? 10 : 1000 * 60 * 1; // 1 minute
this.logger = logger;
this.checksService = checkService;
this.geoChecksService = geoChecksService;
this.SERVICE_NAME = SERVICE_NAME;
this.buffer = [];
this.geoBuffer = [];
this.scheduleNextFlush();
this.logger.info({
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
@@ -49,6 +58,19 @@ class BufferService implements IBufferService {
}
}
addGeoCheckToBuffer(geoCheck: GeoCheck) {
try {
this.geoBuffer.push(geoCheck);
} catch (error: any) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "addGeoCheckToBuffer",
stack: error.stack,
});
}
}
removeCheckFromBuffer(checkToRemove: Check) {
try {
if (!checkToRemove) {
@@ -94,6 +116,7 @@ class BufferService implements IBufferService {
this.bufferTimer = setTimeout(async () => {
try {
await this.flushBuffer();
await this.flushGeoBuffer();
} catch (error: any) {
this.logger.error({
message: `Error in flush cycle: ${error.message}`,
@@ -110,18 +133,47 @@ class BufferService implements IBufferService {
async flushBuffer() {
try {
if (this.buffer.length > 0) {
this.logger.debug({
message: `Flushing ${this.buffer.length} checks to database`,
service: this.SERVICE_NAME,
method: "flushBuffer",
});
await this.checksService.createChecks(this.buffer);
this.buffer = [];
}
} catch (error: any) {
this.logger.error({
message: error.message,
message: `Error flushing checks buffer: ${error.message}`,
service: this.SERVICE_NAME,
method: "flushBuffer",
stack: error.stack,
});
// Clear buffer even on error to prevent infinite retry loops
this.buffer = [];
}
}
this.buffer = [];
async flushGeoBuffer() {
try {
if (this.geoBuffer.length > 0) {
this.logger.debug({
message: `Flushing ${this.geoBuffer.length} geo checks to database`,
service: this.SERVICE_NAME,
method: "flushGeoBuffer",
});
await this.geoChecksService.createGeoChecks(this.geoBuffer);
this.geoBuffer = [];
}
} catch (error: any) {
this.logger.error({
message: `Error flushing geo checks buffer: ${error.message}`,
service: this.SERVICE_NAME,
method: "flushGeoBuffer",
stack: error.stack,
});
// Clear buffer even on error to prevent infinite retry loops
this.geoBuffer = [];
}
}
}
@@ -0,0 +1,210 @@
import type { GeoContinent, GeoCheckResult, GeoCheckTimings, GeoCheckLocation } from "@/types/geoCheck.js";
import type { ILogger } from "@/utils/logger.js";
import got from "got";
const SERVICE_NAME = "GlobalPingService";
const GLOBAL_PING_API_BASE = "https://api.globalping.io/v1";
const POLL_INTERVAL_MS = 2000;
const MAX_POLL_TIMEOUT_MS = 30000;
interface GlobalPingMeasurementRequest {
type: "http";
target: string;
locations: Array<{ continent: GeoContinent }>;
limit: number;
}
interface GlobalPingMeasurementResponse {
id: string;
type: string;
status: "in-progress" | "finished" | "failed";
probesCount: number;
results?: GlobalPingProbeResult[];
}
interface GlobalPingProbeResult {
probe: {
continent: GeoContinent;
region: string;
country: string;
state: string | null;
city: string;
longitude: number;
latitude: number;
};
result: {
status: "finished" | "failed" | "timeout";
statusCode?: number;
statusCodeName?: string;
timings?: {
total: number;
dns: number;
tcp: number;
tls: number;
firstByte: number;
download: number;
};
rawOutput?: string;
};
}
export interface IGlobalPingService {
readonly serviceName: string;
createMeasurement(url: string, locations: GeoContinent[]): Promise<string | null>;
pollForResults(measurementId: string, timeoutMs?: number): Promise<GeoCheckResult[]>;
}
class GlobalPingService implements IGlobalPingService {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
constructor(logger: ILogger) {
this.logger = logger;
}
get serviceName() {
return GlobalPingService.SERVICE_NAME;
}
async createMeasurement(url: string, locations: GeoContinent[]): Promise<string | null> {
try {
// GlobalPing API expects target without protocol (http:// or https://)
const cleanTarget = url.replace(/^https?:\/\//, "");
const requestBody: GlobalPingMeasurementRequest = {
type: "http",
target: cleanTarget,
locations: locations.map((continent) => ({ continent })),
limit: locations.length,
};
const response = await got.post<GlobalPingMeasurementResponse>(`${GLOBAL_PING_API_BASE}/measurements`, {
json: requestBody,
responseType: "json",
timeout: { request: 10000 },
});
const measurementId = response.body.id;
this.logger.debug({
message: `Created GlobalPing measurement: ${measurementId}`,
service: SERVICE_NAME,
method: "createMeasurement",
details: { measurementId, url, locations },
});
return measurementId;
} catch (error: any) {
this.logger.error({
message: "GlobalPing API unavailable, skipping geo check",
service: SERVICE_NAME,
method: "createMeasurement",
details: error,
});
return null;
}
}
async pollForResults(measurementId: string, timeoutMs: number = MAX_POLL_TIMEOUT_MS): Promise<GeoCheckResult[]> {
const startTime = Date.now();
while (Date.now() - startTime < timeoutMs) {
try {
const response = await got.get<GlobalPingMeasurementResponse>(`${GLOBAL_PING_API_BASE}/measurements/${measurementId}`, {
responseType: "json",
timeout: { request: 5000 },
});
const measurement = response.body;
if (measurement.status === "finished") {
const results = this.transformResults(measurement.results || []);
this.logger.debug({
message: `GlobalPing measurement completed: ${measurementId}`,
service: SERVICE_NAME,
method: "pollForResults",
details: { measurementId, resultsCount: results.length },
});
return results;
}
if (measurement.status === "failed") {
this.logger.warn({
message: `GlobalPing measurement failed: ${measurementId}`,
service: SERVICE_NAME,
method: "pollForResults",
});
return [];
}
// Still in-progress, wait and poll again
await this.sleep(POLL_INTERVAL_MS);
} catch (error: any) {
this.logger.error({
message: "Error polling GlobalPing API",
service: SERVICE_NAME,
method: "pollForResults",
details: error.message,
});
return [];
}
}
// Timeout reached
this.logger.warn({
message: `GlobalPing measurement polling timeout: ${measurementId}`,
service: SERVICE_NAME,
method: "pollForResults",
details: { measurementId, timeoutMs },
});
return [];
}
private transformResults(probeResults: GlobalPingProbeResult[]): GeoCheckResult[] {
const successfulResults: GeoCheckResult[] = [];
for (const probeResult of probeResults) {
// Skip failed or timeout results
if (probeResult.result.status !== "finished" || !probeResult.result.statusCode || !probeResult.result.timings) {
continue;
}
const location: GeoCheckLocation = {
continent: probeResult.probe.continent,
region: probeResult.probe.region,
country: probeResult.probe.country,
state: probeResult.probe.state || "",
city: probeResult.probe.city,
longitude: probeResult.probe.longitude,
latitude: probeResult.probe.latitude,
};
const timings: GeoCheckTimings = {
total: probeResult.result.timings.total,
dns: probeResult.result.timings.dns,
tcp: probeResult.result.timings.tcp,
tls: probeResult.result.timings.tls,
firstByte: probeResult.result.timings.firstByte,
download: probeResult.result.timings.download,
};
const result: GeoCheckResult = {
location,
status: probeResult.result.statusCode >= 200 && probeResult.result.statusCode < 300,
statusCode: probeResult.result.statusCode,
timings,
};
successfulResults.push(result);
}
return successfulResults;
}
private sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}
export default GlobalPingService;
@@ -1,11 +1,14 @@
import { HTTPError, RequestError } from "got";
import type { Got, Response } from "got";
import type { Monitor, MonitorStatusResponse, GrpcStatusPayload } from "@/types/index.js";
import type { AxiosStatic } from "axios";
import { AppError } from "@/utils/AppError.js";
import path from "path";
import { fileURLToPath } from "url";
import CacheableLookup from "cacheable-lookup";
import { ISettingsService } from "../system/settingsService.js";
import { ILogger } from "@/utils/logger.js";
const SERVICE_NAME = "NetworkService";
type MonitorStatusResponseOverrides<T> = Partial<Omit<MonitorStatusResponse<T>, "monitorId" | "teamId" | "type">>;
@@ -50,16 +53,16 @@ class NetworkService implements INetworkService {
private NETWORK_ERROR: number;
private PING_ERROR: number;
private axios: any;
private axios: AxiosStatic;
private got: Got;
private https: any;
private jmespath: any;
private GameDig: any;
private ping: any;
private logger: any;
private logger: ILogger;
private Docker: any;
private net: any;
private settingsService: any;
private settingsService: ISettingsService;
private grpc: any;
private protoLoader: any;
@@ -117,35 +120,20 @@ class NetworkService implements INetworkService {
};
};
constructor({
axios,
got,
https,
jmespath,
GameDig,
ping,
logger,
http,
Docker,
net,
settingsService,
grpc,
protoLoader,
}: {
axios: any;
got: Got;
https: any;
jmespath: any;
GameDig: any;
ping: any;
logger: any;
http: any;
Docker: any;
net: any;
settingsService: any;
grpc: any;
protoLoader: any;
}) {
constructor(
axios: AxiosStatic,
got: Got,
https: any,
jmespath: any,
GameDig: any,
ping: any,
logger: ILogger,
Docker: any,
net: any,
settingsService: ISettingsService,
grpc: any,
protoLoader: any
) {
this.TYPE_PING = "ping";
this.TYPE_HTTP = "http";
this.TYPE_PAGESPEED = "pagespeed";
@@ -449,7 +437,6 @@ class NetworkService implements INetworkService {
const docker = new this.Docker({
socketPath: "/var/run/docker.sock",
handleError: true, // Enable error handling
});
const dockerResponse = this.buildStatusResponse({
@@ -545,6 +532,10 @@ class NetworkService implements INetworkService {
private async requestPort(monitor: Monitor): Promise<MonitorStatusResponse> {
try {
const { url, port } = monitor;
if (!port) {
throw new Error("Port is required for port monitor");
}
const { response, responseTime, error } = await this.timeRequest(async () => {
return new Promise((resolve, reject) => {
const socket = this.net.createConnection(
@@ -611,9 +602,9 @@ class NetworkService implements INetworkService {
});
const state = await this.GameDig.query({
type: gameId,
type: gameId ?? "unknown",
host: url,
port: port,
port: port ?? 0,
}).catch((error: any) => {
this.logger.warn({
message: error.message,