resolve conflicts

This commit is contained in:
Alex Holliday committed 2026-02-26 17:32:11 +00:00
commit dfeba1e4fb
48 files changed
+2921 -217

No files matched your search

+3 -11
View File
@@ -1,10 +1,10 @@
import { requireTeamId } from "@/controllers/controllerUtils.js";
import { Types } from "mongoose";
import { IChecksRepository, IMonitorsRepository } from "@/repositories/index.js";
import type { MonitorType, MonitorStatusResponse, CheckErrorInfo, Check } from "@/types/index.js";
import type { MonitorStatusResponse, CheckErrorInfo, Check } from "@/types/index.js";
import type { HardwareStatusPayload, PageSpeedStatusPayload } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ParseBoolean } from "@/utils/utils.js";
import { ILogger } from "@/utils/logger.js";
const SERVICE_NAME = "checkService";
@@ -14,15 +14,7 @@ class CheckService {
private monitorsRepository: IMonitorsRepository;
private checksRepository: IChecksRepository;
private logger: any;
constructor({
monitorsRepository,
logger,
checksRepository,
}: {
monitorsRepository: IMonitorsRepository;
logger: any;
checksRepository: IChecksRepository;
}) {
constructor(monitorsRepository: IMonitorsRepository, logger: ILogger, checksRepository: IChecksRepository) {
this.monitorsRepository = monitorsRepository;
this.logger = logger;
this.checksRepository = checksRepository;
@@ -0,0 +1,192 @@
import type { Monitor, GeoCheck } from "@/types/index.js";
import type { GeoCheckResult } from "@/types/geoCheck.js";
import { Types } from "mongoose";
import type { IGeoChecksRepository } from "@/repositories/index.js";
import type { IMonitorsRepository } from "@/repositories/index.js";
import type { IGlobalPingService } from "@/service/infrastructure/globalPingService.js";
import type { ILogger } from "@/utils/logger.js";
import { AppError } from "@/utils/AppError.js";
const SERVICE_NAME = "GeoChecksService";
export interface IGeoChecksService {
readonly serviceName: string;
buildGeoCheck(monitor: Monitor): Promise<GeoCheck | null>;
createGeoChecks(geoChecks: GeoCheck[]): Promise<GeoCheck[]>;
getGeoChecksByMonitor(args: { monitorId: string; query: any; teamId: string }): Promise<any>;
}
class GeoChecksService implements IGeoChecksService {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private geoChecksRepository: IGeoChecksRepository;
private globalPingService: IGlobalPingService;
private monitorsRepository: IMonitorsRepository;
constructor({
logger,
geoChecksRepository,
globalPingService,
monitorsRepository,
}: {
logger: ILogger;
geoChecksRepository: IGeoChecksRepository;
globalPingService: IGlobalPingService;
monitorsRepository: IMonitorsRepository;
}) {
this.logger = logger;
this.geoChecksRepository = geoChecksRepository;
this.globalPingService = globalPingService;
this.monitorsRepository = monitorsRepository;
}
get serviceName() {
return GeoChecksService.SERVICE_NAME;
}
async buildGeoCheck(monitor: Monitor): Promise<GeoCheck | null> {
try {
if (!monitor.url) {
this.logger.warn({
message: "Monitor missing URL for geo check",
service: SERVICE_NAME,
method: "buildGeoCheck",
details: { monitorId: monitor.id },
});
return null;
}
if (!monitor.geoCheckLocations || monitor.geoCheckLocations.length === 0) {
this.logger.warn({
message: "Monitor missing geo check locations",
service: SERVICE_NAME,
method: "buildGeoCheck",
details: { monitorId: monitor.id },
});
return null;
}
// Step 1: Create measurement request
const measurementId = await this.globalPingService.createMeasurement(monitor.url, monitor.geoCheckLocations);
if (!measurementId) {
// GlobalPing API is down, skip this check
this.logger.debug({
message: "Skipping geo check due to API unavailability",
service: SERVICE_NAME,
method: "buildGeoCheck",
details: { monitorId: monitor.id },
});
return null;
}
// Step 2: Poll for results
const results = await this.globalPingService.pollForResults(measurementId);
if (results.length === 0) {
// No successful results (all locations timed out or failed)
this.logger.debug({
message: "No successful geo check results",
service: SERVICE_NAME,
method: "buildGeoCheck",
details: { monitorId: monitor.id, measurementId },
});
return null;
}
// Step 3: Build GeoCheck document
const geoCheck = this.createGeoCheckDocument(monitor, results);
this.logger.debug({
message: `Geo check completed for monitor ${monitor.id}`,
service: SERVICE_NAME,
method: "buildGeoCheck",
details: { monitorId: monitor.id, resultsCount: results.length },
});
return geoCheck;
} catch (error: any) {
this.logger.error({
message: "Error executing geo check",
service: SERVICE_NAME,
method: "buildGeoCheck",
details: { monitorId: monitor.id, error: error.message },
stack: error.stack,
});
return null;
}
}
private createGeoCheckDocument(monitor: Monitor, results: GeoCheckResult[]): GeoCheck {
const now = new Date();
const ttl = 90 * 24 * 60 * 60 * 1000; // 90 days in ms
const expiryDate = new Date(now.getTime() + ttl);
return {
id: new Types.ObjectId().toString(),
metadata: {
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
},
results,
expiry: expiryDate.toISOString(),
__v: 0,
createdAt: now.toISOString(),
updatedAt: now.toISOString(),
};
}
createGeoChecks = async (geoChecks: GeoCheck[]) => {
return this.geoChecksRepository.createGeoChecks(geoChecks);
};
getGeoChecksByMonitor = async ({ monitorId, query, teamId }: { monitorId: string; query: any; teamId: string }) => {
if (!monitorId) {
throw new AppError({
message: "No monitor ID in request",
service: SERVICE_NAME,
method: "getGeoChecksByMonitor",
status: 400,
});
}
if (!teamId) {
throw new AppError({
message: "No team ID in request",
service: SERVICE_NAME,
method: "getGeoChecksByMonitor",
status: 400,
});
}
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
if (!monitor) {
throw new AppError({
message: `Monitor with ID ${monitorId} not found.`,
service: SERVICE_NAME,
method: "getGeoChecksByMonitor",
status: 404,
});
}
let { sortOrder, dateRange, page, rowsPerPage, continent } = query;
const continents = continent ? (Array.isArray(continent) ? continent : [continent]) : undefined;
this.logger.debug({
message: "getGeoChecksByMonitor query params",
service: SERVICE_NAME,
method: "getGeoChecksByMonitor",
details: { continent, continents, query },
});
const parsedPage = page ? parseInt(page) : page;
const parsedRowsPerPage = rowsPerPage ? parseInt(rowsPerPage) : rowsPerPage;
const result = await this.geoChecksRepository.findByMonitorId(monitorId, sortOrder, dateRange, parsedPage, parsedRowsPerPage, continents);
return result;
};
}
export default GeoChecksService;
+16 -29
View File
@@ -3,42 +3,29 @@ import type { Monitor } from "@/types/monitor.js";
import type { MonitorStatusResponse } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ParseBoolean } from "@/utils/utils.js";
import { getDateForRange } from "@/utils/dataUtils.js";
import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from "@/repositories/index.js";
import type { Incident } from "@/types/index.js";
import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
import type { INotificationMessageBuilder } from "@/service/infrastructure/notificationMessageBuilder.js";
const dateRangeLookup: Record<string, Date | undefined> = {
recent: new Date(new Date().setHours(new Date().getHours() - 2)),
hour: new Date(new Date().setHours(new Date().getHours() - 1)),
day: new Date(new Date().setDate(new Date().getDate() - 1)),
week: new Date(new Date().setDate(new Date().getDate() - 7)),
month: new Date(new Date().setMonth(new Date().getMonth() - 1)),
all: undefined,
};
import type { ILogger } from "@/utils/logger.js";
class IncidentService {
static SERVICE_NAME = SERVICE_NAME;
private logger: any;
private logger: ILogger;
private incidentsRepository: IIncidentsRepository;
private monitorsRepository: IMonitorsRepository;
private usersRepository: IUsersRepository;
private notificationMessageBuilder: INotificationMessageBuilder;
constructor({
logger,
incidentsRepository,
monitorsRepository,
usersRepository,
notificationMessageBuilder,
}: {
logger: any;
incidentsRepository: IIncidentsRepository;
monitorsRepository: IMonitorsRepository;
usersRepository: IUsersRepository;
notificationMessageBuilder: INotificationMessageBuilder;
}) {
constructor(
logger: ILogger,
incidentsRepository: IIncidentsRepository,
monitorsRepository: IMonitorsRepository,
usersRepository: IUsersRepository,
notificationMessageBuilder: INotificationMessageBuilder
) {
this.logger = logger;
this.incidentsRepository = incidentsRepository;
this.monitorsRepository = monitorsRepository;
@@ -151,7 +138,7 @@ class IncidentService {
service: SERVICE_NAME,
method: "resolveIncidentManually",
message: `Incident manually resolved by user`,
details: resolvedIncident.id,
details: { incidentId: resolvedIncident.id },
});
return resolvedIncident;
@@ -160,7 +147,7 @@ class IncidentService {
service: SERVICE_NAME,
method: "resolveIncident",
message: error.message,
details: incidentId,
details: { id: incidentId },
stack: error.stack,
});
throw error;
@@ -182,7 +169,7 @@ class IncidentService {
throw new AppError({ message: "No team ID in request", service: SERVICE_NAME, method: "getIncidentsByTeam", status: 400 });
}
const startDate = dateRangeLookup[dateRange];
const startDate = getDateForRange(dateRange);
const parsedPage = Number.isFinite(parseInt(page)) ? parseInt(page) : 0;
const parsedRowsPerPage = Number.isFinite(parseInt(rowsPerPage)) ? parseInt(rowsPerPage) : 20;
@@ -207,7 +194,7 @@ class IncidentService {
service: SERVICE_NAME,
method: "getIncidentsByTeam",
message: error.message,
details: teamId,
details: { teamId },
stack: error.stack,
});
throw error;
@@ -229,7 +216,7 @@ class IncidentService {
service: SERVICE_NAME,
method: "getIncidentSummary",
message: error.message,
details: teamId,
details: { teamId },
stack: error.stack,
});
throw error;
@@ -250,7 +237,7 @@ class IncidentService {
service: SERVICE_NAME,
method: "getIncidentById",
message: error.message,
details: incidentId,
details: { incidentId },
stack: error.stack,
});
throw error;
@@ -8,8 +8,10 @@ import type {
PageSpeedDetailsResult,
GamesMap,
} from "@/types/monitor.js";
import type { GeoContinent } from "@/types/geoCheck.js";
import type {
IChecksRepository,
IGeoChecksRepository,
IIncidentsRepository,
IMonitorsRepository,
IMonitorStatsRepository,
@@ -36,6 +38,7 @@ export interface IMonitorService {
getUptimeDetailsById(args: { teamId: string; monitorId: string; dateRange: string; normalize?: boolean }): Promise<UptimeDetailsResult>;
getHardwareDetailsById(args: { teamId: string; monitorId: string; dateRange: string }): Promise<HardwareDetailsResult>;
getPageSpeedDetailsById(args: { teamId: string; monitorId: string; dateRange: string }): Promise<PageSpeedDetailsResult>;
getGeoChecksByMonitorId(args: { teamId: string; monitorId: string; dateRange: string; continents?: GeoContinent[] }): Promise<any>;
getMonitorById(args: { teamId: string; monitorId: string }): Promise<Monitor>;
getMonitorsByTeamId(args: {
teamId: string;
@@ -84,6 +87,7 @@ export class MonitorService implements IMonitorService {
private games: any;
private monitorsRepository: IMonitorsRepository;
private checksRepository: IChecksRepository;
private geoChecksRepository: IGeoChecksRepository;
private monitorStatsRepository: IMonitorStatsRepository;
private statusPagesRepository: IStatusPagesRepository;
private incidentsRepository: IIncidentsRepository;
@@ -95,6 +99,7 @@ export class MonitorService implements IMonitorService {
games,
monitorsRepository,
checksRepository,
geoChecksRepository,
monitorStatsRepository,
statusPagesRepository,
incidentsRepository,
@@ -105,6 +110,7 @@ export class MonitorService implements IMonitorService {
games: any;
monitorsRepository: IMonitorsRepository;
checksRepository: IChecksRepository;
geoChecksRepository: IGeoChecksRepository;
monitorStatsRepository: IMonitorStatsRepository;
statusPagesRepository: IStatusPagesRepository;
incidentsRepository: IIncidentsRepository;
@@ -115,6 +121,7 @@ export class MonitorService implements IMonitorService {
this.games = games;
this.monitorsRepository = monitorsRepository;
this.checksRepository = checksRepository;
this.geoChecksRepository = geoChecksRepository;
this.monitorStatsRepository = monitorStatsRepository;
this.statusPagesRepository = statusPagesRepository;
this.incidentsRepository = incidentsRepository;
@@ -316,6 +323,40 @@ export class MonitorService implements IMonitorService {
monitorStats,
};
};
getGeoChecksByMonitorId = async ({
teamId,
monitorId,
dateRange,
continents,
}: {
teamId: string;
monitorId: string;
dateRange: string;
continents?: GeoContinent[];
}): Promise<any> => {
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
if (!monitor) {
throw new AppError({ message: `Monitor with ID ${monitorId} not found.`, status: 404 });
}
if (monitor.type !== "http" || !monitor.geoCheckEnabled) {
return { groupedGeoChecks: [] };
}
const rangeKey = (dateRange as DateRangeKey) ?? "recent";
const { start, end } = this.getDateRange(rangeKey);
const groupedGeoChecks = await this.geoChecksRepository.findGroupedByMonitorIdAndDateRange(
monitor.id,
start,
end,
this.getDateFormat(rangeKey),
continents
);
return { groupedGeoChecks };
};
getMonitorById = async ({ teamId, monitorId }: { teamId: string; monitorId: string }): Promise<Monitor> => {
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
return monitor;
@@ -439,6 +480,14 @@ export class MonitorService implements IMonitorService {
});
});
await this.geoChecksRepository.deleteByMonitorId(monitor.id).catch((err: any) => {
this.logger.warn({
message: `Error deleting geo checks for monitor ${monitor.id} with name ${monitor.name}`,
service: SERVICE_NAME,
stack: err.stack,
});
});
await this.jobQueue.deleteJob(monitor);
return monitor;
};
@@ -450,6 +499,7 @@ export class MonitorService implements IMonitorService {
try {
await this.jobQueue.deleteJob(monitor);
await this.checksRepository.deleteByMonitorId(monitor.id);
await this.geoChecksRepository.deleteByMonitorId(monitor.id);
await this.statusPagesRepository.removeMonitorFromStatusPages(monitor.id);
await this.monitorStatsRepository.deleteByMonitorId(monitor.id);
} catch (error: any) {
@@ -1,5 +1,7 @@
import { IMonitorsRepository } from "@/repositories/index.js";
import { ILogger } from "@/utils/logger.js";
import Scheduler from "super-simple-scheduler";
import { ISuperSimpleQueueHelper } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
const SERVICE_NAME = "JobQueue";
type QueueJobFailure = {
@@ -54,22 +56,12 @@ export interface ISuperSimpleQueue {
class SuperSimpleQueue implements ISuperSimpleQueue {
static SERVICE_NAME = SERVICE_NAME;
private logger: any;
private helper: any;
private logger: ILogger;
private helper: ISuperSimpleQueueHelper;
private monitorsRepository: IMonitorsRepository;
private readonly scheduler: Scheduler;
constructor({
logger,
helper,
monitorsRepository,
scheduler,
}: {
logger: any;
helper: any;
monitorsRepository: IMonitorsRepository;
scheduler: Scheduler;
}) {
constructor(logger: ILogger, helper: ISuperSimpleQueueHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
this.logger = logger;
this.helper = helper;
this.monitorsRepository = monitorsRepository;
@@ -80,14 +72,14 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
return SuperSimpleQueue.SERVICE_NAME;
}
static async create({ logger, helper, monitorsRepository }: { logger: any; helper: any; monitorsRepository: IMonitorsRepository }) {
static async create(logger: ILogger, helper: ISuperSimpleQueueHelper, monitorsRepository: IMonitorsRepository) {
const scheduler = new Scheduler({
// storeType: "mongo",
// storeType: "redis",
logLevel: "debug",
// dbUri: envSettings.dbConnectionString,
});
const instance = new SuperSimpleQueue({ logger, helper, monitorsRepository, scheduler });
const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
await instance.init();
return instance;
}
@@ -97,6 +89,7 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
this.scheduler.start();
this.scheduler.addTemplate("monitor-job", this.helper.getMonitorJob());
this.scheduler.addTemplate("geo-check-job", this.helper.getGeoCheckJob());
this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
const monitors = await this.monitorsRepository.findAll();
if (!monitors) {
@@ -112,7 +105,7 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
return true;
} catch (error) {
} catch (error: any) {
this.logger.error({
message: "Failed to initialize SuperSimpleQueue",
service: SERVICE_NAME,
@@ -131,17 +124,30 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
active: monitor.isActive,
data: monitor,
});
// Add geo check job if enabled for HTTP monitors
if (monitor.geoCheckEnabled && monitor.type === "http") {
this.scheduler.addJob({
id: `${monitorId}-geo`,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
}
};
deleteJob = async (monitor: any) => {
this.scheduler.removeJob(monitor.id);
this.scheduler.removeJob(`${monitor.id}-geo`);
};
pauseJob = async (monitor: any) => {
const result = await this.scheduler.pauseJob(monitor.id);
if (result === false) {
throw new Error("Failed to resume monitor");
throw new Error("Failed to pause monitor");
}
await this.scheduler.pauseJob(`${monitor.id}-geo`);
this.logger.debug({
message: `Paused monitor ${monitor.id}`,
service: SERVICE_NAME,
@@ -154,6 +160,9 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
if (result === false) {
throw new Error("Failed to resume monitor");
}
await this.scheduler.resumeJob(`${monitor.id}-geo`);
this.logger.debug({
message: `Resumed monitor ${monitor.id}`,
service: SERVICE_NAME,
@@ -163,6 +172,29 @@ class SuperSimpleQueue implements ISuperSimpleQueue {
updateJob = async (monitor: any) => {
this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
// Handle geo check job lifecycle
const geoJobId = `${monitor.id}-geo`;
if (monitor.geoCheckEnabled && monitor.type === "http") {
// Check if geo job exists
const existingGeoJob = await this.scheduler.getJob(geoJobId);
if (existingGeoJob) {
// Update existing geo job
this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
} else {
// Create new geo job
this.scheduler.addJob({
id: geoJobId,
template: "geo-check-job",
repeat: monitor.geoCheckInterval,
active: monitor.isActive,
data: monitor,
});
}
} else {
// Remove geo job if disabled or monitor type changed
this.scheduler.removeJob(geoJobId);
}
};
shutdown = async () => {
@@ -4,6 +4,7 @@ import { AppError } from "@/utils/AppError.js";
import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js";
import type { StatusChangeResult, MonitorStatusResponse, HardwareStatusPayload, MonitorStatus } from "@/types/index.js";
import IncidentService from "@/service/business/incidentService.js";
import type { IGeoChecksService } from "@/service/business/geoChecksService.js";
import {
IMaintenanceWindowsRepository,
IMonitorsRepository,
@@ -11,7 +12,18 @@ import {
IMonitorStatsRepository,
IChecksRepository,
IIncidentsRepository,
IGeoChecksRepository,
} from "@/repositories/index.js";
import { ILogger } from "@/utils/logger.js";
import { IBufferService } from "../bufferService.js";
export interface ISuperSimpleQueueHelper {
readonly serviceName: string;
getMonitorJob(): (monitor: Monitor) => Promise<void>;
getCleanupOrphanedJob(): () => Promise<void>;
getGeoCheckJob(): (monitor: Monitor) => Promise<void>;
isInMaintenanceWindow(monitorId: string, teamId: string): Promise<boolean>;
}
export interface MonitorActionDecision {
shouldCreateIncident: boolean;
@@ -27,7 +39,7 @@ export interface MonitorActionDecision {
};
}
class SuperSimpleQueueHelper {
class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
static SERVICE_NAME = SERVICE_NAME;
private logger: any;
@@ -35,7 +47,7 @@ class SuperSimpleQueueHelper {
private statusService: IStatusService;
private notificationsService: INotificationsService;
private checkService: any;
private buffer: any;
private buffer: IBufferService;
private incidentService: IncidentService;
private maintenanceWindowsRepository: IMaintenanceWindowsRepository;
private monitorsRepository: IMonitorsRepository;
@@ -43,36 +55,26 @@ class SuperSimpleQueueHelper {
private monitorStatsRepository: IMonitorStatsRepository;
private checksRepository: IChecksRepository;
private incidentsRepository: IIncidentsRepository;
private geoChecksService: IGeoChecksService;
private geoChecksRepository: IGeoChecksRepository;
constructor({
logger,
networkService,
statusService,
notificationsService,
checkService,
buffer,
incidentService,
maintenanceWindowsRepository,
monitorsRepository,
teamsRepository,
monitorStatsRepository,
checksRepository,
incidentsRepository,
}: {
logger: any;
networkService: INetworkService;
statusService: IStatusService;
notificationsService: INotificationsService;
checkService: any;
buffer: any;
incidentService: IncidentService;
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
monitorsRepository: IMonitorsRepository;
teamsRepository: ITeamsRepository;
monitorStatsRepository: IMonitorStatsRepository;
checksRepository: IChecksRepository;
incidentsRepository: IIncidentsRepository;
}) {
constructor(
logger: ILogger,
networkService: INetworkService,
statusService: IStatusService,
notificationsService: INotificationsService,
checkService: any,
buffer: IBufferService,
incidentService: IncidentService,
maintenanceWindowsRepository: IMaintenanceWindowsRepository,
monitorsRepository: IMonitorsRepository,
teamsRepository: ITeamsRepository,
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository,
incidentsRepository: IIncidentsRepository,
geoChecksService: IGeoChecksService,
geoChecksRepository: IGeoChecksRepository
) {
this.logger = logger;
this.networkService = networkService;
this.statusService = statusService;
@@ -86,6 +88,8 @@ class SuperSimpleQueueHelper {
this.monitorStatsRepository = monitorStatsRepository;
this.checksRepository = checksRepository;
this.incidentsRepository = incidentsRepository;
this.geoChecksService = geoChecksService;
this.geoChecksRepository = geoChecksRepository;
}
get serviceName() {
@@ -240,6 +244,16 @@ class SuperSimpleQueueHelper {
});
}
// Remove orphaned geo checks
const deletedGeoChecksCount = await this.geoChecksRepository.deleteByMonitorIdsNotIn(allMonitorIds);
if (deletedGeoChecksCount > 0) {
this.logger.info({
message: `Deleted ${deletedGeoChecksCount} orphaned geo checks`,
service: SERVICE_NAME,
method: "getCleanupOrphanedJob",
});
}
this.logger.info({
message: "Cleanup of orphaned data completed",
service: SERVICE_NAME,
@@ -257,6 +271,81 @@ class SuperSimpleQueueHelper {
};
};
getGeoCheckJob = () => {
return async (monitor: Monitor) => {
try {
const monitorId = monitor.id;
const teamId = monitor.teamId;
// Step 1: Validate monitor eligibility
if (!monitorId) {
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getGeoCheckJob" });
}
if (monitor.type !== "http") {
this.logger.debug({
message: `Monitor ${monitorId} is not HTTP type, skipping geo check`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
if (!monitor.geoCheckEnabled) {
return;
}
if (!monitor.geoCheckLocations || monitor.geoCheckLocations.length === 0) {
this.logger.warn({
message: `No geo check locations configured for monitor ${monitorId}`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
// Step 2: Check for maintenance window
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitorId, teamId);
if (maintenanceWindowActive) {
this.logger.debug({
message: `Monitor ${monitorId} is in maintenance window, skipping geo check`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
// Step 3: Build geo check (handles API calls and polling)
const geoCheck = await this.geoChecksService.buildGeoCheck(monitor);
if (!geoCheck) {
this.logger.warn({
message: `No geo check could be built for monitor ${monitorId}`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
return;
}
// Step 4: Add geo check to buffer
this.buffer.addGeoCheckToBuffer(geoCheck);
this.logger.debug({
message: `Geo check job executed for monitor ${monitorId}`,
service: SERVICE_NAME,
method: "getGeoCheckJob",
});
} catch (error: any) {
this.logger.error({
message: error.message,
service: SERVICE_NAME,
method: "getGeoCheckJob",
stack: error.stack,
});
// Don't throw - geo check failures shouldn't crash the job scheduler
}
};
};
async isInMaintenanceWindow(monitorId: string, teamId: string) {
const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
// Check for active maintenance window:
@@ -1,29 +1,38 @@
import type { Check } from "@/types/index.js";
import type { GeoCheck } from "@/types/index.js";
import type { IGeoChecksService } from "../business/geoChecksService.js";
import type { ILogger } from "@/utils/logger.js";
import type { ISettingsService } from "@/service/system/settingsService.js";
const SERVICE_NAME = "BufferService";
export interface IBufferService {
addToBuffer(check: Check): void;
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
removeCheckFromBuffer(check: Check): boolean;
scheduleNextFlush(): void;
flushBuffer(): Promise<void>;
flushGeoBuffer(): Promise<void>;
}
class BufferService implements IBufferService {
static SERVICE_NAME = SERVICE_NAME;
private BUFFER_TIMEOUT: number;
private logger: any;
private logger: ILogger;
private SERVICE_NAME: string;
private buffer: any[];
private geoBuffer: any[];
private bufferTimer: NodeJS.Timeout | null = null;
private checksService: any;
private geoChecksService: IGeoChecksService;
constructor({ logger, checkService, settingsService }: { logger: any; checkService: any; settingsService: any }) {
constructor(logger: ILogger, checkService: any, geoChecksService: IGeoChecksService, settingsService: ISettingsService) {
this.BUFFER_TIMEOUT = settingsService.getSettings().nodeEnv === "development" ? 10 : 1000 * 60 * 1; // 1 minute
this.logger = logger;
this.checksService = checkService;
this.geoChecksService = geoChecksService;
this.SERVICE_NAME = SERVICE_NAME;
this.buffer = [];
this.geoBuffer = [];
this.scheduleNextFlush();
this.logger.info({
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
@@ -49,6 +58,19 @@ class BufferService implements IBufferService {
}
}
addGeoCheckToBuffer(geoCheck: GeoCheck) {
try {
this.geoBuffer.push(geoCheck);
} catch (error: any) {
this.logger.error({
message: error.message,
service: this.SERVICE_NAME,
method: "addGeoCheckToBuffer",
stack: error.stack,
});
}
}
removeCheckFromBuffer(checkToRemove: Check) {
try {
if (!checkToRemove) {
@@ -94,6 +116,7 @@ class BufferService implements IBufferService {
this.bufferTimer = setTimeout(async () => {
try {
await this.flushBuffer();
await this.flushGeoBuffer();
} catch (error: any) {
this.logger.error({
message: `Error in flush cycle: ${error.message}`,
@@ -110,18 +133,47 @@ class BufferService implements IBufferService {
async flushBuffer() {
try {
if (this.buffer.length > 0) {
this.logger.debug({
message: `Flushing ${this.buffer.length} checks to database`,
service: this.SERVICE_NAME,
method: "flushBuffer",
});
await this.checksService.createChecks(this.buffer);
this.buffer = [];
}
} catch (error: any) {
this.logger.error({
message: error.message,
message: `Error flushing checks buffer: ${error.message}`,
service: this.SERVICE_NAME,
method: "flushBuffer",
stack: error.stack,
});
// Clear buffer even on error to prevent infinite retry loops
this.buffer = [];
}
}
this.buffer = [];
async flushGeoBuffer() {
try {
if (this.geoBuffer.length > 0) {
this.logger.debug({
message: `Flushing ${this.geoBuffer.length} geo checks to database`,
service: this.SERVICE_NAME,
method: "flushGeoBuffer",
});
await this.geoChecksService.createGeoChecks(this.geoBuffer);
this.geoBuffer = [];
}
} catch (error: any) {
this.logger.error({
message: `Error flushing geo checks buffer: ${error.message}`,
service: this.SERVICE_NAME,
method: "flushGeoBuffer",
stack: error.stack,
});
// Clear buffer even on error to prevent infinite retry loops
this.geoBuffer = [];
}
}
}
@@ -0,0 +1,210 @@
import type { GeoContinent, GeoCheckResult, GeoCheckTimings, GeoCheckLocation } from "@/types/geoCheck.js";
import type { ILogger } from "@/utils/logger.js";
import got from "got";
const SERVICE_NAME = "GlobalPingService";
const GLOBAL_PING_API_BASE = "https://api.globalping.io/v1";
const POLL_INTERVAL_MS = 2000;
const MAX_POLL_TIMEOUT_MS = 30000;
interface GlobalPingMeasurementRequest {
type: "http";
target: string;
locations: Array<{ continent: GeoContinent }>;
limit: number;
}
interface GlobalPingMeasurementResponse {
id: string;
type: string;
status: "in-progress" | "finished" | "failed";
probesCount: number;
results?: GlobalPingProbeResult[];
}
interface GlobalPingProbeResult {
probe: {
continent: GeoContinent;
region: string;
country: string;
state: string | null;
city: string;
longitude: number;
latitude: number;
};
result: {
status: "finished" | "failed" | "timeout";
statusCode?: number;
statusCodeName?: string;
timings?: {
total: number;
dns: number;
tcp: number;
tls: number;
firstByte: number;
download: number;
};
rawOutput?: string;
};
}
export interface IGlobalPingService {
readonly serviceName: string;
createMeasurement(url: string, locations: GeoContinent[]): Promise<string | null>;
pollForResults(measurementId: string, timeoutMs?: number): Promise<GeoCheckResult[]>;
}
class GlobalPingService implements IGlobalPingService {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
constructor(logger: ILogger) {
this.logger = logger;
}
get serviceName() {
return GlobalPingService.SERVICE_NAME;
}
async createMeasurement(url: string, locations: GeoContinent[]): Promise<string | null> {
try {
// GlobalPing API expects target without protocol (http:// or https://)
const cleanTarget = url.replace(/^https?:\/\//, "");
const requestBody: GlobalPingMeasurementRequest = {
type: "http",
target: cleanTarget,
locations: locations.map((continent) => ({ continent })),
limit: locations.length,
};
const response = await got.post<GlobalPingMeasurementResponse>(`${GLOBAL_PING_API_BASE}/measurements`, {
json: requestBody,
responseType: "json",
timeout: { request: 10000 },
});
const measurementId = response.body.id;
this.logger.debug({
message: `Created GlobalPing measurement: ${measurementId}`,
service: SERVICE_NAME,
method: "createMeasurement",
details: { measurementId, url, locations },
});
return measurementId;
} catch (error: any) {
this.logger.error({
message: "GlobalPing API unavailable, skipping geo check",
service: SERVICE_NAME,
method: "createMeasurement",
details: error,
});
return null;
}
}
async pollForResults(measurementId: string, timeoutMs: number = MAX_POLL_TIMEOUT_MS): Promise<GeoCheckResult[]> {
const startTime = Date.now();
while (Date.now() - startTime < timeoutMs) {
try {
const response = await got.get<GlobalPingMeasurementResponse>(`${GLOBAL_PING_API_BASE}/measurements/${measurementId}`, {
responseType: "json",
timeout: { request: 5000 },
});
const measurement = response.body;
if (measurement.status === "finished") {
const results = this.transformResults(measurement.results || []);
this.logger.debug({
message: `GlobalPing measurement completed: ${measurementId}`,
service: SERVICE_NAME,
method: "pollForResults",
details: { measurementId, resultsCount: results.length },
});
return results;
}
if (measurement.status === "failed") {
this.logger.warn({
message: `GlobalPing measurement failed: ${measurementId}`,
service: SERVICE_NAME,
method: "pollForResults",
});
return [];
}
// Still in-progress, wait and poll again
await this.sleep(POLL_INTERVAL_MS);
} catch (error: any) {
this.logger.error({
message: "Error polling GlobalPing API",
service: SERVICE_NAME,
method: "pollForResults",
details: error.message,
});
return [];
}
}
// Timeout reached
this.logger.warn({
message: `GlobalPing measurement polling timeout: ${measurementId}`,
service: SERVICE_NAME,
method: "pollForResults",
details: { measurementId, timeoutMs },
});
return [];
}
private transformResults(probeResults: GlobalPingProbeResult[]): GeoCheckResult[] {
const successfulResults: GeoCheckResult[] = [];
for (const probeResult of probeResults) {
// Skip failed or timeout results
if (probeResult.result.status !== "finished" || !probeResult.result.statusCode || !probeResult.result.timings) {
continue;
}
const location: GeoCheckLocation = {
continent: probeResult.probe.continent,
region: probeResult.probe.region,
country: probeResult.probe.country,
state: probeResult.probe.state || "",
city: probeResult.probe.city,
longitude: probeResult.probe.longitude,
latitude: probeResult.probe.latitude,
};
const timings: GeoCheckTimings = {
total: probeResult.result.timings.total,
dns: probeResult.result.timings.dns,
tcp: probeResult.result.timings.tcp,
tls: probeResult.result.timings.tls,
firstByte: probeResult.result.timings.firstByte,
download: probeResult.result.timings.download,
};
const result: GeoCheckResult = {
location,
status: probeResult.result.statusCode >= 200 && probeResult.result.statusCode < 300,
statusCode: probeResult.result.statusCode,
timings,
};
successfulResults.push(result);
}
return successfulResults;
}
private sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}
export default GlobalPingService;
@@ -1,11 +1,14 @@
import { HTTPError, RequestError } from "got";
import type { Got, Response } from "got";
import type { Monitor, MonitorStatusResponse, GrpcStatusPayload } from "@/types/index.js";
import type { AxiosStatic } from "axios";
import { AppError } from "@/utils/AppError.js";
import path from "path";
import { fileURLToPath } from "url";
import CacheableLookup from "cacheable-lookup";
import { ISettingsService } from "../system/settingsService.js";
import { ILogger } from "@/utils/logger.js";
const SERVICE_NAME = "NetworkService";
type MonitorStatusResponseOverrides<T> = Partial<Omit<MonitorStatusResponse<T>, "monitorId" | "teamId" | "type">>;
@@ -50,16 +53,16 @@ class NetworkService implements INetworkService {
private NETWORK_ERROR: number;
private PING_ERROR: number;
private axios: any;
private axios: AxiosStatic;
private got: Got;
private https: any;
private jmespath: any;
private GameDig: any;
private ping: any;
private logger: any;
private logger: ILogger;
private Docker: any;
private net: any;
private settingsService: any;
private settingsService: ISettingsService;
private grpc: any;
private protoLoader: any;
@@ -117,35 +120,20 @@ class NetworkService implements INetworkService {
};
};
constructor({
axios,
got,
https,
jmespath,
GameDig,
ping,
logger,
http,
Docker,
net,
settingsService,
grpc,
protoLoader,
}: {
axios: any;
got: Got;
https: any;
jmespath: any;
GameDig: any;
ping: any;
logger: any;
http: any;
Docker: any;
net: any;
settingsService: any;
grpc: any;
protoLoader: any;
}) {
constructor(
axios: AxiosStatic,
got: Got,
https: any,
jmespath: any,
GameDig: any,
ping: any,
logger: ILogger,
Docker: any,
net: any,
settingsService: ISettingsService,
grpc: any,
protoLoader: any
) {
this.TYPE_PING = "ping";
this.TYPE_HTTP = "http";
this.TYPE_PAGESPEED = "pagespeed";
@@ -449,7 +437,6 @@ class NetworkService implements INetworkService {
const docker = new this.Docker({
socketPath: "/var/run/docker.sock",
handleError: true, // Enable error handling
});
const dockerResponse = this.buildStatusResponse({
@@ -545,6 +532,10 @@ class NetworkService implements INetworkService {
private async requestPort(monitor: Monitor): Promise<MonitorStatusResponse> {
try {
const { url, port } = monitor;
if (!port) {
throw new Error("Port is required for port monitor");
}
const { response, responseTime, error } = await this.timeRequest(async () => {
return new Promise((resolve, reject) => {
const socket = this.net.createConnection(
@@ -611,9 +602,9 @@ class NetworkService implements INetworkService {
});
const state = await this.GameDig.query({
type: gameId,
type: gameId ?? "unknown",
host: url,
port: port,
port: port ?? 0,
}).catch((error: any) => {
this.logger.warn({
message: error.message,