diff --git a/server/src/config/services.ts b/server/src/config/services.ts index 6843fe64b..ecb49ab40 100644 --- a/server/src/config/services.ts +++ b/server/src/config/services.ts @@ -202,6 +202,10 @@ export const initializeServices = async ({ incidentService, maintenanceWindowsRepository, monitorsRepository, + teamsRepository, + monitorStatsRepository, + checksRepository, + incidentsRepository, }); const superSimpleQueue = await SuperSimpleQueue.create({ diff --git a/server/src/repositories/checks/IChecksRepository.ts b/server/src/repositories/checks/IChecksRepository.ts index b75b18f13..31aac0ec9 100644 --- a/server/src/repositories/checks/IChecksRepository.ts +++ b/server/src/repositories/checks/IChecksRepository.ts @@ -39,4 +39,5 @@ export interface IChecksRepository { //delete deleteByMonitorId(monitorId: string): Promise; deleteByTeamId(teamId: string): Promise; + deleteByMonitorIdsNotIn(monitorIds: string[]): Promise; } diff --git a/server/src/repositories/checks/MongoChecksRepistory.ts b/server/src/repositories/checks/MongoChecksRepistory.ts index c9cdbffd6..bc25420b9 100644 --- a/server/src/repositories/checks/MongoChecksRepistory.ts +++ b/server/src/repositories/checks/MongoChecksRepistory.ts @@ -13,7 +13,7 @@ import type { GotTimings, MonitorType, } from "@/types/index.js"; -import { CheckModel, type CheckDocument } from "@/db/models/index.js"; +import { CheckModel, MonitorModel, type CheckDocument } from "@/db/models/index.js"; import mongoose from "mongoose"; const SERVICE_NAME = "StatusService"; @@ -210,19 +210,16 @@ class MongoChecksRepository implements IChecksRepository { private toDocument = (check: Partial): CheckDocument => { // Map id to _id for MongoDB storage const { id, metadata, ...rest } = check; + if (!metadata || !metadata.monitorId || !metadata.teamId) { + throw new Error(`Check must have valid metadata with monitorId and teamId. Got: ${JSON.stringify({ id, metadata })}`); + } return { _id: id ? new mongoose.Types.ObjectId(id) : new mongoose.Types.ObjectId(), - metadata: metadata - ? { - monitorId: new mongoose.Types.ObjectId(metadata.monitorId), - teamId: new mongoose.Types.ObjectId(metadata.teamId), - type: metadata.type, - } - : { - monitorId: new mongoose.Types.ObjectId(), - teamId: new mongoose.Types.ObjectId(), - type: "http", - }, + metadata: { + monitorId: new mongoose.Types.ObjectId(metadata.monitorId), + teamId: new mongoose.Types.ObjectId(metadata.teamId), + type: metadata.type, + }, ...rest, } as unknown as CheckDocument; }; @@ -417,6 +414,12 @@ class MongoChecksRepository implements IChecksRepository { return deleteResult.deletedCount; }; + deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise => { + const objectIds = monitorIds.map((id) => new mongoose.Types.ObjectId(id)); + const result = await CheckModel.deleteMany({ "metadata.monitorId": { $nin: objectIds } }); + return result.deletedCount ?? 0; + }; + private findUptimeDateRangeChecks = async ( monitorType: Exclude, monitorObjectId: mongoose.Types.ObjectId, diff --git a/server/src/repositories/incidents/IIncidentsRepository.ts b/server/src/repositories/incidents/IIncidentsRepository.ts index 42948a9ee..c4fcef2ae 100644 --- a/server/src/repositories/incidents/IIncidentsRepository.ts +++ b/server/src/repositories/incidents/IIncidentsRepository.ts @@ -24,5 +24,6 @@ export interface IIncidentsRepository { updateById(incidentId: string, teamId: string, updateData: Partial): Promise; // delete deleteByMonitorId(monitorId: string, teamId: string): Promise; + deleteByMonitorIdsNotIn(monitorIds: string[]): Promise; // other } diff --git a/server/src/repositories/incidents/MongoIncidentRepository.ts b/server/src/repositories/incidents/MongoIncidentRepository.ts index b34af4ba7..93ecac248 100644 --- a/server/src/repositories/incidents/MongoIncidentRepository.ts +++ b/server/src/repositories/incidents/MongoIncidentRepository.ts @@ -281,5 +281,11 @@ class MongoIncidentRepository implements IIncidentsRepository { }); return result.deletedCount || 0; }; + + deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise => { + const objectIds = monitorIds.map((id) => new mongoose.Types.ObjectId(id)); + const result = await IncidentModel.deleteMany({ monitorId: { $nin: objectIds } }); + return result.deletedCount ?? 0; + }; } export default MongoIncidentRepository; diff --git a/server/src/repositories/monitor-stats/IMonitorStatsRepository.ts b/server/src/repositories/monitor-stats/IMonitorStatsRepository.ts index c718e7eb3..1ad05596e 100644 --- a/server/src/repositories/monitor-stats/IMonitorStatsRepository.ts +++ b/server/src/repositories/monitor-stats/IMonitorStatsRepository.ts @@ -7,5 +7,7 @@ export interface IMonitorStatsRepository { // update // delete deleteByMonitorId(monitorId: string): Promise; + deleteByMonitorIds(monitorIds: string[]): Promise; + deleteByMonitorIdsNotIn(monitorIds: string[]): Promise; // other } diff --git a/server/src/repositories/monitor-stats/MongoMonitorStatsRepository.ts b/server/src/repositories/monitor-stats/MongoMonitorStatsRepository.ts index 7211c7d89..0a4905fc1 100644 --- a/server/src/repositories/monitor-stats/MongoMonitorStatsRepository.ts +++ b/server/src/repositories/monitor-stats/MongoMonitorStatsRepository.ts @@ -52,6 +52,18 @@ class MongoMonitorStatsRepository implements IMonitorStatsRepository { } return this.toEntity(deleted); }; + + deleteByMonitorIds = async (monitorIds: string[]): Promise => { + const objectIds = monitorIds.map((id) => new mongoose.Types.ObjectId(id)); + const result = await MonitorStatsModel.deleteMany({ monitorId: { $in: objectIds } }); + return result.deletedCount ?? 0; + }; + + deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise => { + const objectIds = monitorIds.map((id) => new mongoose.Types.ObjectId(id)); + const result = await MonitorStatsModel.deleteMany({ monitorId: { $nin: objectIds } }); + return result.deletedCount ?? 0; + }; } export default MongoMonitorStatsRepository; diff --git a/server/src/repositories/monitors/IMonitorsRepository.ts b/server/src/repositories/monitors/IMonitorsRepository.ts index 2fbce3406..63940f591 100644 --- a/server/src/repositories/monitors/IMonitorsRepository.ts +++ b/server/src/repositories/monitors/IMonitorsRepository.ts @@ -41,4 +41,6 @@ export interface IMonitorsRepository { findMonitorsSummaryByTeamId(teamId: string, config?: SummaryConfig): Promise; findGroupsByTeamId(teamId: string): Promise; removeNotificationFromMonitors(notificationId: string): Promise; + deleteByTeamIdsNotIn(teamIds: string[]): Promise; + findAllMonitorIds(): Promise; } diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index c6f3266a6..728d2081f 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -431,6 +431,17 @@ class MongoMonitorsRepository implements IMonitorsRepository { createdAt: toDateString(doc.createdAt), }; }; + + deleteByTeamIdsNotIn = async (teamIds: string[]): Promise => { + const objectIds = teamIds.map((id) => new mongoose.Types.ObjectId(id)); + const result = await MonitorModel.deleteMany({ teamId: { $nin: objectIds } }); + return result.deletedCount ?? 0; + }; + + findAllMonitorIds = async (): Promise => { + const monitors = await MonitorModel.find({}, { _id: 1 }).lean(); + return monitors.map((doc) => doc._id.toString()); + }; } export default MongoMonitorsRepository; diff --git a/server/src/repositories/teams/ITeamsRepository.ts b/server/src/repositories/teams/ITeamsRepository.ts index 9ccdd97fb..ba1b353cf 100644 --- a/server/src/repositories/teams/ITeamsRepository.ts +++ b/server/src/repositories/teams/ITeamsRepository.ts @@ -6,4 +6,5 @@ export interface ITeamsRepository { // update // delete // other + findAllTeamIds(): Promise; } diff --git a/server/src/repositories/teams/MongoTeamsRepository.ts b/server/src/repositories/teams/MongoTeamsRepository.ts index 93a306787..04e2be6f4 100644 --- a/server/src/repositories/teams/MongoTeamsRepository.ts +++ b/server/src/repositories/teams/MongoTeamsRepository.ts @@ -31,6 +31,11 @@ class MongoTeamsRepository implements ITeamsRepository { const team = await TeamModel.create({ email }); return this.toEntity(team); }; + + findAllTeamIds = async (): Promise => { + const teams = await TeamModel.find({}, { _id: 1 }).lean(); + return teams.map((team) => this.toStringId(team._id)); + }; } export default MongoTeamsRepository; diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts index 8fd55ff70..1947549c9 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueue.ts @@ -97,6 +97,7 @@ class SuperSimpleQueue implements ISuperSimpleQueue { this.scheduler.start(); this.scheduler.addTemplate("monitor-job", this.helper.getMonitorJob()); + this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob()); const monitors = await this.monitorsRepository.findAll(); if (!monitors) { return true; @@ -108,6 +109,8 @@ class SuperSimpleQueue implements ISuperSimpleQueue { }, randomOffset); } + this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true }); + return true; } catch (error) { this.logger.error({ diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index 08bc5e257..4d9876f2f 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -4,7 +4,14 @@ import { AppError } from "@/utils/AppError.js"; import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js"; import type { StatusChangeResult, MonitorStatusResponse, HardwareStatusPayload, MonitorStatus } from "@/types/index.js"; import IncidentService from "@/service/business/incidentService.js"; -import { IMaintenanceWindowsRepository, IMonitorsRepository } from "@/repositories/index.js"; +import { + IMaintenanceWindowsRepository, + IMonitorsRepository, + ITeamsRepository, + IMonitorStatsRepository, + IChecksRepository, + IIncidentsRepository, +} from "@/repositories/index.js"; export interface MonitorActionDecision { shouldCreateIncident: boolean; @@ -32,6 +39,10 @@ class SuperSimpleQueueHelper { private incidentService: IncidentService; private maintenanceWindowsRepository: IMaintenanceWindowsRepository; private monitorsRepository: IMonitorsRepository; + private teamsRepository: ITeamsRepository; + private monitorStatsRepository: IMonitorStatsRepository; + private checksRepository: IChecksRepository; + private incidentsRepository: IIncidentsRepository; constructor({ logger, @@ -43,6 +54,10 @@ class SuperSimpleQueueHelper { incidentService, maintenanceWindowsRepository, monitorsRepository, + teamsRepository, + monitorStatsRepository, + checksRepository, + incidentsRepository, }: { logger: any; networkService: INetworkService; @@ -53,6 +68,10 @@ class SuperSimpleQueueHelper { incidentService: IncidentService; maintenanceWindowsRepository: IMaintenanceWindowsRepository; monitorsRepository: IMonitorsRepository; + teamsRepository: ITeamsRepository; + monitorStatsRepository: IMonitorStatsRepository; + checksRepository: IChecksRepository; + incidentsRepository: IIncidentsRepository; }) { this.logger = logger; this.networkService = networkService; @@ -63,6 +82,10 @@ class SuperSimpleQueueHelper { this.incidentService = incidentService; this.maintenanceWindowsRepository = maintenanceWindowsRepository; this.monitorsRepository = monitorsRepository; + this.teamsRepository = teamsRepository; + this.monitorStatsRepository = monitorStatsRepository; + this.checksRepository = checksRepository; + this.incidentsRepository = incidentsRepository; } get serviceName() { @@ -111,7 +134,7 @@ class SuperSimpleQueueHelper { return; } // Step 4 Add check to buffer - this.buffer.addToBuffer({ check }); + this.buffer.addToBuffer(check); // Step 4. Update monitor status const statusChangeResult = await this.statusService.updateMonitorStatus(status, check); @@ -156,10 +179,72 @@ class SuperSimpleQueueHelper { getCleanupOrphanedJob = () => { return async () => { try { - // Remove orphaned monitors - // remove orphaned monitorStats + this.logger.info({ + message: "Starting cleanup of orphaned data", + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + + // Get all valid team IDs + const validTeamIds = await this.teamsRepository.findAllTeamIds(); + this.logger.debug({ + message: `Found ${validTeamIds.length} valid teams`, + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + + // Remove orphaned monitors (monitors without a valid team) + const deletedMonitorCount = await this.monitorsRepository.deleteByTeamIdsNotIn(validTeamIds); + if (deletedMonitorCount > 0) { + this.logger.info({ + message: `Deleted ${deletedMonitorCount} orphaned monitors`, + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + } + + // Remove orphaned monitorStats (stats without a valid monitor) + const allMonitorIds = await this.monitorsRepository.findAllMonitorIds(); + this.logger.debug({ + message: `Found ${allMonitorIds.length} valid monitors`, + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + + const deletedStatsCount = await this.monitorStatsRepository.deleteByMonitorIdsNotIn(allMonitorIds); + if (deletedStatsCount > 0) { + this.logger.info({ + message: `Deleted ${deletedStatsCount} orphaned monitor stats`, + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + } + // Remove orphaned checks + const deletedChecksCount = await this.checksRepository.deleteByMonitorIdsNotIn(allMonitorIds); + if (deletedChecksCount > 0) { + this.logger.info({ + message: `Deleted ${deletedChecksCount} orphaned checks`, + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + } + // Remove orphaned incidents + const deletedIncidentsCount = await this.incidentsRepository.deleteByMonitorIdsNotIn(allMonitorIds); + if (deletedIncidentsCount > 0) { + this.logger.info({ + message: `Deleted ${deletedIncidentsCount} orphaned incidents`, + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); + } + + this.logger.info({ + message: "Cleanup of orphaned data completed", + service: SERVICE_NAME, + method: "getCleanupOrphanedJob", + }); } catch (error: any) { this.logger.warn({ message: error.message, diff --git a/server/src/service/infrastructure/bufferService.ts b/server/src/service/infrastructure/bufferService.ts index 415c22281..c38fd52a4 100755 --- a/server/src/service/infrastructure/bufferService.ts +++ b/server/src/service/infrastructure/bufferService.ts @@ -60,9 +60,9 @@ class BufferService implements IBufferService { return check.id.toString() === checkToRemove.id.toString(); } return ( - check.monitorId?.toString() === checkToRemove.metadata.monitorId && - check.teamId?.toString() === checkToRemove.metadata.teamId && - check.type === checkToRemove.metadata.type && + check.metadata.monitorId?.toString() === checkToRemove.metadata.monitorId && + check.metadata.teamId?.toString() === checkToRemove.metadata.teamId && + check.metadata.type === checkToRemove.metadata.type && check.status === checkToRemove.status && check.statusCode === checkToRemove.statusCode && check.responseTime === checkToRemove.responseTime && diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index b01131bbe..ad112df21 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -362,7 +362,7 @@ export class StatusService implements IStatusService { }); return false; } - this.buffer.addToBuffer({ check }); + this.buffer.addToBuffer(check); return true; } catch (error: any) { this.logger.error({