From 6ad0a6a29d42dee8fb18e245517551e3807d6ad1 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 10 Mar 2026 10:20:23 -0700 Subject: [PATCH] validation fix, batch deleteOlderThan --- client/src/Pages/Settings/index.tsx | 7 +- client/src/Types/Check.ts | 9 +- client/src/Validation/settings.ts | 7 +- server/src/controllers/checkController.ts | 74 ++++++------- .../checks/MongoChecksRepistory.ts | 41 +++++-- server/src/routes/checkRoute.ts | 1 - server/src/service/business/checkService.ts | 103 ++++++++++-------- .../SuperSimpleQueueHelper.ts | 4 +- server/src/types/check.ts | 3 + server/src/validation/checkValidation.ts | 20 ++-- server/src/validation/settingsValidation.ts | 4 +- 11 files changed, 155 insertions(+), 118 deletions(-) diff --git a/client/src/Pages/Settings/index.tsx b/client/src/Pages/Settings/index.tsx index e96fc4837..679d66126 100644 --- a/client/src/Pages/Settings/index.tsx +++ b/client/src/Pages/Settings/index.tsx @@ -37,6 +37,7 @@ import { } from "@/Features/UI/uiSlice.js"; import timezones from "@/Utils/timezones.json"; import type { RootState } from "@/Types/state"; +import { CHECK_TTL_SENTINEL } from "@/Types/Check"; interface Timezone { id: string; @@ -477,18 +478,18 @@ export const SettingsPage = () => { {...field} fieldLabel={t("pages.settings.form.retention.option.days.label")} min={1} - max={366} + max={CHECK_TTL_SENTINEL} sliderMaxWidth={{ xs: "100%", md: "50%" }} value={field.value || 30} onChange={(_, value) => field.onChange(value)} valueLabelDisplay="auto" valueLabelFormat={(value: number) => - value >= 366 + value >= CHECK_TTL_SENTINEL ? t("pages.settings.form.retention.option.days.unlimited") : `${value}` } formatDisplayValue={(value: number) => - value >= 366 + value >= CHECK_TTL_SENTINEL ? t("pages.settings.form.retention.option.days.unlimited") : `${value}` } diff --git a/client/src/Types/Check.ts b/client/src/Types/Check.ts index c69fe7972..27db3d411 100644 --- a/client/src/Types/Check.ts +++ b/client/src/Types/Check.ts @@ -1,3 +1,5 @@ +export const CHECK_TTL_SENTINEL = 366; + export interface CheckMetadata { monitorId: string; teamId: string; @@ -125,8 +127,6 @@ export interface Check { timings?: CheckTimings; statusCode: number; message: string; - ack: boolean; - ackAt?: string | null; expiry: string; cpu?: CheckCpuInfo; memory?: CheckMemoryInfo; @@ -243,10 +243,7 @@ export interface ChecksSummary { downChecks: number; } -export type CheckSnapshot = Omit< - Check, - "metadata" | "ack" | "ackAt" | "expiry" | "__v" | "updatedAt" -> & { +export type CheckSnapshot = Omit & { originalResponseTime: number; }; diff --git a/client/src/Validation/settings.ts b/client/src/Validation/settings.ts index effb819a8..975927d1c 100644 --- a/client/src/Validation/settings.ts +++ b/client/src/Validation/settings.ts @@ -1,3 +1,4 @@ +import { CHECK_TTL_SENTINEL } from "@/Types/Check"; import { z } from "zod"; export const settingsSchema = z.object({ @@ -11,7 +12,11 @@ export const settingsSchema = z.object({ systemEmailSecure: z.boolean().optional(), systemEmailPool: z.boolean().optional(), showURL: z.boolean().optional(), - checkTTL: z.number().int().min(1, "Please enter a value").max(366, "Maximum 366"), + checkTTL: z + .number() + .int() + .min(1, "Please enter a value") + .max(CHECK_TTL_SENTINEL, `Maximum ${CHECK_TTL_SENTINEL}`), pagespeedApiKey: z .string() .transform((val) => (val.trim() === "" ? undefined : val.trim())) diff --git a/server/src/controllers/checkController.ts b/server/src/controllers/checkController.ts index ec6722bd3..879ceaffe 100755 --- a/server/src/controllers/checkController.ts +++ b/server/src/controllers/checkController.ts @@ -4,18 +4,19 @@ import { getChecksParamValidation, getChecksQueryValidation, getTeamChecksQueryValidation, + getChecksSummaryByTeamIdQueryValidation, deleteChecksParamValidation, - deleteChecksByTeamIdParamValidation, - updateChecksTTLBodyValidation, } from "@/validation/checkValidation.js"; +import { ICheckService } from "@/service/index.js"; +import { requireTeamId } from "@/controllers/controllerUtils.js"; const SERVICE_NAME = "checkController"; class CheckController { static SERVICE_NAME = SERVICE_NAME; - private checkService: any; - constructor(checkService: any) { + private checkService: ICheckService; + constructor(checkService: ICheckService) { this.checkService = checkService; } @@ -25,13 +26,20 @@ class CheckController { getChecksByMonitor = async (req: Request, res: Response, next: NextFunction) => { try { - getChecksParamValidation.parse(req.params); - getChecksQueryValidation.parse(req.query); + const validatedParams = getChecksParamValidation.parse(req.params); + const validatedQuery = getChecksQueryValidation.parse(req.query); + + const teamId = requireTeamId(req.user?.teamId); const result = await this.checkService.getChecksByMonitor({ - monitorId: req?.params?.monitorId, - query: req?.query, - teamId: req?.user?.teamId, + monitorId: validatedParams.monitorId, + teamId, + sortOrder: validatedQuery.sortOrder, + dateRange: validatedQuery.dateRange, + filter: validatedQuery.filter, + page: validatedQuery.page, + rowsPerPage: validatedQuery.rowsPerPage, + status: validatedQuery.status, }); return res.status(200).json({ @@ -46,10 +54,16 @@ class CheckController { getChecksByTeam = async (req: Request, res: Response, next: NextFunction) => { try { - getTeamChecksQueryValidation.parse(req.query); + const validatedQuery = getTeamChecksQueryValidation.parse(req.query); + const teamId = requireTeamId(req.user?.teamId); + const checkData = await this.checkService.getChecksByTeam({ - teamId: req?.user?.teamId, - query: req?.query, + teamId, + sortOrder: validatedQuery.sortOrder, + dateRange: validatedQuery.dateRange, + page: validatedQuery.page, + rowsPerPage: validatedQuery.rowsPerPage, + filter: validatedQuery.filter, }); return res.status(200).json({ success: true, @@ -63,8 +77,11 @@ class CheckController { getChecksSummaryByTeamId = async (req: Request, res: Response, next: NextFunction) => { try { - const dateRange = req.query?.dateRange || "hour"; - const summary = await this.checkService.getChecksSummaryByTeamId({ teamId: req?.user?.teamId, dateRange }); + const validatedQuery = getChecksSummaryByTeamIdQueryValidation.parse(req.query); + const teamId = requireTeamId(req.user?.teamId); + const dateRange = validatedQuery.dateRange ?? "hour"; + + const summary = await this.checkService.getChecksSummaryByTeamId({ teamId, dateRange }); return res.status(200).json({ success: true, msg: "Checks summary retrieved successfully", @@ -77,11 +94,12 @@ class CheckController { deleteChecks = async (req: Request, res: Response, next: NextFunction) => { try { - deleteChecksParamValidation.parse(req.params); + const validatedParams = deleteChecksParamValidation.parse(req.params); + const teamId = requireTeamId(req.user?.teamId); const deletedCount = await this.checkService.deleteChecks({ - monitorId: req.params.monitorId as string, - teamId: req?.user?.teamId, + monitorId: validatedParams.monitorId, + teamId, }); return res.status(200).json({ @@ -96,9 +114,9 @@ class CheckController { deleteChecksByTeamId = async (req: Request, res: Response, next: NextFunction) => { try { - deleteChecksByTeamIdParamValidation.parse(req.params); + const teamId = requireTeamId(req.user?.teamId); - const deletedCount = await this.checkService.deleteChecksByTeamId({ teamId: req?.user?.teamId }); + const deletedCount = await this.checkService.deleteChecksByTeamId({ teamId }); return res.status(200).json({ success: true, @@ -109,24 +127,6 @@ class CheckController { next(error); } }; - - updateChecksTTL = async (req: Request, res: Response, next: NextFunction) => { - try { - updateChecksTTLBodyValidation.parse(req.body); - - await this.checkService.updateChecksTTL({ - teamId: req?.user?.teamId, - ttl: req?.body?.ttl, - }); - - return res.status(200).json({ - success: true, - msg: "Checks TTL updated successfully", - }); - } catch (error) { - next(error); - } - }; } export default CheckController; diff --git a/server/src/repositories/checks/MongoChecksRepistory.ts b/server/src/repositories/checks/MongoChecksRepistory.ts index 3a4485f64..6c7d469ce 100644 --- a/server/src/repositories/checks/MongoChecksRepistory.ts +++ b/server/src/repositories/checks/MongoChecksRepistory.ts @@ -13,9 +13,10 @@ import type { GotTimings, MonitorType, } from "@/types/index.js"; -import { CheckModel, MonitorModel, type CheckDocument } from "@/db/models/index.js"; +import { CheckModel, type CheckDocument } from "@/db/models/index.js"; import mongoose from "mongoose"; import { getDateForRange } from "@/utils/dataUtils.js"; +import { ILogger } from "@/utils/logger.js"; const SERVICE_NAME = "StatusService"; @@ -26,8 +27,8 @@ type HardwareUpChecks = { totalChecks: number }; class MongoChecksRepository implements IChecksRepository { static SERVICE_NAME = SERVICE_NAME; - private logger: any; - constructor(logger: any) { + private logger: ILogger; + constructor(logger: ILogger) { this.logger = logger; } @@ -227,7 +228,7 @@ class MongoChecksRepository implements IChecksRepository { status: boolean | undefined ) => { // Match - const matchStage: Record = { + const matchStage: Record = { "metadata.monitorId": new mongoose.Types.ObjectId(monitorId), ...(typeof status !== "undefined" && { status }), ...(getDateForRange(dateRange) && { @@ -279,7 +280,7 @@ class MongoChecksRepository implements IChecksRepository { }; findByTeamId = async (sortOrder: string, dateRange: string, filter: string, page: number, rowsPerPage: number, teamId: string) => { - const matchStage: Record = { + const matchStage: Record = { "metadata.teamId": new mongoose.Types.ObjectId(teamId), ...(getDateForRange(dateRange) && { createdAt: { @@ -348,7 +349,7 @@ class MongoChecksRepository implements IChecksRepository { ); const mapped = results.reduce((acc, { monitorId, docs }) => { - acc[monitorId] = docs.map((doc: any) => this.toEntity(doc as CheckDocument)); + acc[monitorId] = docs.map((doc: CheckDocument) => this.toEntity(doc)); return acc; }, {}); return mapped; @@ -402,11 +403,31 @@ class MongoChecksRepository implements IChecksRepository { return result.deletedCount ?? 0; }; - deleteOlderThan = async (date: Date): Promise => { - const result = await CheckModel.deleteMany({ createdAt: { $lt: date } }); - return result.deletedCount ?? 0; - }; + deleteOlderThan = async (cutoffDate: Date, batchDays: number = 30): Promise => { + // Find the oldest check that is older than the cutoff + const oldest = await CheckModel.findOne({ createdAt: { $lt: cutoffDate } }) + .sort({ createdAt: 1 }) + .select({ createdAt: 1 }); + if (!oldest?.createdAt) return 0; + let totalDeleted = 0; + let batchStart = new Date(oldest.createdAt); + const batchMs = batchDays * 24 * 60 * 60 * 1000; + + // Delete in 30 day chunks until we reach the cutoff + while (batchStart < cutoffDate) { + // Advance startTime by batchMs to get to the end of last batch + const nextStart = batchStart.getTime() + batchMs; + const batchEnd = new Date(Math.min(nextStart, cutoffDate.getTime())); + const result = await CheckModel.deleteMany({ + createdAt: { $gte: batchStart, $lt: batchEnd }, + }); + totalDeleted += result.deletedCount ?? 0; + batchStart = batchEnd; + } + + return totalDeleted; + }; private findUptimeDateRangeChecks = async ( monitorType: Exclude, monitorObjectId: mongoose.Types.ObjectId, diff --git a/server/src/routes/checkRoute.ts b/server/src/routes/checkRoute.ts index 4df286846..71a0cd5f3 100755 --- a/server/src/routes/checkRoute.ts +++ b/server/src/routes/checkRoute.ts @@ -15,7 +15,6 @@ class CheckRoutes { initRoutes() { this.router.get("/team/summary", this.checkController.getChecksSummaryByTeamId); this.router.get("/team", this.checkController.getChecksByTeam); - this.router.put("/team/ttl", isAllowed(["admin", "superadmin"]), this.checkController.updateChecksTTL); this.router.delete("/team", isAllowed(["admin", "superadmin"]), this.checkController.deleteChecksByTeamId); this.router.get("/:monitorId", this.checkController.getChecksByMonitor); this.router.delete("/:monitorId", this.checkController.deleteChecks); diff --git a/server/src/service/business/checkService.ts b/server/src/service/business/checkService.ts index a95939947..719124b10 100644 --- a/server/src/service/business/checkService.ts +++ b/server/src/service/business/checkService.ts @@ -19,13 +19,28 @@ const SERVICE_NAME = "checkService"; export interface ICheckService { createChecks(checks: Check[]): Promise; buildCheck(statusResponse: MonitorStatusResponse): Check | undefined; - getChecksByMonitor(params: { monitorId: string; query: Record; teamId: string }): Promise; - getChecksByTeam(params: { teamId: string; query: Record }): Promise; + getChecksByMonitor(params: { + monitorId: string; + teamId: string; + sortOrder: string; + dateRange: string; + page: number; + rowsPerPage: number; + filter?: string; + status?: boolean; + }): Promise; + getChecksByTeam(params: { + teamId: string; + sortOrder: string; + dateRange: string; + page: number; + rowsPerPage: number; + filter?: string; + }): Promise; getChecksSummaryByTeamId(params: { teamId: string; dateRange: string }): Promise; deleteChecks(params: { monitorId: string; teamId: string }): Promise; deleteChecksByTeamId(params: { teamId: string }): Promise; deleteOlderThan(date: Date): Promise; - updateChecksTTL(params: { teamId: string; ttl: string }): Promise; } export class CheckService implements ICheckService { @@ -124,7 +139,25 @@ export class CheckService implements ICheckService { return check; }; - getChecksByMonitor = async ({ monitorId, query, teamId }: { monitorId: string; query: Record; teamId: string }) => { + getChecksByMonitor = async ({ + monitorId, + teamId, + sortOrder, + dateRange, + filter, + page, + rowsPerPage, + status, + }: { + monitorId: string; + teamId: string; + sortOrder: string; + dateRange: string; + page: number; + rowsPerPage: number; + status?: boolean; + filter?: string; + }) => { if (!monitorId) { throw new AppError({ message: "No monitor ID in request", service: SERVICE_NAME, method: "getChecksByMonitor", status: 400 }); } @@ -135,69 +168,49 @@ export class CheckService implements ICheckService { // For verification, throws an error if monitor doesn't belong to team await this.monitorsRepository.findById(monitorId, teamId); - const { sortOrder, dateRange, filter, page, rowsPerPage, status } = query; - if (!sortOrder || !dateRange || !page || !rowsPerPage) { - throw new AppError({ - message: `Missing required query parameters sortOrder: ${sortOrder}, dateRange: ${dateRange}, filter: ${filter}, page: ${page}, rowsPerPage: ${rowsPerPage}`, - service: SERVICE_NAME, - method: "getChecksByMonitor", - status: 400, - }); - } const parsedStatus = typeof status === "undefined" ? status : ParseBoolean(status); - const parsedPage = page ? parseInt(page) : 0; - const parsedRowsPerPage = rowsPerPage ? parseInt(rowsPerPage) : 5; + const parsedPage = page ? page : 0; + const parsedRowsPerPage = rowsPerPage ? rowsPerPage : 5; const result = await this.checksRepository.findByMonitorId(monitorId, sortOrder, dateRange, filter, parsedPage, parsedRowsPerPage, parsedStatus); return result; }; - getChecksByTeam = async ({ teamId, query }: { teamId: string; query: Record }) => { - const { sortOrder, dateRange, filter, page, rowsPerPage } = query; - if (!sortOrder || !dateRange || !page || !rowsPerPage) { - throw new AppError({ message: "Missing required query parameters", service: SERVICE_NAME, method: "getChecksByTeam", status: 400 }); - } - - if (!teamId) { - throw new AppError({ message: "No team ID in request", service: SERVICE_NAME, method: "getChecksByTeam", status: 400 }); - } - - const parsedPage = page ? parseInt(page) : 0; - const parsedRowsPerPage = rowsPerPage ? parseInt(rowsPerPage) : 5; + getChecksByTeam = async ({ + teamId, + sortOrder, + dateRange, + filter, + page, + rowsPerPage, + }: { + teamId: string; + sortOrder: string; + dateRange: string; + page: number; + rowsPerPage: number; + filter?: string; + }) => { + const parsedPage = page ? page : 0; + const parsedRowsPerPage = rowsPerPage ? rowsPerPage : 5; const checkData = await this.checksRepository.findByTeamId(sortOrder, dateRange, filter, parsedPage, parsedRowsPerPage, teamId); return checkData; }; getChecksSummaryByTeamId = async ({ teamId, dateRange }: { teamId: string; dateRange: string }) => { - if (!teamId) { - throw new AppError({ message: "No team ID in request", service: SERVICE_NAME, method: "getChecksSummaryByTeamId", status: 400 }); - } const summary = await this.checksRepository.findSummaryByTeamId(teamId, dateRange); return summary; }; deleteChecks = async ({ monitorId, teamId }: { monitorId: string; teamId: string }) => { - if (!monitorId) { - throw new AppError({ message: "No monitor ID in request", service: SERVICE_NAME, method: "deleteChecks", status: 400 }); - } - - if (!teamId) { - throw new AppError({ message: "No team ID in request", service: SERVICE_NAME, method: "deleteChecks", status: 400 }); - } - - // For verification, throws an error if monitor doesn't belong to team await this.monitorsRepository.findById(monitorId, teamId); const deletedCount = await this.checksRepository.deleteByMonitorId(monitorId); return deletedCount; }; deleteChecksByTeamId = async ({ teamId }: { teamId: string }) => { - if (!teamId) { - throw new AppError({ message: "No team ID in request", service: SERVICE_NAME, method: "deleteChecksByTeamId", status: 400 }); - } - const deletedCount = await this.checksRepository.deleteByTeamId(teamId); return deletedCount; }; @@ -210,8 +223,4 @@ export class CheckService implements ICheckService { const deletedCount = await this.checksRepository.deleteOlderThan(date); return deletedCount; }; - - updateChecksTTL = async ({ teamId, ttl }: { teamId: string; ttl: string }) => { - throw new AppError({ message: "Not implemented", service: SERVICE_NAME, method: "updateChecksTTL", status: 500, details: { teamId, ttl } }); - }; } diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index 382318945..b6908127b 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -11,7 +11,7 @@ import { IncidentService, type IGeoChecksService, } from "@/service/index.js"; -import type { MaintenanceWindow, StatusChangeResult } from "@/types/index.js"; +import { CHECK_TTL_SENTINEL, type MaintenanceWindow, type StatusChangeResult } from "@/types/index.js"; import { IMaintenanceWindowsRepository, IMonitorsRepository, @@ -391,7 +391,7 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { const checkTTL = settings.checkTTL; // Check TTL is in DAYS, not MS - if (checkTTL === 366) { + if (checkTTL === CHECK_TTL_SENTINEL) { this.logger.info({ message: `Check TTL is set to unlimited, skipping cleanup`, service: SERVICE_NAME, diff --git a/server/src/types/check.ts b/server/src/types/check.ts index 810454fdd..066b898ab 100644 --- a/server/src/types/check.ts +++ b/server/src/types/check.ts @@ -1,5 +1,8 @@ import type { MonitorType } from "@/types/index.js"; import type { Response } from "got"; + +export const CHECK_TTL_SENTINEL = 366; + export type GotTimings = Response["timings"]; export interface CheckMetadata { diff --git a/server/src/validation/checkValidation.ts b/server/src/validation/checkValidation.ts index d4f75f3ac..2a768f068 100644 --- a/server/src/validation/checkValidation.ts +++ b/server/src/validation/checkValidation.ts @@ -12,32 +12,34 @@ export const getChecksParamValidation = z.object({ export const getChecksQueryValidation = z.object({ type: z.enum(["http", "ping", "pagespeed", "hardware", "docker", "port", "game", "grpc", "websocket"]).optional(), - sortOrder: z.enum(["asc", "desc"]).optional(), + sortOrder: z.enum(["asc", "desc"]), limit: z.coerce.number().optional(), - dateRange: z.enum(["recent", "hour", "day", "week", "month", "all"]).optional(), + dateRange: z.enum(["recent", "hour", "day", "week", "month", "all"]), filter: z.enum(["all", "up", "down", "resolve"]).optional(), ack: booleanCoercion.optional(), - page: z.coerce.number().optional(), - rowsPerPage: z.coerce.number().optional(), + page: z.coerce.number(), + rowsPerPage: z.coerce.number(), status: booleanCoercion.optional(), continent: z.union([z.enum(GeoContinents), z.array(z.enum(GeoContinents))]).optional(), }); export const getTeamChecksQueryValidation = z.object({ - sortOrder: z.enum(["asc", "desc"]).optional(), + sortOrder: z.enum(["asc", "desc"]), limit: z.coerce.number().optional(), - dateRange: z.enum(["recent", "hour", "day", "week", "month", "all"]).optional(), + dateRange: z.enum(["recent", "hour", "day", "week", "month", "all"]), filter: z.enum(["all", "up", "down", "resolve"]).optional(), ack: booleanCoercion.optional(), - page: z.coerce.number().optional(), - rowsPerPage: z.coerce.number().optional(), + page: z.coerce.number(), + rowsPerPage: z.coerce.number(), }); export const deleteChecksParamValidation = z.object({ monitorId: z.string().min(1, "Monitor ID is required"), }); -export const deleteChecksByTeamIdParamValidation = z.object({}); +export const getChecksSummaryByTeamIdQueryValidation = z.object({ + dateRange: z.enum(["recent", "hour", "day", "week", "month", "all"]).optional(), +}); export const updateChecksTTLBodyValidation = z.object({ ttl: z.number().min(1, "TTL is required"), diff --git a/server/src/validation/settingsValidation.ts b/server/src/validation/settingsValidation.ts index 0aeec943f..e7f0dc9b4 100644 --- a/server/src/validation/settingsValidation.ts +++ b/server/src/validation/settingsValidation.ts @@ -1,3 +1,4 @@ +import { CHECK_TTL_SENTINEL } from "@/types/check.js"; import { z } from "zod"; //**************************************** @@ -6,9 +7,8 @@ import { z } from "zod"; export const updateAppSettingsBodyValidation = z .object({ - checkTTL: z.union([z.number().int().min(1).max(366), z.literal("")]).optional(), + checkTTL: z.union([z.number().int().min(1).max(CHECK_TTL_SENTINEL), z.literal("")]).optional(), systemEmailPort: z.union([z.number(), z.literal("")]).optional(), - pagespeedApiKey: z.union([z.string(), z.literal("")]).optional(), language: z.union([z.string(), z.literal("")]).optional(), timezone: z.union([z.string(), z.literal("")]).optional(),