slim check snapshot model/types

This commit is contained in:
Alex Holliday committed 2026-08-04 11:27:32 -07:00
1 parent 3646af9c04
commit dfcb3b4b21
12 files changed
+295 -243

No files matched your search

+38 -1
View File
@@ -250,7 +250,44 @@ export interface ChecksSummary {
downChecks: number;
}
export type CheckSnapshot = Omit<Check, "metadata" | "__v" | "updatedAt">;
export type SnapshotCpuInfo = Pick<
CheckCpuInfo,
| "physical_core"
| "logical_core"
| "frequency"
| "current_frequency"
| "temperature"
| "usage_percent"
>;
export type SnapshotMemoryInfo = Pick<
CheckMemoryInfo,
"total_bytes" | "used_bytes" | "usage_percent"
>;
export type SnapshotDiskInfo = Pick<
CheckDiskInfo,
"device" | "total_bytes" | "used_bytes" | "usage_percent"
>;
export type SnapshotHostInfo = Pick<CheckHostInfo, "os" | "platform" | "pretty_name">;
export type CheckSnapshot = Pick<
Check,
| "id"
| "status"
| "responseTime"
| "statusCode"
| "message"
| "createdAt"
| "accessibility"
| "bestPractices"
| "seo"
| "performance"
| "audits"
> & {
cpu?: SnapshotCpuInfo;
memory?: SnapshotMemoryInfo;
disk?: SnapshotDiskInfo[];
host?: SnapshotHostInfo;
};
export interface HasResponseTime {
responseTime: number;
}
@@ -0,0 +1,128 @@
import { MonitorModel } from "../../domain/monitors/monitor.model.js";
import type { ILogger } from "@/utils/logger.js";
const KEEP_KEYS = [
"id",
"status",
"responseTime",
"statusCode",
"message",
"createdAt",
"cpu",
"memory",
"disk",
"host",
"accessibility",
"bestPractices",
"seo",
"performance",
"audits",
];
/**
* Slim stored recentChecks snapshots down to the fields the slim CheckSnapshot type keeps.
* The schema change only affects new writes; paused monitors never rotate their snapshots out.
* Idempotent: re-running on already-slim snapshots is a no-op re-pick.
*/
export async function slimRecentChecks(logger: ILogger): Promise<void> {
const SERVICE_NAME = "Migration:SlimRecentChecks";
try {
logger.info({ service: SERVICE_NAME, message: "Slimming recentChecks snapshots" });
const result = await MonitorModel.updateMany({ "recentChecks.0": { $exists: true } }, [
{
$set: {
recentChecks: {
$map: {
input: "$recentChecks",
as: "snap",
in: {
$mergeObjects: [
// 1) keep whitelisted top-level keys only (drops timings/errors/capture/net)
{
$arrayToObject: {
$filter: {
input: { $objectToArray: "$$snap" },
as: "kv",
cond: { $in: ["$$kv.k", KEEP_KEYS] },
},
},
},
// 2) where a nested group exists, overwrite it with its slim pick
{
$cond: [
{ $eq: [{ $type: "$$snap.cpu" }, "object"] },
{
cpu: {
physical_core: "$$snap.cpu.physical_core",
logical_core: "$$snap.cpu.logical_core",
frequency: "$$snap.cpu.frequency",
current_frequency: "$$snap.cpu.current_frequency",
temperature: "$$snap.cpu.temperature",
usage_percent: "$$snap.cpu.usage_percent",
},
},
{},
],
},
{
$cond: [
{ $eq: [{ $type: "$$snap.memory" }, "object"] },
{
memory: {
total_bytes: "$$snap.memory.total_bytes",
used_bytes: "$$snap.memory.used_bytes",
usage_percent: "$$snap.memory.usage_percent",
},
},
{},
],
},
{
$cond: [
{ $eq: [{ $type: "$$snap.disk" }, "array"] },
{
disk: {
$map: {
input: "$$snap.disk",
as: "d",
in: {
device: "$$d.device",
total_bytes: "$$d.total_bytes",
used_bytes: "$$d.used_bytes",
usage_percent: "$$d.usage_percent",
},
},
},
},
{},
],
},
{
$cond: [
{ $eq: [{ $type: "$$snap.host" }, "object"] },
{
host: {
os: "$$snap.host.os",
platform: "$$snap.host.platform",
pretty_name: "$$snap.host.pretty_name",
},
},
{},
],
},
],
},
},
},
},
},
]);
logger.info({ service: SERVICE_NAME, message: `Slimmed recentChecks on ${result.modifiedCount} monitors` });
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.error({ service: SERVICE_NAME, message: `Error slimming recentChecks: ${errorMessage}` });
throw error;
}
}
+2
View File
@@ -8,6 +8,7 @@ import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplic
import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js";
import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js";
import { recomputeResponseTimeFromTimings } from "./0009_recomputeResponseTimeFromTimings.js";
import { slimRecentChecks } from "./0010_slimRecentChecks.js";
import type { ILogger } from "@/utils/logger.js";
type MigrationEntry = {
@@ -25,6 +26,7 @@ const migrations: MigrationEntry[] = [
{ name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray },
{ name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt },
{ name: "0009_recomputeResponseTimeFromTimings", execute: recomputeResponseTimeFromTimings },
{ name: "0010_slimRecentChecks", execute: slimRecentChecks },
];
const runMigrations = async (logger: ILogger) => {
@@ -7,7 +7,6 @@ import type {
UptimeChecksResult,
} from "@/domain/checks/check.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import type { LatestChecksMap } from "@/domain/checks/check.repository.mongo.js";
import { CheckFilter, DateRange } from "@/types/query.js";
export interface IChecksRepository {
@@ -34,7 +33,6 @@ export interface IChecksRepository {
teamId: string,
filter?: CheckFilter
): Promise<ChecksQueryResult>;
findLatestByMonitorIds(monitorIds: string[], options?: { limitPerMonitor?: number }): Promise<LatestChecksMap>;
findByDateRangeAndMonitorId(
monitorId: string,
dateRange: DateRange,
@@ -27,8 +27,6 @@ import { NETWORK_ERROR } from "@/types/network.js";
const SERVICE_NAME = "StatusService";
export type LatestChecksMap = Record<string, Check[]>;
class MongoChecksRepository implements IChecksRepository {
static SERVICE_NAME = SERVICE_NAME;
@@ -288,32 +286,6 @@ class MongoChecksRepository implements IChecksRepository {
return { checksCount, checks: this.mapDocuments(checks) };
};
findLatestByMonitorIds = async (monitorIds: string[], options?: { limitPerMonitor?: number }): Promise<LatestChecksMap> => {
if (monitorIds.length === 0) {
return {};
}
const limitPerMonitor = options?.limitPerMonitor ?? 25;
const dateFilter = new Date(Date.now() - 24 * 60 * 60 * 1000);
const results = await Promise.all(
monitorIds.map(async (monitorId) => {
const docs = await CheckModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
createdAt: { $gte: dateFilter },
})
.sort({ createdAt: -1 })
.limit(limitPerMonitor)
.lean();
return { monitorId, docs };
})
);
const mapped = results.reduce<LatestChecksMap>((acc, { monitorId, docs }) => {
acc[monitorId] = docs.map((doc: CheckDocument) => this.toEntity(doc));
return acc;
}, {});
return mapped;
};
findByDateRangeAndMonitorId = async (monitorId: string, dateRange: DateRange, options?: { type?: MonitorType }) => {
const monitorObjectId = new mongoose.Types.ObjectId(monitorId);
const start = getDateForRange(dateRange);
@@ -0,0 +1,85 @@
import type {
Check,
CheckAudits,
CheckCpuInfo,
CheckDiskInfo,
CheckHostInfo,
CheckMemoryInfo,
CheckSnapshot,
SnapshotCpuInfo,
SnapshotDiskInfo,
SnapshotHostInfo,
SnapshotMemoryInfo,
} from "@/domain/checks/check.type.js";
import { toDateString } from "@/utils/mongoMappers.js";
export type CheckSnapshotSource = Pick<
Check,
"id" | "status" | "responseTime" | "statusCode" | "message" | "accessibility" | "bestPractices" | "seo" | "performance"
> & {
createdAt: string | Date;
cpu?: CheckCpuInfo;
memory?: CheckMemoryInfo;
disk?: CheckDiskInfo[];
host?: CheckHostInfo;
audits?: CheckAudits;
};
const mapCpu = (cpu?: CheckCpuInfo): SnapshotCpuInfo | undefined =>
cpu && {
physical_core: cpu.physical_core,
logical_core: cpu.logical_core,
frequency: cpu.frequency,
current_frequency: cpu.current_frequency,
temperature: cpu.temperature,
usage_percent: cpu.usage_percent,
};
const mapMemory = (memory?: CheckMemoryInfo): SnapshotMemoryInfo | undefined =>
memory && {
total_bytes: memory.total_bytes,
used_bytes: memory.used_bytes,
usage_percent: memory.usage_percent,
};
const mapDisk = (disk?: CheckDiskInfo[]): SnapshotDiskInfo[] | undefined =>
disk?.map((entry) => ({
device: entry.device,
total_bytes: entry.total_bytes,
used_bytes: entry.used_bytes,
usage_percent: entry.usage_percent,
}));
const mapHost = (host?: CheckHostInfo): SnapshotHostInfo | undefined =>
host && {
os: host.os,
platform: host.platform,
pretty_name: host.pretty_name,
};
const mapAudits = (audits?: CheckAudits): CheckAudits | undefined =>
audits && {
cls: audits.cls,
si: audits.si,
fcp: audits.fcp,
lcp: audits.lcp,
tbt: audits.tbt,
};
export const toCheckSnapshot = (source: CheckSnapshotSource): CheckSnapshot => ({
id: source.id,
status: source.status,
responseTime: source.responseTime,
statusCode: source.statusCode,
message: source.message,
createdAt: toDateString(source.createdAt),
cpu: mapCpu(source.cpu),
memory: mapMemory(source.memory),
disk: mapDisk(source.disk),
host: mapHost(source.host),
accessibility: source.accessibility,
bestPractices: source.bestPractices,
seo: source.seo,
performance: source.performance,
audits: mapAudits(source.audits),
});
+29 -1
View File
@@ -168,8 +168,36 @@ export interface HasResponseTime {
responseTime: number;
}
export type CheckSnapshot = Omit<Check, "metadata" | "updatedAt">;
export type SnapshotCpuInfo = Pick<
CheckCpuInfo,
"physical_core" | "logical_core" | "frequency" | "current_frequency" | "temperature" | "usage_percent"
>;
export type SnapshotMemoryInfo = Pick<CheckMemoryInfo, "total_bytes" | "used_bytes" | "usage_percent">;
export type SnapshotDiskInfo = Pick<CheckDiskInfo, "device" | "total_bytes" | "used_bytes" | "usage_percent">;
export type SnapshotHostInfo = Pick<CheckHostInfo, "os" | "platform" | "pretty_name">;
export type CheckSnapshot = Pick<
Check,
// uptime charts and down check tooltips
| "id"
| "status"
| "responseTime"
| "statusCode"
| "message"
| "createdAt"
// pagespeed
| "accessibility"
| "bestPractices"
| "seo"
| "performance"
| "audits"
> & {
// hardware monitors only
cpu?: SnapshotCpuInfo;
memory?: SnapshotMemoryInfo;
disk?: SnapshotDiskInfo[];
host?: SnapshotHostInfo;
};
export interface HardwareDiskStats {
name: string;
readSpeed: number;
+8 -91
View File
@@ -3,15 +3,11 @@ import type { Monitor, MonitorMatchMethod, CheckSnapshot } from "@/domain/monito
import { DnsRecordTypes, MonitorTypes, MonitorStatuses, PageSpeedStrategies, HttpMethods } from "@/domain/monitors/monitor.type.js";
import type {
CheckAudits,
CheckCaptureInfo,
CheckCpuInfo,
CheckDiskInfo,
CheckErrorInfo,
CheckHostInfo,
CheckMemoryInfo,
CheckNetworkInterfaceInfo,
GotTimings,
ILighthouseAudit,
SnapshotCpuInfo,
SnapshotDiskInfo,
SnapshotHostInfo,
SnapshotMemoryInfo,
} from "@/domain/checks/check.type.js";
type CheckSnapshotDocument = Omit<CheckSnapshot, "createdAt"> & { createdAt: Date };
@@ -37,121 +33,46 @@ interface MonitorDocument extends MonitorDocumentBase {
lastEvaluatedAt: number; // epoch ms
}
const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
{
wait: { type: Number },
dns: { type: Number },
tcp: { type: Number },
tls: { type: Number },
request: { type: Number },
firstByte: { type: Number },
download: { type: Number },
total: { type: Number },
},
{ _id: false }
);
const snapshotTimingsSchema = new Schema<GotTimings>(
{
start: { type: Number },
socket: { type: Number },
lookup: { type: Number },
connect: { type: Number },
secureConnect: { type: Number },
upload: { type: Number },
response: { type: Number },
end: { type: Number },
phases: { type: snapshotTimingPhasesSchema },
},
{ _id: false }
);
const snapshotCpuSchema = new Schema<CheckCpuInfo>(
const snapshotCpuSchema = new Schema<SnapshotCpuInfo>(
{
physical_core: { type: Number },
logical_core: { type: Number },
frequency: { type: Number },
current_frequency: { type: Number },
temperature: { type: [Number] },
free_percent: { type: Number },
usage_percent: { type: Number },
},
{ _id: false }
);
const snapshotMemorySchema = new Schema<CheckMemoryInfo>(
const snapshotMemorySchema = new Schema<SnapshotMemoryInfo>(
{
total_bytes: { type: Number },
available_bytes: { type: Number },
used_bytes: { type: Number },
usage_percent: { type: Number },
},
{ _id: false }
);
const snapshotDiskSchema = new Schema<CheckDiskInfo>(
const snapshotDiskSchema = new Schema<SnapshotDiskInfo>(
{
device: { type: String },
mountpoint: { type: String },
total_bytes: { type: Number },
free_bytes: { type: Number },
used_bytes: { type: Number },
usage_percent: { type: Number },
total_inodes: { type: Number },
free_inodes: { type: Number },
used_inodes: { type: Number },
inodes_usage_percent: { type: Number },
read_bytes: { type: Number },
write_bytes: { type: Number },
read_time: { type: Number },
write_time: { type: Number },
},
{ _id: false }
);
const snapshotHostSchema = new Schema<CheckHostInfo>(
const snapshotHostSchema = new Schema<SnapshotHostInfo>(
{
os: { type: String },
platform: { type: String },
kernel_version: { type: String },
pretty_name: { type: String },
},
{ _id: false }
);
const snapshotErrorSchema = new Schema<CheckErrorInfo>(
{
metric: { type: [String] },
err: { type: String },
},
{ _id: false }
);
const snapshotCaptureSchema = new Schema<CheckCaptureInfo>(
{
version: { type: String },
mode: { type: String },
},
{ _id: false }
);
const snapshotNetworkInterfaceSchema = new Schema<CheckNetworkInterfaceInfo>(
{
name: { type: String },
bytes_sent: { type: Number },
bytes_recv: { type: Number },
packets_sent: { type: Number },
packets_recv: { type: Number },
err_in: { type: Number },
err_out: { type: Number },
drop_in: { type: Number },
drop_out: { type: Number },
fifo_in: { type: Number },
fifo_out: { type: Number },
},
{ _id: false }
);
const snapshotLighthouseAuditSchema = new Schema<ILighthouseAudit>(
{
id: { type: String },
@@ -180,16 +101,12 @@ const checkSnapshotSchema = new Schema<CheckSnapshotDocument>(
id: { type: String, required: true },
status: { type: Boolean, required: true },
responseTime: { type: Number },
timings: { type: snapshotTimingsSchema },
statusCode: { type: Number },
message: { type: String },
cpu: { type: snapshotCpuSchema },
memory: { type: snapshotMemorySchema },
disk: { type: [snapshotDiskSchema] },
host: { type: snapshotHostSchema },
errors: { type: [snapshotErrorSchema] },
capture: { type: snapshotCaptureSchema },
net: { type: [snapshotNetworkInterfaceSchema] },
accessibility: { type: Number },
bestPractices: { type: Number },
seo: { type: Number },
@@ -30,7 +30,6 @@ export interface IMonitorsRepository {
findByTeamId(teamId: string, config: TeamQueryConfig): Promise<Monitor[]>;
findByTeamIdWithStats(teamId: string, config: TeamQueryConfig): Promise<Monitor[]>;
findByIds(monitorIds: string[]): Promise<Monitor[]>;
findByIdsWithChecks(monitorIds: string[], checksCount?: number): Promise<Monitor[]>;
// update
updateById(monitorId: string, teamId: string, updates: Partial<Monitor>): Promise<Monitor>;
@@ -7,6 +7,7 @@ import { MongoBulkWriteError } from "mongodb";
import { AppError } from "@/utils/AppError.js";
import { IMonitorsRepository, TeamQueryConfig, SummaryConfig } from "@/domain/monitors/monitor.repository.interface.js";
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js";
class MongoMonitorsRepository implements IMonitorsRepository {
create = async (monitor: Monitor, teamId: string, userId: string) => {
const monitorModel = new MonitorModel({ ...monitor, teamId, userId });
@@ -131,70 +132,6 @@ class MongoMonitorsRepository implements IMonitorsRepository {
return documents.map((doc) => this.toEntity(doc));
};
findByIdsWithChecks = async (monitorIds: string[], checksCount: number = 25): Promise<Monitor[]> => {
if (!monitorIds.length) {
return [];
}
const objectIds = monitorIds.map((id) => new mongoose.Types.ObjectId(id));
const pipeline: PipelineStage[] = [
{ $match: { _id: { $in: objectIds } } },
{
$lookup: {
from: "checks",
let: { monitorId: "$_id" },
pipeline: [{ $match: { $expr: { $eq: ["$metadata.monitorId", "$$monitorId"] } } }, { $sort: { createdAt: -1 } }, { $limit: checksCount }],
as: "checks",
},
},
{
$lookup: {
from: "maintenancewindows",
let: { monitorId: "$_id" },
pipeline: [{ $match: { $expr: { $in: ["$$monitorId", { $ifNull: ["$monitorIds", []] }] } } }],
as: "maintenanceWindows",
},
},
{
$lookup: {
from: "monitorstats",
localField: "_id",
foreignField: "monitorId",
as: "stats",
},
},
{
$addFields: {
isMaintenance: {
$reduce: {
input: "$maintenanceWindows",
initialValue: false,
in: {
$or: [
"$$value",
{
$and: [{ $eq: ["$$this.active", true] }, { $lte: ["$$this.start", "$$NOW"] }, { $gte: ["$$this.end", "$$NOW"] }],
},
],
},
},
},
uptimePercentage: { $arrayElemAt: ["$stats.uptimePercentage", 0] },
},
},
{
$project: {
maintenanceWindows: 0,
stats: 0,
},
},
];
const documents = await MonitorModel.aggregate(pipeline);
return documents.map((doc) => this.toEntity(doc));
};
findMonitorCountByTeamIdAndType = async (teamId: string, config: TeamQueryConfig): Promise<number> => {
const query = this.queryBuilder(config, teamId);
const count = await MonitorModel.countDocuments(query);
@@ -473,7 +410,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
grpcServiceName: doc.grpcServiceName ?? undefined,
strategy: doc.strategy ?? undefined,
group: doc.group ?? null,
recentChecks: (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => this.toCheckSnapshot(check)),
recentChecks: (doc.recentChecks ?? []).map((check: CheckSnapshotDocument) => toCheckSnapshot(check)),
geoCheckEnabled: doc.geoCheckEnabled ?? false,
geoCheckLocations: doc.geoCheckLocations ?? [],
geoCheckInterval: doc.geoCheckInterval ?? 300000,
@@ -485,30 +422,6 @@ class MongoMonitorsRepository implements IMonitorsRepository {
};
};
private toCheckSnapshot = (doc: CheckSnapshotDocument): CheckSnapshot => {
return {
id: doc.id,
status: doc.status,
responseTime: doc.responseTime,
timings: doc.timings,
statusCode: doc.statusCode,
message: doc.message,
cpu: doc.cpu,
memory: doc.memory,
disk: doc.disk,
host: doc.host,
errors: doc.errors,
capture: doc.capture,
net: doc.net,
accessibility: doc.accessibility,
bestPractices: doc.bestPractices,
seo: doc.seo,
performance: doc.performance,
audits: doc.audits,
createdAt: toDateString(doc.createdAt),
};
};
deleteByTeamIdsNotIn = async (teamIds: string[]): Promise<number> => {
const objectIds = teamIds.map((id) => new mongoose.Types.ObjectId(id));
const result = await MonitorModel.deleteMany({ teamId: { $nin: objectIds } });
+3 -26
View File
@@ -1,6 +1,6 @@
import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
import type { Check, CheckDiskInfo, CheckSnapshot } from "@/domain/checks/check.type.js";
import type { Check, CheckDiskInfo } from "@/domain/checks/check.type.js";
import { MonitorStatuses, type Monitor, type MonitorStatus } from "@/domain/monitors/monitor.type.js";
import type {
DockerStatusPayload,
@@ -18,6 +18,7 @@ import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import type { HardwareStatusMetrics } from "@/types/network.js";
import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.type.js";
import { toCheckSnapshot } from "@/domain/checks/check.snapshot.js";
const SERVICE_NAME = "StatusService";
const HARDWARE_ALERT_COUNTER_START = 5;
@@ -87,30 +88,6 @@ export class StatusService implements IStatusService {
}
};
private toCheckSnapshot = (check: Check): CheckSnapshot => {
return {
id: check.id,
status: check.status,
responseTime: check.responseTime,
timings: check.timings,
statusCode: check.statusCode,
message: check.message,
cpu: check.cpu,
memory: check.memory,
disk: check.disk,
host: check.host,
errors: check.errors,
capture: check.capture,
net: check.net,
accessibility: check.accessibility,
bestPractices: check.bestPractices,
seo: check.seo,
performance: check.performance,
audits: check.audits,
createdAt: check.createdAt,
};
};
private computeReachability = (
currentStatus: MonitorStatus,
window: Array<boolean>,
@@ -207,7 +184,7 @@ export class StatusService implements IStatusService {
await this.tryUpdateRunningStats(monitor, statusResponse);
const prevStatus = monitor.status;
const checkSnapshot = this.toCheckSnapshot(check);
const checkSnapshot = toCheckSnapshot(check);
// Project the window as it will look after updating DB
// This is done because we need the updated status window to compute new status, but we don't
@@ -45,10 +45,6 @@ export class InMemoryMonitorsRepository implements IMonitorsRepository {
return monitor ? { ...monitor } : null;
}
async findByIdsWithChecks(monitorIds: string[], _checksCount?: number): Promise<Monitor[]> {
return this.findByIds(monitorIds);
}
async updateById(monitorId: string, teamId: string, updates: Partial<Monitor>): Promise<Monitor> {
const index = this.monitors.findIndex((m) => m.id === monitorId && m.teamId === teamId);
if (index === -1) {