aggregates

This commit is contained in:
Alex Holliday committed 2025-12-17 21:11:51 +00:00
1 parent 95e415ed13
commit c8dc5b93cb
14 files changed
+302 -99

No files matched your search

+43 -13
View File
@@ -64,8 +64,7 @@ export interface IMonitorService {
getEmbedChecks: (
teamId: string,
monitorId: string,
range: string,
status?: string
range: string
) => Promise<MonitorWithChecksResponse>;
togglePause: (
userId: string,
@@ -504,11 +503,11 @@ class MonitorService implements IMonitorService {
return {};
};
// status parameter deprecated and ignored; retained for backward compatibility
getEmbedChecks = async (
teamId: string,
monitorId: string,
range: string,
status: string | undefined
range: string
): Promise<MonitorWithChecksResponse> => {
const monitor = await Monitor.findOne({ _id: monitorId, teamId });
if (!monitor) {
@@ -519,7 +518,9 @@ class MonitorService implements IMonitorService {
// Route for aggregated stats (24h, 7d, 30d)
if (range === "24h" || range === "7d" || range === "30d") {
const monitorStats = await MonitorStats.findOne({ monitorId: monitor._id });
const monitorStats = await MonitorStats.findOne({
monitorId: monitor._id,
});
if (!monitorStats) {
throw new ApiError("Monitor stats not found", 404);
}
@@ -540,6 +541,10 @@ class MonitorService implements IMonitorService {
_id: new Date(d.windowStart).toISOString(),
count: d.count ?? 0,
avgResponseTime: d.avgResponseTime ?? 0,
upChecks: d.upChecks ?? 0,
downChecks: d.downChecks ?? 0,
avgResponseTimeUp: d.avgResponseTimeUp ?? null,
avgResponseTimeDown: d.avgResponseTimeDown ?? null,
};
if (monitor.type === "pagespeed") {
return {
@@ -579,16 +584,11 @@ class MonitorService implements IMonitorService {
const matchStage: {
"metadata.monitorId": mongoose.Types.ObjectId;
createdAt: { $gte: Date };
status?: string;
} = {
"metadata.monitorId": monitor._id,
createdAt: { $gte: startDate },
};
if (status) {
matchStage.status = status;
}
const dateFormat = "%Y-%m-%dT%H:%M:00Z";
let groupClause;
if (monitor.type === "pagespeed") {
@@ -620,11 +620,41 @@ class MonitorService implements IMonitorService {
$match: matchStage,
},
{ $sort: { createdAt: 1 } },
{ $project: projectStage },
{ $group: groupClause },
{ $project: { ...projectStage, status: 1 } },
{
$group: {
...groupClause,
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
sumResponseUp: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
},
sumResponseDown: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
},
},
},
{ $sort: { _id: -1 } },
{
$project: finalProjection,
$project: {
...finalProjection,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
},
},
]);
@@ -35,6 +35,46 @@ class StatsAggregationService implements IStatsAggregationService {
type: { $first: "$metadata.type" },
count: { $sum: 1 },
avgResponseTime: { $avg: "$responseTime" },
upChecks: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] },
},
downChecks: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] },
},
sumResponseUp: {
$sum: {
$cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0],
},
},
sumResponseDown: {
$sum: {
$cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0],
},
},
},
},
{
$project: {
teamId: 1,
type: 1,
count: 1,
avgResponseTime: 1,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
},
},
]);
@@ -44,6 +84,7 @@ class StatsAggregationService implements IStatsAggregationService {
{ $match: { ...matchWindow, "metadata.type": "pagespeed" } },
{
$project: {
status: 1,
responseTime: 1,
createdAt: 1,
"lighthouse.accessibility": 1,
@@ -67,6 +108,14 @@ class StatsAggregationService implements IStatsAggregationService {
type: { $first: "$metadata.type" },
count: { $sum: 1 },
avgResponseTime: { $avg: "$responseTime" },
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
sumResponseUp: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
},
sumResponseDown: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
},
accessibility: { $avg: "$lighthouse.accessibility" },
bestPractices: { $avg: "$lighthouse.bestPractices" },
seo: { $avg: "$lighthouse.seo" },
@@ -78,6 +127,39 @@ class StatsAggregationService implements IStatsAggregationService {
tbt: { $avg: "$lighthouse.audits.tbt.score" },
},
},
{
$project: {
teamId: 1,
type: 1,
count: 1,
avgResponseTime: 1,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
accessibility: 1,
bestPractices: 1,
seo: 1,
performance: 1,
cls: 1,
si: 1,
fcp: 1,
lcp: 1,
tbt: 1,
},
},
]);
// 3) Infrastructure
@@ -85,6 +167,7 @@ class StatsAggregationService implements IStatsAggregationService {
{ $match: { ...matchWindow, "metadata.type": "infrastructure" } },
{
$project: {
status: 1,
responseTime: 1,
createdAt: 1,
"system.cpu.physical_core": 1,
@@ -116,6 +199,14 @@ class StatsAggregationService implements IStatsAggregationService {
type: { $first: "$metadata.type" },
count: { $sum: 1 },
avgResponseTime: { $avg: "$responseTime" },
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
sumResponseUp: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
},
sumResponseDown: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
},
physicalCores: { $last: "$system.cpu.physical_core" },
logicalCores: { $last: "$system.cpu.logical_core" },
frequency: { $avg: "$system.cpu.frequency" },
@@ -142,6 +233,22 @@ class StatsAggregationService implements IStatsAggregationService {
type: 1,
count: 1,
avgResponseTime: 1,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
cpu: {
physical_core: "$physicalCores",
logical_core: "$logicalCores",
@@ -291,6 +398,10 @@ class StatsAggregationService implements IStatsAggregationService {
finalized: false,
count: d.count ?? 0,
avgResponseTime: d.avgResponseTime ?? 0,
upChecks: d.upChecks ?? 0,
downChecks: d.downChecks ?? 0,
avgResponseTimeUp: (d.avgResponseTimeUp ?? null) as any,
avgResponseTimeDown: (d.avgResponseTimeDown ?? null) as any,
accessibility: d.accessibility,
bestPractices: d.bestPractices,
seo: d.seo,
@@ -354,8 +465,12 @@ class StatsAggregationService implements IStatsAggregationService {
type: string;
// totals
count: number;
upChecks: number;
downChecks: number;
// base
rt: Weighted;
rtUp: Weighted;
rtDown: Weighted;
// pagespeed
accessibility: Weighted;
bestPractices: Weighted;
@@ -400,7 +515,11 @@ class StatsAggregationService implements IStatsAggregationService {
teamId: h.teamId as any,
type: h.type,
count: 0,
upChecks: 0,
downChecks: 0,
rt: { sum: 0, w: 0 },
rtUp: { sum: 0, w: 0 },
rtDown: { sum: 0, w: 0 },
accessibility: { sum: 0, w: 0 },
bestPractices: { sum: 0, w: 0 },
seo: { sum: 0, w: 0 },
@@ -428,7 +547,13 @@ class StatsAggregationService implements IStatsAggregationService {
const w = h.count ?? 0;
g.count += w;
const up = (h as any).upChecks ?? 0;
const down = (h as any).downChecks ?? 0;
g.upChecks += up;
g.downChecks += down;
addW(g.rt, h.avgResponseTime, w);
addW(g.rtUp, (h as any).avgResponseTimeUp, up);
addW(g.rtDown, (h as any).avgResponseTimeDown, down);
// pagespeed
addW(g.accessibility, (h as any).accessibility, w);
@@ -601,7 +726,11 @@ class StatsAggregationService implements IStatsAggregationService {
$set: {
finalized: finalize,
count: g.count,
upChecks: g.upChecks,
downChecks: g.downChecks,
avgResponseTime: avg(g.rt) ?? 0,
avgResponseTimeUp: (avg(g.rtUp) ?? null) as any,
avgResponseTimeDown: (avg(g.rtDown) ?? null) as any,
accessibility: avg(g.accessibility),
bestPractices: avg(g.bestPractices),
seo: avg(g.seo),
@@ -144,7 +144,7 @@ class JobGenerator implements IJobGenerator {
generateStatsAggregationJob = () => {
return async () => {
// Get previous hour window in UTC
// Compute hour window in UTC based on flag
const now = new Date();
const utcNow = new Date(
Date.UTC(
@@ -157,18 +157,37 @@ class JobGenerator implements IJobGenerator {
0
)
);
const hourEnd = new Date(
Date.UTC(
now.getUTCFullYear(),
now.getUTCMonth(),
now.getUTCDate(),
now.getUTCHours(),
0,
0,
0
)
);
const hourStart = new Date(hourEnd.getTime() - 60 * 60 * 1000);
let hourStart: Date;
let hourEnd: Date;
if (config.STATS_USE_CURRENT_HOUR) {
// Current hour [hh:00, hh+1:00)
hourStart = new Date(
Date.UTC(
now.getUTCFullYear(),
now.getUTCMonth(),
now.getUTCDate(),
now.getUTCHours(),
0,
0,
0
)
);
hourEnd = new Date(hourStart.getTime() + 60 * 60 * 1000);
} else {
// Previous hour [hh-1:00, hh:00)
hourEnd = new Date(
Date.UTC(
now.getUTCFullYear(),
now.getUTCMonth(),
now.getUTCDate(),
now.getUTCHours(),
0,
0,
0
)
);
hourStart = new Date(hourEnd.getTime() - 60 * 60 * 1000);
}
// Upsert hourly stats for the last full hour
await this.statsAggregationService.upsertHourly(hourStart, hourEnd);
@@ -199,3 +218,4 @@ class JobGenerator implements IJobGenerator {
}
export default JobGenerator;
import { config } from "@/config/index.js";
+10 -1
View File
@@ -3,6 +3,7 @@ import { Monitor, IMonitor } from "@/db/models/index.js";
import Scheduler from "super-simple-scheduler";
import { IJobGenerator } from "./JobGenerator.js";
import { getChildLogger } from "@/logger/Logger.js";
import { config } from "@/config/index.js";
const SERVICE_NAME = "JobQueue";
const logger = getChildLogger(SERVICE_NAME);
@@ -86,7 +87,7 @@ export default class JobQueue implements IJobQueue {
active: true,
});
// Stats aggregation job (hourly)
// Stats aggregation job (frequency depends on config)
this.scheduler.addTemplate(
"stats-aggregation-job",
this.jobGenerator.generateStatsAggregationJob()
@@ -98,6 +99,14 @@ export default class JobQueue implements IJobQueue {
active: true,
});
// Trigger an immediate run to populate stats on startup
try {
const runNow = this.jobGenerator.generateStatsAggregationJob();
await runNow();
} catch (e) {
logger.warn("Initial stats aggregation run failed", e as any);
}
const monitors = await Monitor.find();
for (const monitor of monitors) {
const randomOffset = 1000 + Math.random() * 1000;