aggregates
This commit is contained in:
1 parent
95e415ed13
commit
c8dc5b93cb
14 files changed
+302
-99
No files matched your search
@@ -15,6 +15,7 @@ export interface IEnvConfig {
|
||||
SMTP_PASS: string;
|
||||
STRIPE_SECRET: string;
|
||||
STRIPE_WEBHOOK_SECRET: string;
|
||||
STATS_USE_CURRENT_HOUR?: boolean;
|
||||
}
|
||||
|
||||
dotenv.config();
|
||||
@@ -34,4 +35,5 @@ export const config: IEnvConfig = {
|
||||
SMTP_PASS: process.env.SMTP_PASS || "not_set",
|
||||
STRIPE_SECRET: process.env.STRIPE_SECRET || "not_set",
|
||||
STRIPE_WEBHOOK_SECRET: process.env.STRIPE_WEBHOOK_SECRET || "not_set",
|
||||
STATS_USE_CURRENT_HOUR: process.env.STATS_USE_CURRENT_HOUR === "true",
|
||||
};
|
||||
@@ -205,11 +205,6 @@ class MonitorController {
|
||||
|
||||
let monitor;
|
||||
|
||||
const status = req.query.status;
|
||||
if (status && typeof status !== "string") {
|
||||
throw new ApiError("Status query parameter must be a string", 400);
|
||||
}
|
||||
|
||||
if (req.query.embedChecks === "true") {
|
||||
const range = req.query.range;
|
||||
if (!range || typeof range !== "string")
|
||||
@@ -218,8 +213,7 @@ class MonitorController {
|
||||
monitor = await this.monitorService.getEmbedChecks(
|
||||
teamId,
|
||||
monitorId,
|
||||
range,
|
||||
status
|
||||
range
|
||||
);
|
||||
} else {
|
||||
monitor = await this.monitorService.get(teamId, monitorId);
|
||||
|
||||
@@ -100,6 +100,10 @@ const StatsDailySchema = new Schema<IStatsDaily>(
|
||||
finalized: { type: Boolean, default: false },
|
||||
count: { type: Number, required: true, default: 0 },
|
||||
avgResponseTime: { type: Number, required: true, default: 0 },
|
||||
upChecks: { type: Number, required: false },
|
||||
downChecks: { type: Number, required: false },
|
||||
avgResponseTimeUp: { type: Number, required: false },
|
||||
avgResponseTimeDown: { type: Number, required: false },
|
||||
|
||||
// Pagespeed
|
||||
accessibility: { type: Number },
|
||||
@@ -129,4 +133,3 @@ export const StatsDaily = mongoose.model<IStatsDaily>(
|
||||
"StatsDaily",
|
||||
StatsDailySchema
|
||||
);
|
||||
|
||||
@@ -11,6 +11,10 @@ export interface IStatsBase {
|
||||
finalized: boolean;
|
||||
count: number; // number of checks in window
|
||||
avgResponseTime: number; // ms
|
||||
upChecks?: number;
|
||||
downChecks?: number;
|
||||
avgResponseTimeUp?: number;
|
||||
avgResponseTimeDown?: number;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -170,6 +174,10 @@ const StatsHourlySchema = new Schema<IStatsHourly>(
|
||||
finalized: { type: Boolean, default: false },
|
||||
count: { type: Number, required: true, default: 0 },
|
||||
avgResponseTime: { type: Number, required: true, default: 0 },
|
||||
upChecks: { type: Number, required: false },
|
||||
downChecks: { type: Number, required: false },
|
||||
avgResponseTimeUp: { type: Number, required: false },
|
||||
avgResponseTimeDown: { type: Number, required: false },
|
||||
|
||||
// Pagespeed
|
||||
accessibility: { type: Number },
|
||||
@@ -199,4 +207,3 @@ export const StatsHourly = mongoose.model<IStatsHourly>(
|
||||
"StatsHourly",
|
||||
StatsHourlySchema
|
||||
);
|
||||
|
||||
@@ -64,8 +64,7 @@ export interface IMonitorService {
|
||||
getEmbedChecks: (
|
||||
teamId: string,
|
||||
monitorId: string,
|
||||
range: string,
|
||||
status?: string
|
||||
range: string
|
||||
) => Promise<MonitorWithChecksResponse>;
|
||||
togglePause: (
|
||||
userId: string,
|
||||
@@ -504,11 +503,11 @@ class MonitorService implements IMonitorService {
|
||||
return {};
|
||||
};
|
||||
|
||||
// status parameter deprecated and ignored; retained for backward compatibility
|
||||
getEmbedChecks = async (
|
||||
teamId: string,
|
||||
monitorId: string,
|
||||
range: string,
|
||||
status: string | undefined
|
||||
range: string
|
||||
): Promise<MonitorWithChecksResponse> => {
|
||||
const monitor = await Monitor.findOne({ _id: monitorId, teamId });
|
||||
if (!monitor) {
|
||||
@@ -519,7 +518,9 @@ class MonitorService implements IMonitorService {
|
||||
|
||||
// Route for aggregated stats (24h, 7d, 30d)
|
||||
if (range === "24h" || range === "7d" || range === "30d") {
|
||||
const monitorStats = await MonitorStats.findOne({ monitorId: monitor._id });
|
||||
const monitorStats = await MonitorStats.findOne({
|
||||
monitorId: monitor._id,
|
||||
});
|
||||
if (!monitorStats) {
|
||||
throw new ApiError("Monitor stats not found", 404);
|
||||
}
|
||||
@@ -540,6 +541,10 @@ class MonitorService implements IMonitorService {
|
||||
_id: new Date(d.windowStart).toISOString(),
|
||||
count: d.count ?? 0,
|
||||
avgResponseTime: d.avgResponseTime ?? 0,
|
||||
upChecks: d.upChecks ?? 0,
|
||||
downChecks: d.downChecks ?? 0,
|
||||
avgResponseTimeUp: d.avgResponseTimeUp ?? null,
|
||||
avgResponseTimeDown: d.avgResponseTimeDown ?? null,
|
||||
};
|
||||
if (monitor.type === "pagespeed") {
|
||||
return {
|
||||
@@ -579,16 +584,11 @@ class MonitorService implements IMonitorService {
|
||||
const matchStage: {
|
||||
"metadata.monitorId": mongoose.Types.ObjectId;
|
||||
createdAt: { $gte: Date };
|
||||
status?: string;
|
||||
} = {
|
||||
"metadata.monitorId": monitor._id,
|
||||
createdAt: { $gte: startDate },
|
||||
};
|
||||
|
||||
if (status) {
|
||||
matchStage.status = status;
|
||||
}
|
||||
|
||||
const dateFormat = "%Y-%m-%dT%H:%M:00Z";
|
||||
let groupClause;
|
||||
if (monitor.type === "pagespeed") {
|
||||
@@ -620,11 +620,41 @@ class MonitorService implements IMonitorService {
|
||||
$match: matchStage,
|
||||
},
|
||||
{ $sort: { createdAt: 1 } },
|
||||
{ $project: projectStage },
|
||||
{ $group: groupClause },
|
||||
{ $project: { ...projectStage, status: 1 } },
|
||||
{
|
||||
$group: {
|
||||
...groupClause,
|
||||
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
|
||||
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
|
||||
sumResponseUp: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
|
||||
},
|
||||
sumResponseDown: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
|
||||
},
|
||||
},
|
||||
},
|
||||
{ $sort: { _id: -1 } },
|
||||
{
|
||||
$project: finalProjection,
|
||||
$project: {
|
||||
...finalProjection,
|
||||
upChecks: 1,
|
||||
downChecks: 1,
|
||||
avgResponseTimeUp: {
|
||||
$cond: [
|
||||
{ $gt: ["$upChecks", 0] },
|
||||
{ $divide: ["$sumResponseUp", "$upChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
avgResponseTimeDown: {
|
||||
$cond: [
|
||||
{ $gt: ["$downChecks", 0] },
|
||||
{ $divide: ["$sumResponseDown", "$downChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
|
||||
@@ -35,6 +35,46 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
type: { $first: "$metadata.type" },
|
||||
count: { $sum: 1 },
|
||||
avgResponseTime: { $avg: "$responseTime" },
|
||||
upChecks: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] },
|
||||
},
|
||||
downChecks: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] },
|
||||
},
|
||||
sumResponseUp: {
|
||||
$sum: {
|
||||
$cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0],
|
||||
},
|
||||
},
|
||||
sumResponseDown: {
|
||||
$sum: {
|
||||
$cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
teamId: 1,
|
||||
type: 1,
|
||||
count: 1,
|
||||
avgResponseTime: 1,
|
||||
upChecks: 1,
|
||||
downChecks: 1,
|
||||
avgResponseTimeUp: {
|
||||
$cond: [
|
||||
{ $gt: ["$upChecks", 0] },
|
||||
{ $divide: ["$sumResponseUp", "$upChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
avgResponseTimeDown: {
|
||||
$cond: [
|
||||
{ $gt: ["$downChecks", 0] },
|
||||
{ $divide: ["$sumResponseDown", "$downChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
@@ -44,6 +84,7 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
{ $match: { ...matchWindow, "metadata.type": "pagespeed" } },
|
||||
{
|
||||
$project: {
|
||||
status: 1,
|
||||
responseTime: 1,
|
||||
createdAt: 1,
|
||||
"lighthouse.accessibility": 1,
|
||||
@@ -67,6 +108,14 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
type: { $first: "$metadata.type" },
|
||||
count: { $sum: 1 },
|
||||
avgResponseTime: { $avg: "$responseTime" },
|
||||
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
|
||||
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
|
||||
sumResponseUp: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
|
||||
},
|
||||
sumResponseDown: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
|
||||
},
|
||||
accessibility: { $avg: "$lighthouse.accessibility" },
|
||||
bestPractices: { $avg: "$lighthouse.bestPractices" },
|
||||
seo: { $avg: "$lighthouse.seo" },
|
||||
@@ -78,6 +127,39 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
tbt: { $avg: "$lighthouse.audits.tbt.score" },
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
teamId: 1,
|
||||
type: 1,
|
||||
count: 1,
|
||||
avgResponseTime: 1,
|
||||
upChecks: 1,
|
||||
downChecks: 1,
|
||||
avgResponseTimeUp: {
|
||||
$cond: [
|
||||
{ $gt: ["$upChecks", 0] },
|
||||
{ $divide: ["$sumResponseUp", "$upChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
avgResponseTimeDown: {
|
||||
$cond: [
|
||||
{ $gt: ["$downChecks", 0] },
|
||||
{ $divide: ["$sumResponseDown", "$downChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
accessibility: 1,
|
||||
bestPractices: 1,
|
||||
seo: 1,
|
||||
performance: 1,
|
||||
cls: 1,
|
||||
si: 1,
|
||||
fcp: 1,
|
||||
lcp: 1,
|
||||
tbt: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
// 3) Infrastructure
|
||||
@@ -85,6 +167,7 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
{ $match: { ...matchWindow, "metadata.type": "infrastructure" } },
|
||||
{
|
||||
$project: {
|
||||
status: 1,
|
||||
responseTime: 1,
|
||||
createdAt: 1,
|
||||
"system.cpu.physical_core": 1,
|
||||
@@ -116,6 +199,14 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
type: { $first: "$metadata.type" },
|
||||
count: { $sum: 1 },
|
||||
avgResponseTime: { $avg: "$responseTime" },
|
||||
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
|
||||
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
|
||||
sumResponseUp: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
|
||||
},
|
||||
sumResponseDown: {
|
||||
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
|
||||
},
|
||||
physicalCores: { $last: "$system.cpu.physical_core" },
|
||||
logicalCores: { $last: "$system.cpu.logical_core" },
|
||||
frequency: { $avg: "$system.cpu.frequency" },
|
||||
@@ -142,6 +233,22 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
type: 1,
|
||||
count: 1,
|
||||
avgResponseTime: 1,
|
||||
upChecks: 1,
|
||||
downChecks: 1,
|
||||
avgResponseTimeUp: {
|
||||
$cond: [
|
||||
{ $gt: ["$upChecks", 0] },
|
||||
{ $divide: ["$sumResponseUp", "$upChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
avgResponseTimeDown: {
|
||||
$cond: [
|
||||
{ $gt: ["$downChecks", 0] },
|
||||
{ $divide: ["$sumResponseDown", "$downChecks"] },
|
||||
null,
|
||||
],
|
||||
},
|
||||
cpu: {
|
||||
physical_core: "$physicalCores",
|
||||
logical_core: "$logicalCores",
|
||||
@@ -291,6 +398,10 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
finalized: false,
|
||||
count: d.count ?? 0,
|
||||
avgResponseTime: d.avgResponseTime ?? 0,
|
||||
upChecks: d.upChecks ?? 0,
|
||||
downChecks: d.downChecks ?? 0,
|
||||
avgResponseTimeUp: (d.avgResponseTimeUp ?? null) as any,
|
||||
avgResponseTimeDown: (d.avgResponseTimeDown ?? null) as any,
|
||||
accessibility: d.accessibility,
|
||||
bestPractices: d.bestPractices,
|
||||
seo: d.seo,
|
||||
@@ -354,8 +465,12 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
type: string;
|
||||
// totals
|
||||
count: number;
|
||||
upChecks: number;
|
||||
downChecks: number;
|
||||
// base
|
||||
rt: Weighted;
|
||||
rtUp: Weighted;
|
||||
rtDown: Weighted;
|
||||
// pagespeed
|
||||
accessibility: Weighted;
|
||||
bestPractices: Weighted;
|
||||
@@ -400,7 +515,11 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
teamId: h.teamId as any,
|
||||
type: h.type,
|
||||
count: 0,
|
||||
upChecks: 0,
|
||||
downChecks: 0,
|
||||
rt: { sum: 0, w: 0 },
|
||||
rtUp: { sum: 0, w: 0 },
|
||||
rtDown: { sum: 0, w: 0 },
|
||||
accessibility: { sum: 0, w: 0 },
|
||||
bestPractices: { sum: 0, w: 0 },
|
||||
seo: { sum: 0, w: 0 },
|
||||
@@ -428,7 +547,13 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
|
||||
const w = h.count ?? 0;
|
||||
g.count += w;
|
||||
const up = (h as any).upChecks ?? 0;
|
||||
const down = (h as any).downChecks ?? 0;
|
||||
g.upChecks += up;
|
||||
g.downChecks += down;
|
||||
addW(g.rt, h.avgResponseTime, w);
|
||||
addW(g.rtUp, (h as any).avgResponseTimeUp, up);
|
||||
addW(g.rtDown, (h as any).avgResponseTimeDown, down);
|
||||
|
||||
// pagespeed
|
||||
addW(g.accessibility, (h as any).accessibility, w);
|
||||
@@ -601,7 +726,11 @@ class StatsAggregationService implements IStatsAggregationService {
|
||||
$set: {
|
||||
finalized: finalize,
|
||||
count: g.count,
|
||||
upChecks: g.upChecks,
|
||||
downChecks: g.downChecks,
|
||||
avgResponseTime: avg(g.rt) ?? 0,
|
||||
avgResponseTimeUp: (avg(g.rtUp) ?? null) as any,
|
||||
avgResponseTimeDown: (avg(g.rtDown) ?? null) as any,
|
||||
accessibility: avg(g.accessibility),
|
||||
bestPractices: avg(g.bestPractices),
|
||||
seo: avg(g.seo),
|
||||
|
||||
@@ -144,7 +144,7 @@ class JobGenerator implements IJobGenerator {
|
||||
|
||||
generateStatsAggregationJob = () => {
|
||||
return async () => {
|
||||
// Get previous hour window in UTC
|
||||
// Compute hour window in UTC based on flag
|
||||
const now = new Date();
|
||||
const utcNow = new Date(
|
||||
Date.UTC(
|
||||
@@ -157,18 +157,37 @@ class JobGenerator implements IJobGenerator {
|
||||
0
|
||||
)
|
||||
);
|
||||
const hourEnd = new Date(
|
||||
Date.UTC(
|
||||
now.getUTCFullYear(),
|
||||
now.getUTCMonth(),
|
||||
now.getUTCDate(),
|
||||
now.getUTCHours(),
|
||||
0,
|
||||
0,
|
||||
0
|
||||
)
|
||||
);
|
||||
const hourStart = new Date(hourEnd.getTime() - 60 * 60 * 1000);
|
||||
let hourStart: Date;
|
||||
let hourEnd: Date;
|
||||
if (config.STATS_USE_CURRENT_HOUR) {
|
||||
// Current hour [hh:00, hh+1:00)
|
||||
hourStart = new Date(
|
||||
Date.UTC(
|
||||
now.getUTCFullYear(),
|
||||
now.getUTCMonth(),
|
||||
now.getUTCDate(),
|
||||
now.getUTCHours(),
|
||||
0,
|
||||
0,
|
||||
0
|
||||
)
|
||||
);
|
||||
hourEnd = new Date(hourStart.getTime() + 60 * 60 * 1000);
|
||||
} else {
|
||||
// Previous hour [hh-1:00, hh:00)
|
||||
hourEnd = new Date(
|
||||
Date.UTC(
|
||||
now.getUTCFullYear(),
|
||||
now.getUTCMonth(),
|
||||
now.getUTCDate(),
|
||||
now.getUTCHours(),
|
||||
0,
|
||||
0,
|
||||
0
|
||||
)
|
||||
);
|
||||
hourStart = new Date(hourEnd.getTime() - 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
// Upsert hourly stats for the last full hour
|
||||
await this.statsAggregationService.upsertHourly(hourStart, hourEnd);
|
||||
@@ -199,3 +218,4 @@ class JobGenerator implements IJobGenerator {
|
||||
}
|
||||
|
||||
export default JobGenerator;
|
||||
import { config } from "@/config/index.js";
|
||||
@@ -3,6 +3,7 @@ import { Monitor, IMonitor } from "@/db/models/index.js";
|
||||
import Scheduler from "super-simple-scheduler";
|
||||
import { IJobGenerator } from "./JobGenerator.js";
|
||||
import { getChildLogger } from "@/logger/Logger.js";
|
||||
import { config } from "@/config/index.js";
|
||||
|
||||
const SERVICE_NAME = "JobQueue";
|
||||
const logger = getChildLogger(SERVICE_NAME);
|
||||
@@ -86,7 +87,7 @@ export default class JobQueue implements IJobQueue {
|
||||
active: true,
|
||||
});
|
||||
|
||||
// Stats aggregation job (hourly)
|
||||
// Stats aggregation job (frequency depends on config)
|
||||
this.scheduler.addTemplate(
|
||||
"stats-aggregation-job",
|
||||
this.jobGenerator.generateStatsAggregationJob()
|
||||
@@ -98,6 +99,14 @@ export default class JobQueue implements IJobQueue {
|
||||
active: true,
|
||||
});
|
||||
|
||||
// Trigger an immediate run to populate stats on startup
|
||||
try {
|
||||
const runNow = this.jobGenerator.generateStatsAggregationJob();
|
||||
await runNow();
|
||||
} catch (e) {
|
||||
logger.warn("Initial stats aggregation run failed", e as any);
|
||||
}
|
||||
|
||||
const monitors = await Monitor.find();
|
||||
for (const monitor of monitors) {
|
||||
const randomOffset = 1000 + Math.random() * 1000;
|
||||
|
||||
Reference in new issue
Block a user