aggregates

This commit is contained in:
Alex Holliday committed 2025-12-17 21:11:51 +00:00
1 parent 95e415ed13
commit c8dc5b93cb
14 files changed
+302 -99

No files matched your search

+2 -2
View File
@@ -2,6 +2,6 @@ window.__ENV = {
GLOBAL_REFRESH: "60000",
APP_HOST: "http://localhost:5173",
APP_API_BASE_URL: "http://localhost:52345/api/v1",
// APP_DEPLOYMENT_MODE: "self_hosted",
APP_DEPLOYMENT_MODE: "saas",
APP_DEPLOYMENT_MODE: "self_hosted",
// APP_DEPLOYMENT_MODE: "saas",
};
@@ -94,9 +94,12 @@ export const ChartResponseTime = ({
"avgResponseTime"
);
return (
<BaseChart icon={<Clock size={20} strokeWidth={1.5} />} title="Response times">
<BaseChart
icon={<Clock size={20} strokeWidth={1.5} />}
title="Response times"
>
<ResponsiveContainer width="100%" height={300}>
<AreaChart data={normalized?.slice().reverse()}>
<AreaChart data={normalized?.slice()}>
<CartesianGrid
stroke={theme.palette.divider}
strokeWidth={1}
@@ -1,7 +1,14 @@
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { BaseChart } from "@/components/monitors/Chart";
import { ResponsiveContainer, BarChart, XAxis, Bar, Cell } from "recharts";
import {
ResponsiveContainer,
BarChart,
XAxis,
Bar,
Cell,
Tooltip,
} from "recharts";
import { TrendingUp, AlertTriangle } from "lucide-react";
import type { GroupedCheck } from "@/types/check";
@@ -65,7 +72,6 @@ export const HistogramStatus = ({
title: string;
}) => {
const { t } = useTranslation();
const uiTimezone = useAppSelector((state: any) => state.ui.timezone);
const icon =
status === "up" ? (
@@ -74,10 +80,31 @@ export const HistogramStatus = ({
<AlertTriangle size={20} strokeWidth={1.5} />
);
const theme = useTheme();
const [idx, setIdx] = useState<number | null>(null);
const dateFormat = range === "1d" || range === "2h" ? "MMM D, h A" : "MMM D";
const normalChecks = normalizeResponseTimes(checks, "avgResponseTime");
const CustomTooltip = ({ active, payload }: any) => {
if (!active || !payload?.length) return null;
const d = payload[0]?.payload as GroupedCheck & {
avgResponseTime?: number;
};
const avg = d?.avgResponseTime ?? 0;
const titleText = t("common.charts.uptime.avgResponseTime");
return (
<Stack
sx={{
p: 1,
bgcolor: "background.paper",
border: 1,
borderColor: "divider",
borderRadius: 1,
}}
>
<Typography variant="caption">{titleText}</Typography>
<Typography variant="body2">{Math.floor(avg)} ms</Typography>
</Stack>
);
};
if (normalChecks.length === 0) {
return (
<BaseChart icon={icon} title={title}>
@@ -105,23 +132,7 @@ export const HistogramStatus = ({
justifyContent="space-between"
>
<Stack>
<Typography>Total checks</Typography>
{idx ? (
<Stack>
<Typography variant="h2">
{normalChecks?.[idx]?.count}
</Typography>
<Typography position={"absolute"} top={"100%"}>
{formatDateWithTz(
normalChecks?.[idx]?._id,
dateFormat,
uiTimezone
)}
</Typography>
</Stack>
) : (
<Typography variant="h2">{totalChecks}</Typography>
)}
<Typography>Checks: {totalChecks}</Typography>
</Stack>
</Stack>
<ResponsiveContainer width="100%" height={155}>
@@ -138,19 +149,18 @@ export const HistogramStatus = ({
/>
}
/>
<Tooltip cursor={false} content={<CustomTooltip />} />
<Bar
dataKey="normalResponseTime"
maxBarSize={7}
background={{ fill: "transparent" }}
>
{normalChecks?.map((groupedCheck, idx) => {
{normalChecks?.map((groupedCheck) => {
const fillColor = getResponseTimeColor(
groupedCheck.normalResponseTime
);
return (
<Cell
onMouseEnter={() => setIdx(idx)}
onMouseLeave={() => setIdx(null)}
key={groupedCheck._id}
fill={theme.palette[fillColor].main}
/>
+2 -1
View File
@@ -211,7 +211,8 @@
"downTitle": "Downtime",
"emptyDown": "Great, no downtime!",
"emptyUp": "No checks yet",
"upTitle": "Uptime"
"upTitle": "Uptime",
"avgResponseTime": "Avg response time"
}
},
"dialog": {
+25 -34
View File
@@ -40,25 +40,7 @@ const UptimeDetailsPage = () => {
{ refreshInterval: GLOBAL_REFRESH, keepPreviousData: true }
);
const {
response: upResponse,
loading: upIsLoading,
error: upError,
} = useGet<ApiResponse<IMonitorWithMonitorStats>>(
`/monitors/${id}?embedChecks=true&range=${range}&status=up`,
{},
{ keepPreviousData: true }
);
const {
response: downResponse,
error: downError,
loading: downIsLoading,
} = useGet<ApiResponse<IMonitorWithMonitorStats>>(
`/monitors/${id}?embedChecks=true&range=${range}&status=down`,
{},
{ keepPreviousData: true }
);
// Up/Down derived from single response (server returns up/down/all in one payload)
const {
patch,
@@ -85,20 +67,33 @@ const UptimeDetailsPage = () => {
: -1;
const checks = response?.data?.checks || [];
const upChecks = upResponse?.data?.checks
? [...upResponse.data.checks].reverse()
: [];
const downChecks = downResponse?.data?.checks
? [...downResponse.data.checks].reverse()
: [];
const checksAll = checks.length ? [...checks].reverse() : [];
const upChecksMapped = checksAll.map((c: any) => ({
...c,
count: c.upChecks ?? 0,
avgResponseTime: c.avgResponseTimeUp ?? 0,
}));
const totalUp = upChecksMapped.reduce(
(sum: number, c: any) => sum + (c.count ?? 0),
0
);
const upChecks = totalUp === 0 ? [] : upChecksMapped;
const downChecksMapped = checksAll.map((c: any) => ({
...c,
count: c.downChecks ?? 0,
avgResponseTime: c.avgResponseTimeDown ?? 0,
}));
const totalDown = downChecksMapped.reduce(
(sum: number, c: any) => sum + (c.count ?? 0),
0
);
const downChecks = totalDown === 0 ? [] : downChecksMapped;
const palette = getStatusPalette(monitor?.status);
if (error || upError || downError || postError) {
if (error || postError) {
console.error("Error fetching monitor data:", {
error,
upError,
downError,
postError,
});
}
@@ -145,11 +140,7 @@ const UptimeDetailsPage = () => {
}
/>
</Stack>
<HeaderRange
loading={loading || upIsLoading || downIsLoading}
range={range}
setRange={setRange}
/>
<HeaderRange loading={loading} range={range} setRange={setRange} />
<Stack direction={isSmall ? "column" : "row"} gap={theme.spacing(8)}>
<HistogramStatus
title={t("common.charts.uptime.upTitle")}
@@ -165,7 +156,7 @@ const UptimeDetailsPage = () => {
/>
<ChartAvgResponse avg={avgResponseTime} max={maxResponseTime} />
</Stack>
<ChartResponseTime checks={checks} range={range} />
<ChartResponseTime checks={checksAll} range={range} />
<CheckTable monitorId={monitor?._id} />
</BasePage>
);
+4
View File
@@ -147,6 +147,10 @@ export interface GroupedCheck {
_id: string;
avgResponseTime: number;
count: number;
upChecks?: number;
downChecks?: number;
avgResponseTimeUp?: number | null;
avgResponseTimeDown?: number | null;
}
export interface LatestCheck {
+2
View File
@@ -15,6 +15,7 @@ export interface IEnvConfig {
SMTP_PASS: string;
STRIPE_SECRET: string;
STRIPE_WEBHOOK_SECRET: string;
STATS_USE_CURRENT_HOUR?: boolean;
}
dotenv.config();
@@ -34,4 +35,5 @@ export const config: IEnvConfig = {
SMTP_PASS: process.env.SMTP_PASS || "not_set",
STRIPE_SECRET: process.env.STRIPE_SECRET || "not_set",
STRIPE_WEBHOOK_SECRET: process.env.STRIPE_WEBHOOK_SECRET || "not_set",
STATS_USE_CURRENT_HOUR: process.env.STATS_USE_CURRENT_HOUR === "true",
};
+1 -7
View File
@@ -205,11 +205,6 @@ class MonitorController {
let monitor;
const status = req.query.status;
if (status && typeof status !== "string") {
throw new ApiError("Status query parameter must be a string", 400);
}
if (req.query.embedChecks === "true") {
const range = req.query.range;
if (!range || typeof range !== "string")
@@ -218,8 +213,7 @@ class MonitorController {
monitor = await this.monitorService.getEmbedChecks(
teamId,
monitorId,
range,
status
range
);
} else {
monitor = await this.monitorService.get(teamId, monitorId);
+4 -1
View File
@@ -100,6 +100,10 @@ const StatsDailySchema = new Schema<IStatsDaily>(
finalized: { type: Boolean, default: false },
count: { type: Number, required: true, default: 0 },
avgResponseTime: { type: Number, required: true, default: 0 },
upChecks: { type: Number, required: false },
downChecks: { type: Number, required: false },
avgResponseTimeUp: { type: Number, required: false },
avgResponseTimeDown: { type: Number, required: false },
// Pagespeed
accessibility: { type: Number },
@@ -129,4 +133,3 @@ export const StatsDaily = mongoose.model<IStatsDaily>(
"StatsDaily",
StatsDailySchema
);
+8 -1
View File
@@ -11,6 +11,10 @@ export interface IStatsBase {
finalized: boolean;
count: number; // number of checks in window
avgResponseTime: number; // ms
upChecks?: number;
downChecks?: number;
avgResponseTimeUp?: number;
avgResponseTimeDown?: number;
createdAt: Date;
updatedAt: Date;
}
@@ -170,6 +174,10 @@ const StatsHourlySchema = new Schema<IStatsHourly>(
finalized: { type: Boolean, default: false },
count: { type: Number, required: true, default: 0 },
avgResponseTime: { type: Number, required: true, default: 0 },
upChecks: { type: Number, required: false },
downChecks: { type: Number, required: false },
avgResponseTimeUp: { type: Number, required: false },
avgResponseTimeDown: { type: Number, required: false },
// Pagespeed
accessibility: { type: Number },
@@ -199,4 +207,3 @@ export const StatsHourly = mongoose.model<IStatsHourly>(
"StatsHourly",
StatsHourlySchema
);
+43 -13
View File
@@ -64,8 +64,7 @@ export interface IMonitorService {
getEmbedChecks: (
teamId: string,
monitorId: string,
range: string,
status?: string
range: string
) => Promise<MonitorWithChecksResponse>;
togglePause: (
userId: string,
@@ -504,11 +503,11 @@ class MonitorService implements IMonitorService {
return {};
};
// status parameter deprecated and ignored; retained for backward compatibility
getEmbedChecks = async (
teamId: string,
monitorId: string,
range: string,
status: string | undefined
range: string
): Promise<MonitorWithChecksResponse> => {
const monitor = await Monitor.findOne({ _id: monitorId, teamId });
if (!monitor) {
@@ -519,7 +518,9 @@ class MonitorService implements IMonitorService {
// Route for aggregated stats (24h, 7d, 30d)
if (range === "24h" || range === "7d" || range === "30d") {
const monitorStats = await MonitorStats.findOne({ monitorId: monitor._id });
const monitorStats = await MonitorStats.findOne({
monitorId: monitor._id,
});
if (!monitorStats) {
throw new ApiError("Monitor stats not found", 404);
}
@@ -540,6 +541,10 @@ class MonitorService implements IMonitorService {
_id: new Date(d.windowStart).toISOString(),
count: d.count ?? 0,
avgResponseTime: d.avgResponseTime ?? 0,
upChecks: d.upChecks ?? 0,
downChecks: d.downChecks ?? 0,
avgResponseTimeUp: d.avgResponseTimeUp ?? null,
avgResponseTimeDown: d.avgResponseTimeDown ?? null,
};
if (monitor.type === "pagespeed") {
return {
@@ -579,16 +584,11 @@ class MonitorService implements IMonitorService {
const matchStage: {
"metadata.monitorId": mongoose.Types.ObjectId;
createdAt: { $gte: Date };
status?: string;
} = {
"metadata.monitorId": monitor._id,
createdAt: { $gte: startDate },
};
if (status) {
matchStage.status = status;
}
const dateFormat = "%Y-%m-%dT%H:%M:00Z";
let groupClause;
if (monitor.type === "pagespeed") {
@@ -620,11 +620,41 @@ class MonitorService implements IMonitorService {
$match: matchStage,
},
{ $sort: { createdAt: 1 } },
{ $project: projectStage },
{ $group: groupClause },
{ $project: { ...projectStage, status: 1 } },
{
$group: {
...groupClause,
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
sumResponseUp: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
},
sumResponseDown: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
},
},
},
{ $sort: { _id: -1 } },
{
$project: finalProjection,
$project: {
...finalProjection,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
},
},
]);
@@ -35,6 +35,46 @@ class StatsAggregationService implements IStatsAggregationService {
type: { $first: "$metadata.type" },
count: { $sum: 1 },
avgResponseTime: { $avg: "$responseTime" },
upChecks: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] },
},
downChecks: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] },
},
sumResponseUp: {
$sum: {
$cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0],
},
},
sumResponseDown: {
$sum: {
$cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0],
},
},
},
},
{
$project: {
teamId: 1,
type: 1,
count: 1,
avgResponseTime: 1,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
},
},
]);
@@ -44,6 +84,7 @@ class StatsAggregationService implements IStatsAggregationService {
{ $match: { ...matchWindow, "metadata.type": "pagespeed" } },
{
$project: {
status: 1,
responseTime: 1,
createdAt: 1,
"lighthouse.accessibility": 1,
@@ -67,6 +108,14 @@ class StatsAggregationService implements IStatsAggregationService {
type: { $first: "$metadata.type" },
count: { $sum: 1 },
avgResponseTime: { $avg: "$responseTime" },
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
sumResponseUp: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
},
sumResponseDown: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
},
accessibility: { $avg: "$lighthouse.accessibility" },
bestPractices: { $avg: "$lighthouse.bestPractices" },
seo: { $avg: "$lighthouse.seo" },
@@ -78,6 +127,39 @@ class StatsAggregationService implements IStatsAggregationService {
tbt: { $avg: "$lighthouse.audits.tbt.score" },
},
},
{
$project: {
teamId: 1,
type: 1,
count: 1,
avgResponseTime: 1,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
accessibility: 1,
bestPractices: 1,
seo: 1,
performance: 1,
cls: 1,
si: 1,
fcp: 1,
lcp: 1,
tbt: 1,
},
},
]);
// 3) Infrastructure
@@ -85,6 +167,7 @@ class StatsAggregationService implements IStatsAggregationService {
{ $match: { ...matchWindow, "metadata.type": "infrastructure" } },
{
$project: {
status: 1,
responseTime: 1,
createdAt: 1,
"system.cpu.physical_core": 1,
@@ -116,6 +199,14 @@ class StatsAggregationService implements IStatsAggregationService {
type: { $first: "$metadata.type" },
count: { $sum: 1 },
avgResponseTime: { $avg: "$responseTime" },
upChecks: { $sum: { $cond: [{ $eq: ["$status", "up"] }, 1, 0] } },
downChecks: { $sum: { $cond: [{ $eq: ["$status", "down"] }, 1, 0] } },
sumResponseUp: {
$sum: { $cond: [{ $eq: ["$status", "up"] }, "$responseTime", 0] },
},
sumResponseDown: {
$sum: { $cond: [{ $eq: ["$status", "down"] }, "$responseTime", 0] },
},
physicalCores: { $last: "$system.cpu.physical_core" },
logicalCores: { $last: "$system.cpu.logical_core" },
frequency: { $avg: "$system.cpu.frequency" },
@@ -142,6 +233,22 @@ class StatsAggregationService implements IStatsAggregationService {
type: 1,
count: 1,
avgResponseTime: 1,
upChecks: 1,
downChecks: 1,
avgResponseTimeUp: {
$cond: [
{ $gt: ["$upChecks", 0] },
{ $divide: ["$sumResponseUp", "$upChecks"] },
null,
],
},
avgResponseTimeDown: {
$cond: [
{ $gt: ["$downChecks", 0] },
{ $divide: ["$sumResponseDown", "$downChecks"] },
null,
],
},
cpu: {
physical_core: "$physicalCores",
logical_core: "$logicalCores",
@@ -291,6 +398,10 @@ class StatsAggregationService implements IStatsAggregationService {
finalized: false,
count: d.count ?? 0,
avgResponseTime: d.avgResponseTime ?? 0,
upChecks: d.upChecks ?? 0,
downChecks: d.downChecks ?? 0,
avgResponseTimeUp: (d.avgResponseTimeUp ?? null) as any,
avgResponseTimeDown: (d.avgResponseTimeDown ?? null) as any,
accessibility: d.accessibility,
bestPractices: d.bestPractices,
seo: d.seo,
@@ -354,8 +465,12 @@ class StatsAggregationService implements IStatsAggregationService {
type: string;
// totals
count: number;
upChecks: number;
downChecks: number;
// base
rt: Weighted;
rtUp: Weighted;
rtDown: Weighted;
// pagespeed
accessibility: Weighted;
bestPractices: Weighted;
@@ -400,7 +515,11 @@ class StatsAggregationService implements IStatsAggregationService {
teamId: h.teamId as any,
type: h.type,
count: 0,
upChecks: 0,
downChecks: 0,
rt: { sum: 0, w: 0 },
rtUp: { sum: 0, w: 0 },
rtDown: { sum: 0, w: 0 },
accessibility: { sum: 0, w: 0 },
bestPractices: { sum: 0, w: 0 },
seo: { sum: 0, w: 0 },
@@ -428,7 +547,13 @@ class StatsAggregationService implements IStatsAggregationService {
const w = h.count ?? 0;
g.count += w;
const up = (h as any).upChecks ?? 0;
const down = (h as any).downChecks ?? 0;
g.upChecks += up;
g.downChecks += down;
addW(g.rt, h.avgResponseTime, w);
addW(g.rtUp, (h as any).avgResponseTimeUp, up);
addW(g.rtDown, (h as any).avgResponseTimeDown, down);
// pagespeed
addW(g.accessibility, (h as any).accessibility, w);
@@ -601,7 +726,11 @@ class StatsAggregationService implements IStatsAggregationService {
$set: {
finalized: finalize,
count: g.count,
upChecks: g.upChecks,
downChecks: g.downChecks,
avgResponseTime: avg(g.rt) ?? 0,
avgResponseTimeUp: (avg(g.rtUp) ?? null) as any,
avgResponseTimeDown: (avg(g.rtDown) ?? null) as any,
accessibility: avg(g.accessibility),
bestPractices: avg(g.bestPractices),
seo: avg(g.seo),
@@ -144,7 +144,7 @@ class JobGenerator implements IJobGenerator {
generateStatsAggregationJob = () => {
return async () => {
// Get previous hour window in UTC
// Compute hour window in UTC based on flag
const now = new Date();
const utcNow = new Date(
Date.UTC(
@@ -157,18 +157,37 @@ class JobGenerator implements IJobGenerator {
0
)
);
const hourEnd = new Date(
Date.UTC(
now.getUTCFullYear(),
now.getUTCMonth(),
now.getUTCDate(),
now.getUTCHours(),
0,
0,
0
)
);
const hourStart = new Date(hourEnd.getTime() - 60 * 60 * 1000);
let hourStart: Date;
let hourEnd: Date;
if (config.STATS_USE_CURRENT_HOUR) {
// Current hour [hh:00, hh+1:00)
hourStart = new Date(
Date.UTC(
now.getUTCFullYear(),
now.getUTCMonth(),
now.getUTCDate(),
now.getUTCHours(),
0,
0,
0
)
);
hourEnd = new Date(hourStart.getTime() + 60 * 60 * 1000);
} else {
// Previous hour [hh-1:00, hh:00)
hourEnd = new Date(
Date.UTC(
now.getUTCFullYear(),
now.getUTCMonth(),
now.getUTCDate(),
now.getUTCHours(),
0,
0,
0
)
);
hourStart = new Date(hourEnd.getTime() - 60 * 60 * 1000);
}
// Upsert hourly stats for the last full hour
await this.statsAggregationService.upsertHourly(hourStart, hourEnd);
@@ -199,3 +218,4 @@ class JobGenerator implements IJobGenerator {
}
export default JobGenerator;
import { config } from "@/config/index.js";
+10 -1
View File
@@ -3,6 +3,7 @@ import { Monitor, IMonitor } from "@/db/models/index.js";
import Scheduler from "super-simple-scheduler";
import { IJobGenerator } from "./JobGenerator.js";
import { getChildLogger } from "@/logger/Logger.js";
import { config } from "@/config/index.js";
const SERVICE_NAME = "JobQueue";
const logger = getChildLogger(SERVICE_NAME);
@@ -86,7 +87,7 @@ export default class JobQueue implements IJobQueue {
active: true,
});
// Stats aggregation job (hourly)
// Stats aggregation job (frequency depends on config)
this.scheduler.addTemplate(
"stats-aggregation-job",
this.jobGenerator.generateStatsAggregationJob()
@@ -98,6 +99,14 @@ export default class JobQueue implements IJobQueue {
active: true,
});
// Trigger an immediate run to populate stats on startup
try {
const runNow = this.jobGenerator.generateStatsAggregationJob();
await runNow();
} catch (e) {
logger.warn("Initial stats aggregation run failed", e as any);
}
const monitors = await Monitor.find();
for (const monitor of monitors) {
const randomOffset = 1000 + Math.random() * 1000;