Merge pull request #3815 from bluewave-labs/feat/stacked-timing-chart

feat: stacked timing chart

client/src/Components/design-elements/BaseChart.tsx

@@ -13,6 +13,7 @@ type BaseChartProps = React.PropsWithChildren<{
 	width?: number | string;
 	maxWidth?: number | string;
 	padding?: number | string | ResponsiveStyleValue<number | string>;
+	onClick?: () => void;
 }>;
 
 export const BaseChart = ({
@@ -21,16 +22,22 @@ export const BaseChart = ({
 	width = "100%",
 	maxWidth = "100%",
 	padding,
+	onClick,
 }: BaseChartProps) => {
 	const theme = useTheme();
 	return (
 		<BaseBox
+			onClick={onClick}
 			sx={{
 				padding: padding ?? theme.spacing(LAYOUT.MD),
 				display: "flex",
 				flex: 1,
 				width: width,
 				maxWidth: maxWidth,
+				...(onClick && {
+					cursor: "pointer",
+					"& .recharts-wrapper": { cursor: "pointer !important" },
+				}),
 			}}
 		>
 			<Stack

client/src/Components/monitors/charts/ChartUtils.ts

@@ -0,0 +1,33 @@
+export const computeYAxisCap = (values: number[]): number | undefined => {
+	const sorted = values.slice().sort((a, b) => a - b);
+	if (sorted.length < 2) return undefined;
+	const p95 = sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))];
+	return p95 > 0 ? Math.ceil(p95 * 2) : undefined;
+};
+
+export const PHASE_KEYS = [
+	"avgDns",
+	"avgTcp",
+	"avgTls",
+	"avgRequest",
+	"avgFirstByte",
+	"avgDownload",
+] as const;
+
+export const PHASE_COLOR_KEYS = {
+	avgDns: "dns",
+	avgTcp: "tcp",
+	avgTls: "tls",
+	avgRequest: "request",
+	avgFirstByte: "firstByte",
+	avgDownload: "download",
+} as const;
+
+export const PHASE_LABELS: Record<(typeof PHASE_KEYS)[number], string> = {
+	avgDns: "DNS",
+	avgTcp: "TCP",
+	avgTls: "TLS",
+	avgRequest: "Request",
+	avgFirstByte: "First byte",
+	avgDownload: "Download",
+};

client/src/Components/monitors/charts/HistogramDetails.tsx

@@ -1,53 +1,39 @@
+import Stack from "@mui/material/Stack";
 import { BaseChart, BaseBox } from "@/Components/design-elements";
 import { Clock } from "lucide-react";
+import { Dot } from "@/Components/design-elements";
 import {
 	AreaChart,
 	Area,
 	XAxis,
+	YAxis,
 	Tooltip,
 	CartesianGrid,
 	ResponsiveContainer,
-	Text,
 } from "recharts";
 import type { TooltipProps } from "recharts";
 import type { NameType, ValueType } from "recharts/types/component/DefaultTooltipContent";
 import Typography from "@mui/material/Typography";
-
+import prettyMilliseconds from "pretty-ms";
+import { XTick } from "@/Components/monitors/charts/XTick";
 import { useSelector } from "react-redux";
-import {
-	formatDateWithTz,
-	tickDateFormatLookup,
-	tooltipDateFormatLookup,
-} from "@/Utils/TimeUtils";
+import { formatDateWithTz, tooltipDateFormatLookup } from "@/Utils/TimeUtils";
 import { useTheme } from "@mui/material/styles";
 import type { Theme } from "@mui/material/styles";
-import type { GroupedCheck } from "@/Types/Check";
+import type { GroupedCheck, GroupedUptimeCheck } from "@/Types/Check";
 import type { RootState } from "@/Types/state";
+import { computeYAxisCap } from "@/Components/monitors/charts/ChartUtils";
 
-type XTickProps = {
-	x: number;
-	y: number;
-	payload: { value: string };
-	range: string;
-};
+import { useMemo, useState } from "react";
+import { SPACING } from "@/Utils/Theme/constants";
+import {
+	PHASE_COLOR_KEYS,
+	PHASE_KEYS,
+	PHASE_LABELS,
+} from "@/Components/monitors/charts/ChartUtils";
 
-export const XTick = ({ x, y, payload, range }: XTickProps) => {
-	const format = tickDateFormatLookup(range);
-	const theme = useTheme();
-	const uiTimezone = useSelector((state: RootState) => state.ui.timezone);
-	return (
-		<Text
-			x={x}
-			y={y + 10}
-			textAnchor="middle"
-			fill={theme.palette.text.secondary}
-			fontSize={11}
-			fontWeight={400}
-		>
-			{formatDateWithTz(payload?.value, format, uiTimezone)}
-		</Text>
-	);
-};
+type DetailsCheck = GroupedCheck &
+	Partial<Pick<GroupedUptimeCheck, (typeof PHASE_KEYS)[number]>>;
 
 type ResponseTimeToolTipProps = TooltipProps<ValueType, NameType> & {
 	range: string;
@@ -68,12 +54,59 @@ const ResponseTimeToolTip = ({
 	if (!active) return null;
 
 	const format = tooltipDateFormatLookup(range);
-	console.log(payload);
-	const responseTime = Math.floor(payload[0]?.payload?.originalAvgResponseTime || 0);
+	const responseTime = Math.floor(payload[0]?.payload?.avgResponseTime || 0);
 	return (
 		<BaseBox sx={{ py: theme.spacing(2), px: theme.spacing(4) }}>
 			<Typography>{formatDateWithTz(String(label), format, uiTimezone)}</Typography>
-			<Typography>Response time: {responseTime} ms</Typography>
+			<Typography>
+				Response time:{" "}
+				{prettyMilliseconds(responseTime, { formatSubMilliseconds: true, compact: true })}
+			</Typography>
+		</BaseBox>
+	);
+};
+
+type TimingPhaseToolTipProps = TooltipProps<ValueType, NameType> & {
+	range: string;
+	theme: Theme;
+	uiTimezone: string;
+};
+
+const TimingPhasesToolTip = ({
+	active,
+	payload,
+	range,
+	theme,
+	uiTimezone,
+}: TimingPhaseToolTipProps) => {
+	if (!active || !payload?.length) return null;
+	const bucket = payload[0].payload as GroupedUptimeCheck;
+	const total = PHASE_KEYS.reduce((sum, key) => sum + bucket[key], 0);
+	const format = tooltipDateFormatLookup(range);
+	return (
+		<BaseBox sx={{ py: theme.spacing(2), px: theme.spacing(4) }}>
+			<Typography>{formatDateWithTz(bucket.bucketDate, format, uiTimezone)}</Typography>
+			<Typography>Total: {prettyMilliseconds(total)}</Typography>
+			{PHASE_KEYS.map((key) => (
+				<Stack
+					alignItems={"center"}
+					direction="row"
+					key={key}
+					gap={SPACING.LG}
+				>
+					<Dot
+						size={"12px"}
+						color={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+					/>
+					<Typography>
+						{PHASE_LABELS[key]}:{" "}
+						{prettyMilliseconds(bucket[key], {
+							formatSubMilliseconds: true,
+							compact: true,
+						})}
+					</Typography>
+				</Stack>
+			))}
 		</BaseBox>
 	);
 };
@@ -82,11 +115,25 @@ export const HistogramDetails = ({
 	checks,
 	range,
 }: {
-	checks: GroupedCheck[];
+	checks: DetailsCheck[];
 	range: string;
 }) => {
+	const [showTimingPhases, setShowTimingPhases] = useState(false);
 	const theme = useTheme();
 	const uiTimezone = useSelector((state: RootState) => state.ui.timezone);
+	const yMax = useMemo(() => {
+		const totals = checks.map((check) => {
+			const phaseSum = PHASE_KEYS.reduce((sum, key) => sum + (check[key] ?? 0), 0);
+			return Math.max(check.avgResponseTime ?? 0, phaseSum);
+		});
+		return computeYAxisCap(totals);
+	}, [checks]);
+
+	const hasTimingPhases = useMemo(
+		() => (checks ?? []).some((check) => PHASE_KEYS.some((key) => (check[key] ?? 0) > 0)),
+		[checks]
+	);
+
 	return (
 		<BaseChart
 			icon={
@@ -95,7 +142,11 @@ export const HistogramDetails = ({
 					strokeWidth={1.5}
 				/>
 			}
-			title="Response times"
+			title={showTimingPhases ? "Timing Phases" : "Response Time"}
+			onClick={() => {
+				if (!hasTimingPhases) return;
+				setShowTimingPhases((prev) => !prev);
+			}}
 		>
 			<ResponsiveContainer
 				width="100%"
@@ -128,7 +179,34 @@ export const HistogramDetails = ({
 								stopOpacity={0}
 							/>
 						</linearGradient>
+						{PHASE_KEYS.map((key) => (
+							<linearGradient
+								key={key}
+								id={`phaseGradient-${key}`}
+								x1="0"
+								y1="0"
+								x2="0"
+								y2="1"
+							>
+								<stop
+									offset="0%"
+									stopColor={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+									stopOpacity={0.8}
+								/>
+								<stop
+									offset="100%"
+									stopColor={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+									stopOpacity={0.2}
+								/>
+							</linearGradient>
+						))}
 					</defs>
+					<YAxis
+						hide
+						// scale="sqrt"
+						domain={[0, yMax ?? "auto"]}
+						allowDataOverflow
+					/>
 					<XAxis
 						axisLine={false}
 						tickLine={false}
@@ -141,6 +219,18 @@ export const HistogramDetails = ({
 						)}
 					/>
 
+					{showTimingPhases ? (
+						<Tooltip
+							content={(props) => (
+								<TimingPhasesToolTip
+									{...props}
+									range={range}
+									theme={theme}
+									uiTimezone={uiTimezone}
+								/>
+							)}
+						/>
+					) : (
 						<Tooltip
 							content={(props) => (
 								<ResponseTimeToolTip
@@ -151,12 +241,29 @@ export const HistogramDetails = ({
 								/>
 							)}
 						/>
+					)}
+					{!showTimingPhases && (
 						<Area
 							type="monotone"
 							dataKey="avgResponseTime"
 							stroke={theme.palette.primary.main}
 							fill="url(#colorUv)"
 						/>
+					)}
+					{hasTimingPhases &&
+						showTimingPhases &&
+						PHASE_KEYS.map((key) => {
+							return (
+								<Area
+									key={key}
+									type="monotone"
+									stackId={1}
+									dataKey={key}
+									stroke={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+									fill={`url(#phaseGradient-${key})`}
+								/>
+							);
+						})}
 				</AreaChart>
 			</ResponsiveContainer>
 		</BaseChart>

client/src/Components/monitors/charts/HistogramDetailsStacked.tsx

@@ -0,0 +1,207 @@
+import type { GroupedUptimeCheck } from "@/Types/Check";
+import type { TooltipProps } from "recharts";
+import type { NameType, ValueType } from "recharts/types/component/DefaultTooltipContent";
+import type { Theme } from "@mui/material/styles";
+import type { RootState } from "@/Types/state";
+import { Clock } from "lucide-react";
+import prettyMilliseconds from "pretty-ms";
+
+import Stack from "@mui/material/Stack";
+import {
+	Area,
+	AreaChart,
+	CartesianGrid,
+	ResponsiveContainer,
+	YAxis,
+	XAxis,
+	Tooltip,
+} from "recharts";
+import { BaseBox, BaseChart, Dot } from "@/Components/design-elements";
+import Typography from "@mui/material/Typography";
+import { XTick } from "@/Components/monitors/charts/XTick";
+
+import { formatDateWithTz, tooltipDateFormatLookup } from "@/Utils/TimeUtils";
+import { useSelector } from "react-redux";
+import { useMemo } from "react";
+import { useTheme } from "@mui/material";
+import { SPACING } from "@/Utils/Theme/constants";
+import { computeYAxisCap } from "@/Components/monitors/charts/ChartUtils";
+
+export const PHASE_KEYS = [
+	"avgDns",
+	"avgTcp",
+	"avgTls",
+	"avgRequest",
+	"avgFirstByte",
+	"avgDownload",
+] as const;
+
+const PHASE_COLOR_KEYS = {
+	avgDns: "dns",
+	avgTcp: "tcp",
+	avgTls: "tls",
+	avgRequest: "request",
+	avgFirstByte: "firstByte",
+	avgDownload: "download",
+} as const;
+
+const PHASE_LABELS: Record<(typeof PHASE_KEYS)[number], string> = {
+	avgDns: "DNS",
+	avgTcp: "TCP",
+	avgTls: "TLS",
+	avgRequest: "Request",
+	avgFirstByte: "First byte",
+	avgDownload: "Download",
+};
+
+type TimingPhaseToolTipProps = TooltipProps<ValueType, NameType> & {
+	range: string;
+	theme: Theme;
+	uiTimezone: string;
+};
+
+const TimingPhasesToolTip = ({
+	active,
+	payload,
+	range,
+	theme,
+	uiTimezone,
+}: TimingPhaseToolTipProps) => {
+	if (!active || !payload?.length) return null;
+	const bucket = payload[0].payload as GroupedUptimeCheck;
+	const total = PHASE_KEYS.reduce((sum, key) => sum + bucket[key], 0);
+	const format = tooltipDateFormatLookup(range);
+	return (
+		<BaseBox sx={{ py: theme.spacing(2), px: theme.spacing(4) }}>
+			<Typography>{formatDateWithTz(bucket.bucketDate, format, uiTimezone)}</Typography>
+			<Typography>Total: {prettyMilliseconds(total)}</Typography>
+			{PHASE_KEYS.map((key) => (
+				<Stack
+					alignItems={"center"}
+					direction="row"
+					key={key}
+					gap={SPACING.LG}
+				>
+					<Dot
+						size={"12px"}
+						color={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+					/>
+					<Typography>
+						{PHASE_LABELS[key]}:{" "}
+						{prettyMilliseconds(bucket[key], {
+							formatSubMilliseconds: true,
+							compact: true,
+						})}
+					</Typography>
+				</Stack>
+			))}
+		</BaseBox>
+	);
+};
+
+export const HistogramDetailsStacked = ({
+	checks = [],
+	range,
+}: {
+	checks: GroupedUptimeCheck[];
+	range: string;
+}) => {
+	const theme = useTheme();
+	const uiTimezone = useSelector((state: RootState) => state.ui.timezone);
+
+	// Cap the y domain at 2x the p95 of bucket totals, so typical stacks sit
+	// around mid-height with headroom above
+	const yMax = useMemo(() => {
+		const totals = checks.map((check) =>
+			PHASE_KEYS.reduce((sum, key) => sum + check[key], 0)
+		);
+		return computeYAxisCap(totals);
+	}, [checks]);
+
+	return (
+		<BaseChart
+			icon={
+				<Clock
+					size={20}
+					strokeWidth={1.5}
+				/>
+			}
+			title="Timing phases"
+		>
+			<ResponsiveContainer
+				width="100%"
+				height={300}
+			>
+				<AreaChart data={checks.slice()}>
+					<CartesianGrid
+						stroke={theme.palette.divider}
+						strokeWidth={1}
+						strokeOpacity={1}
+						fill="transparent"
+						vertical={false}
+					/>
+					<defs>
+						{PHASE_KEYS.map((key) => (
+							<linearGradient
+								key={key}
+								id={`phaseGradient-${key}`}
+								x1="0"
+								y1="0"
+								x2="0"
+								y2="1"
+							>
+								<stop
+									offset="0%"
+									stopColor={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+									stopOpacity={0.8}
+								/>
+								<stop
+									offset="100%"
+									stopColor={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+									stopOpacity={0.2}
+								/>
+							</linearGradient>
+						))}
+					</defs>
+					<YAxis
+						hide
+						// scale="sqrt"
+						domain={[0, yMax ?? "auto"]}
+						allowDataOverflow
+					/>
+					<XAxis
+						axisLine={false}
+						tickLine={false}
+						dataKey="bucketDate"
+						tick={(props) => (
+							<XTick
+								{...props}
+								range={range}
+							/>
+						)}
+					/>
+					<Tooltip
+						content={(props) => (
+							<TimingPhasesToolTip
+								{...props}
+								range={range}
+								theme={theme}
+								uiTimezone={uiTimezone}
+							/>
+						)}
+					/>
+					{PHASE_KEYS.map((key) => (
+						<Area
+							key={key}
+							type="monotone"
+							stackId={1}
+							dataKey={key}
+							stroke={theme.palette.chart.phases[PHASE_COLOR_KEYS[key]]}
+							fill={`url(#phaseGradient-${key})`}
+						/>
+					))}
+				</AreaChart>
+			</ResponsiveContainer>
+		</BaseChart>
+	);
+};

client/src/Components/monitors/charts/HistogramInfrastructure.tsx

@@ -8,7 +8,7 @@ import {
 	ResponsiveContainer,
 } from "recharts";
 import { Fragment, useId } from "react";
-import { XTick } from "@/Components/monitors";
+import { XTick } from "@/Components/monitors/charts/XTick";
 
 import { useTheme } from "@mui/material/styles";
 import type { HardwareCheckStats } from "@/Types/Monitor";

client/src/Components/monitors/charts/HistogramPageSpeedDetails.tsx

@@ -1,6 +1,6 @@
 import { BaseChart } from "@/Components/design-elements";
 import { TrendingUp } from "lucide-react";
-import { XTick } from "@/Components/monitors";
+import { XTick } from "@/Components/monitors/charts/XTick";
 import {
 	XAxis,
 	AreaChart,

client/src/Components/monitors/charts/XTick.tsx

@@ -0,0 +1,30 @@
+import { formatDateWithTz, tickDateFormatLookup } from "@/Utils/TimeUtils";
+import { useSelector } from "react-redux";
+import { Text } from "recharts";
+import { useTheme } from "@mui/material";
+import type { RootState } from "@/Types/state";
+
+type XTickProps = {
+	x: number;
+	y: number;
+	payload: { value: string };
+	range: string;
+};
+
+export const XTick = ({ x, y, payload, range }: XTickProps) => {
+	const format = tickDateFormatLookup(range);
+	const theme = useTheme();
+	const uiTimezone = useSelector((state: RootState) => state.ui.timezone);
+	return (
+		<Text
+			x={x}
+			y={y + 10}
+			textAnchor="middle"
+			fill={theme.palette.text.secondary}
+			fontSize={11}
+			fontWeight={400}
+		>
+			{formatDateWithTz(payload?.value, format, uiTimezone)}
+		</Text>
+	);
+};

client/src/Pages/Uptime/Details/index.tsx

@@ -57,7 +57,6 @@ const UptimeDetailsPage = () => {
 		}
 		const params = new URLSearchParams();
 		params.append("dateRange", dateRange);
-		params.append("normalize", "true");
 		return `/monitors/uptime/details/${monitorId}?${params.toString()}`;
 	}, [monitorId, dateRange]);
 
@@ -72,6 +71,7 @@ const UptimeDetailsPage = () => {
 	);
 
 	const monitorData = monitorDetailsData?.monitorData;
+
 	const monitor = monitorData?.monitor;
 	const monitorStats = monitorDetailsData?.monitorStats ?? null;
 
@@ -165,10 +165,18 @@ const UptimeDetailsPage = () => {
 		{ keepPreviousData: true, revalidateOnFocus: false }
 	);
 
+	if (!monitorData) {
+		return null;
+	}
+
+	if (!monitor) {
+		return null;
+	}
+
 	const geoChecksForTable = geoChecksTableData?.geoChecks ?? [];
 	const geoChecksCount = geoChecksTableData?.geoChecksCount ?? 0;
 
-	const geoLocations = monitor?.geoCheckLocations;
+	const geoLocations = monitor.geoCheckLocations;
 
 	const checks = checksData?.checks ?? [];
 	const checksCount = checksData?.checksCount ?? 0;
@@ -215,7 +223,7 @@ const UptimeDetailsPage = () => {
 				/>
 			</Stack>
 			<HistogramDetails
-				checks={monitorData?.groupedChecks || []}
+				checks={monitorData.groupedChecks}
 				range={dateRange}
 			/>
 			<ChecksTable

client/src/Types/Check.ts

@@ -149,6 +149,15 @@ export interface GroupedCheck {
 	totalChecks: number;
 }
 
+export interface GroupedUptimeCheck extends GroupedCheck {
+	avgDns: number;
+	avgTcp: number;
+	avgTls: number;
+	avgRequest: number;
+	avgFirstByte: number;
+	avgDownload: number;
+}
+
 export interface PageSpeedGroupedCheck {
 	bucketDate: string;
 	performance: number;
@@ -252,7 +261,3 @@ export interface HasResponseTime {
 export type NormalizedCheck<T extends HasResponseTime = Check> = T & {
 	originalResponseTime: number;
 };
-
-export type NormalizedUptimeCheck<T extends GroupedCheck = GroupedCheck> = T & {
-	originalAvgResponseTime: number;
-};

client/src/Types/Monitor.ts

@@ -1,4 +1,4 @@
-import type { GroupedCheck, CheckSnapshot } from "@/Types/Check";
+import type { GroupedCheck, CheckSnapshot, GroupedUptimeCheck } from "@/Types/Check";
 import type { PageSpeedGroupedCheck } from "@/Types/Check";
 import type { GeoContinent } from "@/Types/GeoCheck";
 export type { GeoContinent } from "@/Types/GeoCheck";
@@ -137,7 +137,7 @@ export interface MonitorStats {
 
 export interface MonitorData {
 	monitor: Monitor;
-	groupedChecks: GroupedCheck[];
+	groupedChecks: GroupedUptimeCheck[];
 	groupedUpChecks: GroupedCheck[];
 	groupedDownChecks: GroupedCheck[];
 	groupedAvgResponseTime: number;

client/src/Utils/Theme/Palette.ts

@@ -20,8 +20,19 @@ export const colors = {
 	brandGreen: "#13715B",
 	brandGreenLight: "#4DAF94",
 	green200: "#ECF7F2",
+	green700: "#008300",
 	yellow200: "#FFF4E5",
+	yellow500: "#EDA100",
+	yellow600: "#C98500",
 	orange500: "#FFA500",
+	orange600: "#EB6834",
+	orange700: "#D95926",
+	blue400: "#3987E5",
+	blue500: "#2A78D6",
+	aqua500: "#1BAF7A",
+	aqua600: "#199E70",
+	magenta300: "#E87BA4",
+	magenta400: "#D55181",
 };
 
 export const lightPalette = {
@@ -38,6 +49,16 @@ export const lightPalette = {
 		running: colors.green200,
 		paused: colors.yellow200,
 	},
+	chart: {
+		phases: {
+			dns: colors.magenta300,
+			tcp: colors.orange600,
+			tls: colors.aqua500,
+			request: colors.yellow500,
+			firstByte: colors.blue500,
+			download: colors.green700,
+		},
+	},
 };
 
 export const darkPalette = {
@@ -58,4 +79,14 @@ export const darkPalette = {
 		running: alpha(colors.brandGreen, 0.18),
 		paused: alpha(colors.orange500, 0.18),
 	},
+	chart: {
+		phases: {
+			dns: colors.magenta400,
+			tcp: colors.orange700,
+			tls: colors.aqua600,
+			request: colors.yellow600,
+			firstByte: colors.blue500,
+			download: colors.green700,
+		},
+	},
 };

client/src/Utils/Theme/Theme.ts

@@ -16,10 +16,30 @@ declare module "@mui/material/styles" {
 	interface Palette {
 		sidebar: { accent: string };
 		rowStatus: { running: string; paused: string };
+		chart: {
+			phases: {
+				dns: string;
+				tcp: string;
+				tls: string;
+				request: string;
+				firstByte: string;
+				download: string;
+			};
+		};
 	}
 	interface PaletteOptions {
 		sidebar?: { accent: string };
 		rowStatus?: { running: string; paused: string };
+		chart?: {
+			phases: {
+				dns: string;
+				tcp: string;
+				tls: string;
+				request: string;
+				firstByte: string;
+				download: string;
+			};
+		};
 	}
 }
 

server/src/api/validation/monitorValidation.ts

@@ -197,7 +197,6 @@ export const getUptimeDetailsByIdParamValidation = z.object({
 
 export const getUptimeDetailsByIdQueryValidation = z.object({
 	dateRange: z.enum(DateRanges),
-	normalize: booleanCoercion.optional(),
 });
 
 const importedMonitorSchema = z

server/src/db/migration/0009_recomputeResponseTimeFromTimings.ts

@@ -0,0 +1,209 @@
+import mongoose from "mongoose";
+import type { Document } from "mongodb";
+import type { ILogger } from "@/utils/logger.js";
+
+const SERVICE_NAME = "Migration:RecomputeResponseTimeFromTimings";
+const CHECKS_COLLECTION = "checks";
+const BACKUP_COLLECTION = "checks_backup_response_time";
+const FAILED_DOCS_COLLECTION = "checks_response_time_migration_failed";
+const BATCH_SIZE = 1000;
+const NAMESPACE_EXISTS_CODE = 48;
+
+interface MigrationStats {
+	totalSource: number;
+	totalMigrated: number;
+	totalFailed: number;
+}
+
+const getDb = () => {
+	const db = mongoose.connection.db;
+	if (!db) {
+		throw new Error("Database connection is not initialized");
+	}
+	return db;
+};
+
+// `checks` is a time-series collection (0002): measurement fields cannot be updated in place,
+// so recomputing responseTime requires a backup → drop → recreate → re-insert rewrite.
+const createTimeSeriesCollection = async () => {
+	const db = getDb();
+	const existing = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
+	if (existing.length === 0) {
+		try {
+			await db.createCollection(CHECKS_COLLECTION, {
+				timeseries: {
+					timeField: "createdAt",
+					metaField: "metadata",
+					granularity: "seconds",
+				},
+			});
+		} catch (error) {
+			// Mongoose's autoCreate can create `checks` concurrently with this migration
+			if ((error as { code?: number })?.code !== NAMESPACE_EXISTS_CODE) {
+				throw error;
+			}
+		}
+	}
+
+	await db
+		.collection(CHECKS_COLLECTION)
+		.createIndexes([
+			{ key: { createdAt: 1 } },
+			{ key: { "metadata.monitorId": 1, createdAt: 1 } },
+			{ key: { "metadata.monitorId": 1, createdAt: -1 } },
+			{ key: { "metadata.teamId": 1, createdAt: -1 } },
+			{ key: { "metadata.monitorId": 1, "metadata.type": 1, createdAt: -1 } },
+			{ key: { "metadata.teamId": 1, status: 1, createdAt: -1 } },
+		]);
+};
+
+const stripId = (doc: Document): Document => {
+	const { _id, ...rest } = doc;
+	void _id;
+	return rest;
+};
+
+const recomputeResponseTime = (doc: Document): Document => {
+	const rest = stripId(doc);
+	const total = doc.timings?.phases?.total;
+	if (typeof total === "number") {
+		rest.responseTime = total;
+	}
+	return rest;
+};
+
+const storeFailedDocument = async (doc: Document, reason: string) => {
+	const db = getDb();
+	await db.collection(FAILED_DOCS_COLLECTION).insertOne({
+		originalDoc: doc,
+		reason,
+		failedAt: new Date(),
+	});
+};
+
+const streamBackupIntoChecks = async (transform: (doc: Document) => Document, collectFailures: boolean): Promise<MigrationStats> => {
+	const db = getDb();
+	const source = db.collection(BACKUP_COLLECTION);
+	const target = db.collection(CHECKS_COLLECTION);
+
+	const stats: MigrationStats = {
+		totalSource: await source.countDocuments(),
+		totalMigrated: 0,
+		totalFailed: 0,
+	};
+
+	const cursor = source.find();
+	// Derive the op type from the exact bulkWrite we call so it matches whichever `mongodb` copy
+	// Mongoose resolves (see 0002 for the nested-node_modules background)
+	type BulkOp = Parameters<typeof target.bulkWrite>[0][number];
+	const operations: BulkOp[] = [];
+	const invalidDocs: Document[] = [];
+
+	while (await cursor.hasNext()) {
+		const doc = await cursor.next();
+		if (!doc) {
+			continue;
+		}
+
+		// Time-series inserts reject documents without the timeField
+		if (!doc.createdAt) {
+			invalidDocs.push(doc);
+			continue;
+		}
+
+		operations.push({ insertOne: { document: transform(doc) } });
+
+		if (operations.length >= BATCH_SIZE) {
+			const result = await target.bulkWrite(operations, { ordered: false });
+			stats.totalMigrated += result.insertedCount;
+			operations.length = 0;
+		}
+	}
+
+	if (operations.length) {
+		const result = await target.bulkWrite(operations, { ordered: false });
+		stats.totalMigrated += result.insertedCount;
+	}
+
+	if (collectFailures) {
+		for (const doc of invalidDocs) {
+			await storeFailedDocument(doc, "Missing createdAt (required timeField)");
+			stats.totalFailed++;
+		}
+	} else {
+		stats.totalFailed = invalidDocs.length;
+	}
+
+	await cursor.close();
+
+	return stats;
+};
+
+// Restore cannot be a simple rename like 0002's: $out produced a standard collection, and
+// renaming it to `checks` would silently lose the time-series type. Rebuild and re-insert instead.
+const restoreFromBackup = async () => {
+	const db = getDb();
+	const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray();
+	if (backupExists.length === 0) {
+		throw new Error("Cannot restore: backup collection does not exist");
+	}
+
+	const checksExists = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
+	if (checksExists.length > 0) {
+		await db.collection(CHECKS_COLLECTION).drop();
+	}
+
+	await createTimeSeriesCollection();
+	await streamBackupIntoChecks(stripId, false);
+};
+
+export async function recomputeResponseTimeFromTimings(logger: ILogger): Promise<void> {
+	const db = getDb();
+
+	const checksExists = await db.listCollections({ name: CHECKS_COLLECTION }).toArray();
+	if (checksExists.length === 0) {
+		logger.info({ service: SERVICE_NAME, message: "No checks collection, nothing to migrate" });
+		return;
+	}
+
+	// Fresh installs and non-HTTP-only deployments have no stored timings — skip the rewrite entirely
+	const needsMigration = await db.collection(CHECKS_COLLECTION).countDocuments({ "timings.phases.total": { $exists: true } }, { limit: 1 });
+	if (needsMigration === 0) {
+		logger.info({ service: SERVICE_NAME, message: "No checks with timings.phases.total, nothing to migrate" });
+		return;
+	}
+
+	const backupExists = await db.listCollections({ name: BACKUP_COLLECTION }).toArray();
+	if (backupExists.length > 0) {
+		throw new Error(`Backup collection "${BACKUP_COLLECTION}" already exists. Please remove it manually before running migration.`);
+	}
+
+	await db
+		.collection(CHECKS_COLLECTION)
+		.aggregate([{ $match: {} }, { $out: BACKUP_COLLECTION }])
+		.toArray();
+	await db.collection(CHECKS_COLLECTION).drop();
+
+	try {
+		await createTimeSeriesCollection();
+		const stats = await streamBackupIntoChecks(recomputeResponseTime, true);
+		logger.info({
+			service: SERVICE_NAME,
+			message:
+				`Migrated ${stats.totalMigrated}/${stats.totalSource} checks, ${stats.totalFailed} failed ` +
+				`(failures stored in ${FAILED_DOCS_COLLECTION}); backup retained in ${BACKUP_COLLECTION}`,
+		});
+	} catch (error) {
+		try {
+			await restoreFromBackup();
+			logger.error({ service: SERVICE_NAME, message: "Migration failed, checks restored from backup" });
+		} catch (restoreError) {
+			const restoreMessage = restoreError instanceof Error ? restoreError.message : String(restoreError);
+			logger.error({
+				service: SERVICE_NAME,
+				message: `Migration failed AND rollback failed (${restoreMessage}) — original data is preserved in ${BACKUP_COLLECTION}`,
+			});
+		}
+		throw error;
+	}
+}

server/src/db/migration/index.ts

@@ -7,6 +7,7 @@ import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArra
 import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js";
 import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js";
 import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js";
+import { recomputeResponseTimeFromTimings } from "./0009_recomputeResponseTimeFromTimings.js";
 import type { ILogger } from "@/utils/logger.js";
 
 type MigrationEntry = {
@@ -23,6 +24,7 @@ const migrations: MigrationEntry[] = [
 	{ name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex },
 	{ name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray },
 	{ name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt },
+	{ name: "0009_recomputeResponseTimeFromTimings", execute: recomputeResponseTimeFromTimings },
 ];
 
 const runMigrations = async (logger: ILogger) => {

server/src/domain/checks/check.repository.mongo.ts

@@ -445,11 +445,30 @@ class MongoChecksRepository implements IChecksRepository {
 									$dateToString: { format: dateString, date: "$createdAt" },
 								},
 								avgResponseTime: { $avg: "$responseTime" },
+								avgDns: { $avg: { $ifNull: ["$timings.phases.dns", 0] } },
+								avgTcp: { $avg: { $ifNull: ["$timings.phases.tcp", 0] } },
+								avgTls: { $avg: { $ifNull: ["$timings.phases.tls", 0] } },
+								avgRequest: { $avg: { $ifNull: ["$timings.phases.request", 0] } },
+								avgFirstByte: { $avg: { $ifNull: ["$timings.phases.firstByte", 0] } },
+								avgDownload: { $avg: { $ifNull: ["$timings.phases.download", 0] } },
 								totalChecks: { $sum: 1 },
 							},
 						},
 						{ $sort: { _id: 1 } },
-						{ $project: { bucketDate: "$_id", avgResponseTime: 1, totalChecks: 1, _id: 0 } },
+						{
+							$project: {
+								bucketDate: "$_id",
+								avgResponseTime: 1,
+								totalChecks: 1,
+								avgDns: 1,
+								avgTcp: 1,
+								avgTls: 1,
+								avgRequest: 1,
+								avgFirstByte: 1,
+								avgDownload: 1,
+								_id: 0,
+							},
+						},
 					],
 					groupedUpChecks: [
 						{ $match: { status: true } },

server/src/domain/checks/check.type.ts

@@ -132,6 +132,15 @@ export interface GroupedCheck {
 	totalChecks: number;
 }
 
+export interface GroupedUptimeCheck extends GroupedCheck {
+	avgDns: number;
+	avgTcp: number;
+	avgTls: number;
+	avgRequest: number;
+	avgFirstByte: number;
+	avgDownload: number;
+}
+
 export interface PageSpeedGroupedCheck {
 	bucketDate: string;
 	performance: number;
@@ -143,7 +152,7 @@ export interface PageSpeedGroupedCheck {
 
 export interface UptimeChecksResult {
 	monitorType: Exclude<MonitorType, "hardware" | "pagespeed">;
-	groupedChecks: GroupedCheck[];
+	groupedChecks: GroupedUptimeCheck[];
 	groupedUpChecks: GroupedCheck[];
 	groupedDownChecks: GroupedCheck[];
 	uptimePercentage: number;
@@ -163,10 +172,6 @@ export type NormalizedCheck<T extends HasResponseTime = Check> = T & {
 	originalResponseTime: number;
 };
 
-export type NormalizedUptimeCheck<T extends GroupedCheck = GroupedCheck> = T & {
-	originalAvgResponseTime: number;
-};
-
 export type CheckSnapshot = Omit<Check, "metadata" | "updatedAt">;
 
 export interface HardwareDiskStats {

server/src/domain/monitors/monitor.service.ts

@@ -1,4 +1,4 @@
-import { NormalizeData, NormalizeDataUptimeDetails } from "@/utils/dataUtils.js";
+import { NormalizeData } from "@/utils/dataUtils.js";
 import { type Monitor } from "@/domain/monitors/monitor.type.js";
 import type {
 	MonitorType,
@@ -193,9 +193,9 @@ export class MonitorService implements IMonitorService {
 		return {
 			monitorData: {
 				monitor,
-				groupedChecks: NormalizeDataUptimeDetails(checksData.groupedChecks, 10, 100),
-				groupedUpChecks: NormalizeDataUptimeDetails(checksData.groupedUpChecks, 10, 100),
-				groupedDownChecks: NormalizeDataUptimeDetails(checksData.groupedDownChecks, 10, 100),
+				groupedChecks: checksData.groupedChecks,
+				groupedUpChecks: checksData.groupedUpChecks,
+				groupedDownChecks: checksData.groupedDownChecks,
 				groupedAvgResponseTime: checksData.avgResponseTime,
 				groupedUptimePercentage: checksData.uptimePercentage,
 			},

server/src/service/network/HttpProvider.ts

@@ -117,7 +117,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 		if (error instanceof HTTPError || error instanceof RequestError) {
 			const statusCode = error.response?.statusCode;
 			const statusUp = isStatusUp(statusCode, monitor.customUpCodes);
-			const responseTime = error.timings?.phases?.firstByte ?? error.timings?.phases?.total ?? 0;
+			const responseTime = error.timings?.phases?.total ?? 0;
 
 			if (!statusUp) {
 				return {
@@ -184,7 +184,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 				statusCode: response.statusCode,
 				statusUp,
 				message: response.statusMessage ?? "OK",
-				responseTime: response.timings.phases.firstByte ?? 0,
+				responseTime: response.timings.phases.total ?? 0,
 				timings: response.timings,
 			});
 		} catch (error: unknown) {

server/src/utils/dataUtils.ts

@@ -1,4 +1,4 @@
-import type { GroupedCheck, HasResponseTime, NormalizedCheck, NormalizedUptimeCheck } from "@/domain/checks/check.type.js";
+import type { HasResponseTime, NormalizedCheck } from "@/domain/checks/check.type.js";
 import { type DateRange } from "@/types/query.js";
 
 export const getDateForRange = (dateRange: DateRange): Date => {
@@ -61,17 +61,3 @@ export const NormalizeData = <T extends HasResponseTime>(checks: T[], rangeMin:
 		originalResponseTime: original(check),
 	}));
 };
-
-export const NormalizeDataUptimeDetails = <T extends GroupedCheck>(checks: T[], rangeMin: number, rangeMax: number): NormalizedUptimeCheck<T>[] => {
-	const original = (check: T) => (typeof check.avgResponseTime === "number" ? check.avgResponseTime : 0);
-	if (checks.length <= 1) {
-		return checks.map((check) => ({ ...check, originalAvgResponseTime: original(check) }));
-	}
-	const min = percentileBy(checks, 0, (c) => c.avgResponseTime);
-	const max = percentileBy(checks, 95, (c) => c.avgResponseTime);
-	return checks.map((check) => ({
-		...check,
-		avgResponseTime: rescale(check.avgResponseTime, min, max, rangeMin, rangeMax),
-		originalAvgResponseTime: original(check),
-	}));
-};

server/test/unit/providers/network/httpProvider.test.ts

@@ -112,7 +112,7 @@ describe("HttpProvider", () => {
 					status: true,
 					code: 200,
 					message: "OK",
-					responseTime: 100,
+					responseTime: 120,
 				})
 			);
 		});
@@ -182,8 +182,8 @@ describe("HttpProvider", () => {
 			);
 		});
 
-		it("defaults responseTime to 0 when timings.phases.firstByte is undefined", async () => {
-			mockGot.mockResolvedValue(makeGotResponse({ timings: { phases: { firstByte: undefined } } }));
+		it("defaults responseTime to 0 when timings.phases.total is undefined", async () => {
+			mockGot.mockResolvedValue(makeGotResponse({ timings: { phases: { total: undefined } } }));
 			const { provider } = createProvider();
 
 			const result = await provider.handle(makeMonitor());
@@ -224,11 +224,11 @@ describe("HttpProvider", () => {
 			expect(result.message).toBe("Response is not JSON");
 		});
 
-		it("defaults responseTime to 0 in non-JSON jsonPath response when firstByte is undefined", async () => {
+		it("defaults responseTime to 0 in non-JSON jsonPath response when total is undefined", async () => {
 			mockGot.mockResolvedValue(
 				makeGotResponse({
 					headers: { "content-type": "text/html" },
-					timings: { phases: { firstByte: undefined } },
+					timings: { phases: { total: undefined } },
 				})
 			);
 			const { provider } = createProvider();

server/test/unit/utils/dataUtils.test.ts

@@ -1,6 +1,5 @@
 import { describe, expect, it } from "@jest/globals";
-import { getDateForRange, getDateFormat, NormalizeData, NormalizeDataUptimeDetails } from "../../../src/utils/dataUtils.ts";
-import type { GroupedCheck } from "../../../src/domain/checks/check.type.ts";
+import { getDateForRange, getDateFormat, NormalizeData } from "../../../src/utils/dataUtils.ts";
 import { DateRanges, type DateRange } from "../../../src/types/query.ts";
 
 // ── Helpers ──────────────────────────────────────────────────────────────────
@@ -12,13 +11,6 @@ const makeCheck = (responseTime: number, overrides?: Record<string, unknown>) =>
 	...overrides,
 });
 
-const makeGrouped = (avgResponseTime: number, overrides?: Partial<GroupedCheck>): GroupedCheck => ({
-	bucketDate: `bucket-${avgResponseTime}`,
-	avgResponseTime,
-	totalChecks: 5,
-	...overrides,
-});
-
 // ── Tests ────────────────────────────────────────────────────────────────────
 
 describe("getDateForRange", () => {
@@ -126,50 +118,3 @@ describe("NormalizeData", () => {
 		});
 	});
 });
-
-describe("NormalizeDataUptimeDetails", () => {
-	it("rescales avgResponseTime into [rangeMin, rangeMax] anchored to the p0–p95 window", () => {
-		const checks = [100, 200, 300, 400, 500].map((avg) => makeGrouped(avg));
-
-		const result = NormalizeDataUptimeDetails(checks, 10, 100);
-
-		expect(result[0]!.avgResponseTime).toBe(10);
-		expect(result[1]!.avgResponseTime).toBeCloseTo(33.6842, 4);
-		expect(result[4]!.avgResponseTime).toBe(100);
-	});
-
-	it("preserves the raw value in originalAvgResponseTime and spreads other fields", () => {
-		const checks = [makeGrouped(100), makeGrouped(500)];
-
-		const result = NormalizeDataUptimeDetails(checks, 10, 100);
-
-		expect(result[0]!.originalAvgResponseTime).toBe(100);
-		expect(result[1]!.originalAvgResponseTime).toBe(500);
-		expect(result[0]).toMatchObject({ bucketDate: "bucket-100", totalChecks: 5 });
-	});
-
-	it("passes a single bucket through, only adding originalAvgResponseTime", () => {
-		const result = NormalizeDataUptimeDetails([makeGrouped(250)], 10, 100);
-
-		expect(result).toEqual([{ bucketDate: "bucket-250", avgResponseTime: 250, totalChecks: 5, originalAvgResponseTime: 250 }]);
-	});
-
-	it("returns an empty array for empty input", () => {
-		expect(NormalizeDataUptimeDetails([], 10, 100)).toEqual([]);
-	});
-
-	it("records originalAvgResponseTime as 0 for a null avgResponseTime ($avg over an empty bucket)", () => {
-		const result = NormalizeDataUptimeDetails([makeGrouped(null as unknown as number)], 10, 100);
-
-		expect(result[0]!.originalAvgResponseTime).toBe(0);
-	});
-
-	it("produces NaN when all values are equal (division by zero — known edge case)", () => {
-		const result = NormalizeDataUptimeDetails([makeGrouped(50), makeGrouped(50)], 10, 100);
-
-		result.forEach((check) => {
-			expect(check.avgResponseTime).toBeNaN();
-			expect(check.originalAvgResponseTime).toBe(50);
-		});
-	});
-});