feat: Add Proxmox LXC container monitoring support

## Summary Add support for monitoring Proxmox LXC containers through the Capture agent's /api/v1/metrics/proxmox endpoint. This feature is opt-in via a new setting in the infrastructure monitor configuration. ## Backend Changes - Add enableProxmoxContainers field to Monitor schema and types - Update networkService to conditionally fetch Proxmox metrics - Handle HTTP 207 multi-status responses and merge errors - Add containers field to Check schema for storing container data - Update validation schemas for the new setting ## Frontend Changes - Add ContainerStats component with charts and stat boxes - Add Containers tab to infrastructure monitor details (conditional) - Add enableProxmoxContainers switch in monitor create/edit form - Fix chart colors for disk I/O and network I/O to be distinct - Add dotColor2 prop to InfrastructureTooltip for multi-series charts - Add translation keys to all 16 locale files ## How It Works 1. User enables "Proxmox container monitoring" in monitor settings 2. Checkmate appends /proxmox to the metrics URL when fetching 3. Container data is displayed in a dedicated tab with: - CPU, memory, disk, swap, and uptime stat boxes - Interactive charts for CPU, memory, disk I/O, network I/O 4. Errors from Capture are merged and logged appropriately
2026-01-16 19:07:49 -05:00
1 parent 173a4e17d2
commit e9da602af6
@@ -4,6 +4,7 @@ import type {
 	Check,
 	CheckAudits,
 	CheckCaptureInfo,
+	CheckContainerInfo,
 	CheckCpuInfo,
 	CheckDiskInfo,
 	CheckErrorInfo,
@@ -143,6 +144,31 @@ const networkInterfaceSchema = new Schema<CheckNetworkInterfaceInfo>(
 	{ _id: false }
 );

+const containerSchema = new Schema<CheckContainerInfo>(
+	{
+		vmid: { type: String, default: "" },
+		name: { type: String, default: "" },
+		node: { type: String, default: "" },
+		status: { type: String, default: "" },
+		type: { type: String, default: "lxc" },
+		uptime: { type: Number, default: 0 },
+		cpu_cores: { type: Number, default: 0 },
+		cpu_usage: { type: Number, default: 0 },
+		memory_used: { type: Number, default: 0 },
+		memory_total: { type: Number, default: 0 },
+		memory_usage: { type: Number, default: 0 },
+		swap_used: { type: Number, default: 0 },
+		swap_total: { type: Number, default: 0 },
+		disk_used: { type: Number, default: 0 },
+		disk_total: { type: Number, default: 0 },
+		disk_read: { type: Number, default: 0 },
+		disk_write: { type: Number, default: 0 },
+		network_in: { type: Number, default: 0 },
+		network_out: { type: Number, default: 0 },
+	},
+	{ _id: false }
+);
+
 const lighthouseAuditSchema = new Schema<ILighthouseAudit>(
 	{
 		id: { type: String },
@@ -256,6 +282,10 @@ const CheckSchema = new Schema<CheckDocument>(
 			type: [networkInterfaceSchema],
 			default: () => [],
 		},
+		containers: {
+			type: [containerSchema],
+			default: () => [],
+		},
 		accessibility: {
 			type: Number,
 		},
@@ -80,6 +80,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
 			type: Boolean,
 			default: false,
 		},
+		enableProxmoxContainers: {
+			type: Boolean,
+			default: false,
+		},
 		jsonPath: {
 			type: String,
 		},
@@ -3,6 +3,7 @@ import type {
 	Check,
 	CheckAudits,
 	CheckCaptureInfo,
+	CheckContainerInfo,
 	CheckCpuInfo,
 	CheckDiskInfo,
 	CheckErrorInfo,
@@ -137,6 +138,29 @@ class MongoChecksRepository implements IChecksRepository {
 				fifo_out: iface?.fifo_out ?? 0,
 			}));

+		const mapContainers = (containers?: CheckContainerInfo[]): CheckContainerInfo[] =>
+			(containers ?? []).map((container) => ({
+				vmid: container?.vmid ?? "",
+				name: container?.name ?? "",
+				node: container?.node ?? "",
+				status: container?.status ?? "",
+				type: container?.type ?? "lxc",
+				uptime: container?.uptime ?? 0,
+				cpu_cores: container?.cpu_cores ?? 0,
+				cpu_usage: container?.cpu_usage ?? 0,
+				memory_used: container?.memory_used ?? 0,
+				memory_total: container?.memory_total ?? 0,
+				memory_usage: container?.memory_usage ?? 0,
+				swap_used: container?.swap_used ?? 0,
+				swap_total: container?.swap_total ?? 0,
+				disk_used: container?.disk_used ?? 0,
+				disk_total: container?.disk_total ?? 0,
+				disk_read: container?.disk_read ?? 0,
+				disk_write: container?.disk_write ?? 0,
+				network_in: container?.network_in ?? 0,
+				network_out: container?.network_out ?? 0,
+			}));
+
 		const mapAudits = (audits?: CheckAudits): CheckAudits | undefined => {
 			if (!audits) {
 				return undefined;
@@ -174,6 +198,7 @@ class MongoChecksRepository implements IChecksRepository {
 			errors: mapErrors(doc.errors),
 			capture: mapCapture(doc.capture),
 			net: mapNet(doc.net),
+			containers: mapContainers(doc.containers),
 			accessibility: doc.accessibility,
 			bestPractices: doc.bestPractices,
 			seo: doc.seo,
@@ -374,6 +399,24 @@ class MongoChecksRepository implements IChecksRepository {
 				deltaFifoIn: iface?.deltaFifoIn ?? 0,
 				deltaFifoOut: iface?.deltaFifoOut ?? 0,
 			})),
+			containers: (metric.containers ?? []).map((container: { [key: string]: number | string | undefined }) => ({
+				vmid: container?.vmid ?? "",
+				name: container?.name ?? "",
+				node: container?.node ?? "",
+				status: container?.status ?? "",
+				type: container?.type ?? "",
+				cpu_cores: container?.cpu_cores ?? 0,
+				avgCpuUsage: container?.avgCpuUsage ?? 0,
+				avgMemoryUsage: container?.avgMemoryUsage ?? 0,
+				memoryTotal: container?.memoryTotal ?? 0,
+				swapTotal: container?.swapTotal ?? 0,
+				diskTotal: container?.diskTotal ?? 0,
+				diskUsed: container?.diskUsed ?? 0,
+				deltaDiskRead: container?.deltaDiskRead ?? 0,
+				deltaDiskWrite: container?.deltaDiskWrite ?? 0,
+				deltaNetIn: container?.deltaNetIn ?? 0,
+				deltaNetOut: container?.deltaNetOut ?? 0,
+			})),
 		}));

 		return {
@@ -451,6 +494,7 @@ class MongoChecksRepository implements IChecksRepository {
 					avgTemperatures: { $push: { $ifNull: ["$cpu.temperature", [0]] } },
 					disks: { $push: "$disk" },
 					net: { $push: "$net" },
+					containers: { $push: "$containers" },
 					updatedAts: { $push: "$updatedAt" },
 					sampleDoc: { $first: "$$ROOT" },
 				},
@@ -570,6 +614,66 @@ class MongoChecksRepository implements IChecksRepository {
 							},
 						},
 					},
+					containers: {
+						$map: {
+							input: { $range: [0, { $size: { $ifNull: ["$sampleDoc.containers", []] } }] },
+							as: "cIdx",
+							in: {
+								vmid: { $arrayElemAt: ["$sampleDoc.containers.vmid", "$$cIdx"] },
+								name: { $arrayElemAt: ["$sampleDoc.containers.name", "$$cIdx"] },
+								node: { $arrayElemAt: ["$sampleDoc.containers.node", "$$cIdx"] },
+								status: { $arrayElemAt: ["$sampleDoc.containers.status", "$$cIdx"] },
+								type: { $arrayElemAt: ["$sampleDoc.containers.type", "$$cIdx"] },
+								cpu_cores: { $arrayElemAt: ["$sampleDoc.containers.cpu_cores", "$$cIdx"] },
+								avgCpuUsage: { $avg: { $map: { input: "$containers", as: "cA", in: { $arrayElemAt: ["$$cA.cpu_usage", "$$cIdx"] } } } },
+								avgMemoryUsage: { $avg: { $map: { input: "$containers", as: "cA", in: { $arrayElemAt: ["$$cA.memory_usage", "$$cIdx"] } } } },
+								memoryTotal: { $arrayElemAt: ["$sampleDoc.containers.memory_total", "$$cIdx"] },
+								swapTotal: { $arrayElemAt: ["$sampleDoc.containers.swap_total", "$$cIdx"] },
+								diskTotal: { $arrayElemAt: ["$sampleDoc.containers.disk_total", "$$cIdx"] },
+								diskUsed: { $avg: { $map: { input: "$containers", as: "cA", in: { $arrayElemAt: ["$$cA.disk_used", "$$cIdx"] } } } },
+								deltaDiskRead: {
+									$let: {
+										vars: {
+											tDiff: { $divide: [{ $subtract: [{ $last: "$updatedAts" }, { $first: "$updatedAts" }] }, 1000] },
+											f: { $arrayElemAt: [{ $map: { input: { $first: "$containers" }, as: "c", in: "$$c.disk_read" } }, "$$cIdx"] },
+											l: { $arrayElemAt: [{ $map: { input: { $last: "$containers" }, as: "c", in: "$$c.disk_read" } }, "$$cIdx"] },
+										},
+										in: { $cond: [{ $gt: ["$$tDiff", 0] }, { $divide: [{ $subtract: ["$$l", "$$f"] }, "$$tDiff"] }, 0] },
+									},
+								},
+								deltaDiskWrite: {
+									$let: {
+										vars: {
+											tDiff: { $divide: [{ $subtract: [{ $last: "$updatedAts" }, { $first: "$updatedAts" }] }, 1000] },
+											f: { $arrayElemAt: [{ $map: { input: { $first: "$containers" }, as: "c", in: "$$c.disk_write" } }, "$$cIdx"] },
+											l: { $arrayElemAt: [{ $map: { input: { $last: "$containers" }, as: "c", in: "$$c.disk_write" } }, "$$cIdx"] },
+										},
+										in: { $cond: [{ $gt: ["$$tDiff", 0] }, { $divide: [{ $subtract: ["$$l", "$$f"] }, "$$tDiff"] }, 0] },
+									},
+								},
+								deltaNetIn: {
+									$let: {
+										vars: {
+											tDiff: { $divide: [{ $subtract: [{ $last: "$updatedAts" }, { $first: "$updatedAts" }] }, 1000] },
+											f: { $arrayElemAt: [{ $map: { input: { $first: "$containers" }, as: "c", in: "$$c.network_in" } }, "$$cIdx"] },
+											l: { $arrayElemAt: [{ $map: { input: { $last: "$containers" }, as: "c", in: "$$c.network_in" } }, "$$cIdx"] },
+										},
+										in: { $cond: [{ $gt: ["$$tDiff", 0] }, { $divide: [{ $subtract: ["$$l", "$$f"] }, "$$tDiff"] }, 0] },
+									},
+								},
+								deltaNetOut: {
+									$let: {
+										vars: {
+											tDiff: { $divide: [{ $subtract: [{ $last: "$updatedAts" }, { $first: "$updatedAts" }] }, 1000] },
+											f: { $arrayElemAt: [{ $map: { input: { $first: "$containers" }, as: "c", in: "$$c.network_out" } }, "$$cIdx"] },
+											l: { $arrayElemAt: [{ $map: { input: { $last: "$containers" }, as: "c", in: "$$c.network_out" } }, "$$cIdx"] },
+										},
+										in: { $cond: [{ $gt: ["$$tDiff", 0] }, { $divide: [{ $subtract: ["$$l", "$$f"] }, "$$tDiff"] }, 0] },
+									},
+								},
+							},
+						},
+					},
 				},
 			},
 			{ $sort: { _id: 1 } },
@@ -225,6 +225,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			statusWindowThreshold: doc.statusWindowThreshold,
 			type: doc.type,
 			ignoreTlsErrors: doc.ignoreTlsErrors,
+			enableProxmoxContainers: doc.enableProxmoxContainers,
 			jsonPath: doc.jsonPath ?? undefined,
 			expectedValue: doc.expectedValue ?? undefined,
 			matchMethod: doc.matchMethod ?? undefined,
@@ -105,7 +105,7 @@ class CheckService {

 		if (type === "hardware") {
 			const hardwarePayload = payload as HardwareStatusPayload | undefined;
-			const { cpu, memory, disk, host, net } = hardwarePayload?.data ?? {};
+			const { cpu, memory, disk, host, net, containers } = hardwarePayload?.data ?? {};
 			const errorsSource = Array.isArray(hardwarePayload?.errors)
 				? hardwarePayload?.errors
 				: (hardwarePayload?.errors as { errors?: CheckErrorInfo[] } | undefined)?.errors;
@@ -116,6 +116,7 @@ class CheckService {
 			check.errors = errorsSource;
 			check.capture = hardwarePayload?.capture;
 			check.net = net;
+			check.containers = containers;
 		}
 		return check;
 	};
@@ -304,7 +304,8 @@ class NetworkService implements INetworkService {

 			Object.assign(httpResponse, {
 				code: response.statusCode,
-				status: response.ok,
+				// 207 Multi-Status is used by Capture when some metrics fail - still a success
+				status: response.ok || response.statusCode === 207,
 				message: response.statusMessage ?? "",
 				responseTime: response.timings.phases.total || 0,
 				payload,
@@ -424,7 +425,112 @@ class NetworkService implements INetworkService {

 	private async requestHardware(monitor: Monitor): Promise<MonitorStatusResponse> {
 		try {
-			return await this.requestHttp(monitor);
+			// Fetch main metrics
+			this.logger.info({
+				message: "Fetching hardware metrics",
+				service: this.SERVICE_NAME,
+				method: "requestHardware",
+				details: { url: monitor.url },
+			});
+			const mainResponse = await this.requestHttp(monitor);
+
+			// If main request failed, return early
+			if (!mainResponse.status) {
+				this.logger.warn({
+					message: "Main metrics request failed",
+					service: this.SERVICE_NAME,
+					method: "requestHardware",
+					details: { url: monitor.url, code: mainResponse.code, msg: mainResponse.message },
+				});
+				return mainResponse;
+			}
+
+			// Only fetch Proxmox metrics if enabled for this monitor
+			if (monitor?.enableProxmoxContainers === true) {
+				try {
+					const proxmoxUrl = this.buildProxmoxUrl(monitor.url);
+					this.logger.info({
+						message: "Attempting to fetch Proxmox metrics",
+						service: this.SERVICE_NAME,
+						method: "requestHardware",
+						details: { proxmoxUrl },
+					});
+					if (proxmoxUrl) {
+						const proxmoxResponse = await this.requestHttp({
+							...monitor,
+							url: proxmoxUrl,
+						});
+
+						// Proxmox response structure: { data: [...], errors: [...] | null, capture: {...} }
+						const proxmoxPayload = proxmoxResponse.payload as {
+							data?: unknown[] | null;
+							errors?: Array<{ metric: string[]; err: string }> | null;
+						};
+						const mainPayload = mainResponse.payload as {
+							data?: Record<string, unknown>;
+							errors?: Array<{ metric: string[]; err: string }>;
+						};
+
+						// Handle based on Capture's response scenarios:
+						// 1. Not configured: data=[], errors=null -> log and skip
+						// 2. Connection failed: data=null, errors=[...] -> merge errors
+						// 3. Partial failure: data=[...], errors=[...] -> merge both
+						// 4. Success: data=[...], errors=null -> merge data
+
+						if (proxmoxResponse.status && mainPayload?.data && typeof mainPayload.data === "object") {
+							// Merge container data if available
+							if (proxmoxPayload?.data && Array.isArray(proxmoxPayload.data)) {
+								if (proxmoxPayload.data.length > 0) {
+									(mainPayload.data as Record<string, unknown>).containers = proxmoxPayload.data;
+									this.logger.info({
+										message: "Successfully merged Proxmox container data",
+										service: this.SERVICE_NAME,
+										method: "requestHardware",
+										details: { containerCount: proxmoxPayload.data.length },
+									});
+								} else {
+									this.logger.debug({
+										message: "Proxmox returned empty container list (not configured or no containers running)",
+										service: this.SERVICE_NAME,
+										method: "requestHardware",
+									});
+								}
+							}
+
+							// Merge Proxmox errors into main errors array
+							if (proxmoxPayload?.errors && Array.isArray(proxmoxPayload.errors) && proxmoxPayload.errors.length > 0) {
+								if (!mainPayload.errors) {
+									mainPayload.errors = [];
+								}
+								mainPayload.errors.push(...proxmoxPayload.errors);
+								this.logger.warn({
+									message: "Proxmox returned errors",
+									service: this.SERVICE_NAME,
+									method: "requestHardware",
+									details: { errors: proxmoxPayload.errors },
+								});
+							}
+						} else if (!proxmoxResponse.status) {
+							this.logger.warn({
+								message: "Proxmox request failed",
+								service: this.SERVICE_NAME,
+								method: "requestHardware",
+								details: { status: proxmoxResponse.status, code: proxmoxResponse.code },
+							});
+						}
+					}
+				} catch (proxmoxErr: any) {
+					// Log but don't fail - Proxmox is optional
+					this.logger.warn({
+						message: "Proxmox fetch error (optional)",
+						service: this.SERVICE_NAME,
+						method: "requestHardware",
+						details: { error: proxmoxErr.message },
+					});
+				}
+			}
+
+			return mainResponse;
 		} catch (err: any) {
 			err.service = this.SERVICE_NAME;
 			err.method = "requestHardware";
@@ -432,6 +538,22 @@ class NetworkService implements INetworkService {
 		}
 	}

+	/**
+	 * Builds the Proxmox metrics URL from the main metrics URL
+	 * e.g., http://localhost:59232/api/v1/metrics -> http://localhost:59232/api/v1/metrics/proxmox
+	 */
+	private buildProxmoxUrl(baseUrl: string | undefined): string | null {
+		if (!baseUrl) return null;
+		try {
+			const url = new URL(baseUrl);
+			// Append /proxmox to the path
+			url.pathname = url.pathname.replace(/\/?$/, "/proxmox");
+			return url.toString();
+		} catch {
+			return null;
+		}
+	}
+
 	private async requestDocker(monitor: Monitor): Promise<MonitorStatusResponse> {
 		try {
 			if (!monitor.url) {
@@ -64,6 +64,28 @@ export interface CheckNetworkInterfaceInfo {
 	fifo_out: number;
 }

+export interface CheckContainerInfo {
+	vmid: string;
+	name: string;
+	node: string;
+	status: string;
+	type: string;
+	uptime: number;
+	cpu_cores: number;
+	cpu_usage: number;
+	memory_used: number;
+	memory_total: number;
+	memory_usage: number;
+	swap_used: number;
+	swap_total: number;
+	disk_used: number;
+	disk_total: number;
+	disk_read: number;
+	disk_write: number;
+	network_in: number;
+	network_out: number;
+}
+
 export interface CheckAudits {
 	cls?: ILighthouseAudit;
 	si?: ILighthouseAudit;
@@ -99,6 +121,7 @@ export interface Check {
 	errors?: CheckErrorInfo[];
 	capture?: CheckCaptureInfo;
 	net?: CheckNetworkInterfaceInfo[];
+	containers?: CheckContainerInfo[];
 	accessibility?: number;
 	bestPractices?: number;
 	seo?: number;
@@ -22,6 +22,7 @@ export interface Monitor {
 	statusWindowThreshold: number;
 	type: MonitorType;
 	ignoreTlsErrors: boolean;
+	enableProxmoxContainers: boolean;
 	jsonPath?: string;
 	expectedValue?: string;
 	matchMethod?: MonitorMatchMethod;
@@ -1,5 +1,6 @@
 import type {
 	CheckCaptureInfo,
+	CheckContainerInfo,
 	CheckCpuInfo,
 	CheckDiskInfo,
 	CheckErrorInfo,
@@ -75,6 +76,7 @@ export interface HardwareStatusMetrics {
 	disk?: CheckDiskInfo[];
 	host?: CheckHostInfo;
 	net?: CheckNetworkInterfaceInfo[];
+	containers?: CheckContainerInfo[];
 }

 export interface HardwareStatusPayload {
@@ -158,6 +158,7 @@ const createMonitorBodyValidation = joi.object({
 	statusWindowThreshold: joi.number().min(1).max(100).default(60),
 	url: joi.string().required(),
 	ignoreTlsErrors: joi.boolean().default(false),
+	enableProxmoxContainers: joi.boolean().default(false),
 	port: joi.number(),
 	isActive: joi.boolean(),
 	interval: joi.number(),
@@ -193,6 +194,7 @@ const editMonitorBodyValidation = joi.object({
 	notifications: joi.array().items(joi.string()),
 	secret: joi.string(),
 	ignoreTlsErrors: joi.boolean(),
+	enableProxmoxContainers: joi.boolean(),
 	jsonPath: joi.string().allow(""),
 	expectedValue: joi.string().allow(""),
 	matchMethod: joi.string().allow(null, ""),