From 1ca38a31c93ce8586bbe79a4f55ba5f188af36f2 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 28 Aug 2026 12:24:44 -0700 Subject: [PATCH] add new docker payload types, promote docker type monitor to first class monitor --- server/src/domain/monitors/monitor.model.ts | 3 + .../monitors/monitor.repository.mongo.ts | 1 + server/src/domain/monitors/monitor.type.ts | 1 + server/src/service/network/DockerProvider.ts | 250 ++++++++++-------- server/src/types/network.ts | 34 ++- 5 files changed, 177 insertions(+), 112 deletions(-) diff --git a/server/src/domain/monitors/monitor.model.ts b/server/src/domain/monitors/monitor.model.ts index 15b277efb..ba803dc67 100644 --- a/server/src/domain/monitors/monitor.model.ts +++ b/server/src/domain/monitors/monitor.model.ts @@ -233,6 +233,9 @@ const MonitorSchema = new Schema( secret: { type: String, }, + sshPrivateKey: { + type: String, + }, cpuAlertThreshold: { type: Number, default: 100, diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index 8f773f983..9617ba3f6 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -436,6 +436,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { tags: tagIds, customUpCodes: doc.customUpCodes ?? [], secret: doc.secret ?? undefined, + sshPrivateKey: doc.sshPrivateKey ?? undefined, cpuAlertThreshold: doc.cpuAlertThreshold, cpuAlertCounter: doc.cpuAlertCounter, memoryAlertThreshold: doc.memoryAlertThreshold, diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index 59d91ced9..1767ba17f 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -99,6 +99,7 @@ export interface Monitor { tags: string[]; customUpCodes: HttpStatusCode[]; secret?: string; + sshPrivateKey?: string; cpuAlertThreshold: number; cpuAlertCounter: number; memoryAlertThreshold: number; diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index 9f2a99de7..543f73011 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -1,5 +1,14 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js"; -import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js"; +import { + DockerContainerInfo, + DockerContainerState, + DockerContainerStates, + DockerContainerSummary, + DockerHealthStatus, + DockerHealthStatuses, + DockerStatusPayload, + MonitorStatusResponse, +} from "@/types/network.js"; import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js"; import { ILogger } from "@/utils/logger.js"; import { AppError } from "@/utils/AppError.js"; @@ -9,6 +18,7 @@ import { NETWORK_ERROR } from "@/types/network.js"; type DockerodeType = typeof Dockerode; const SERVICE_NAME = "DockerProvider"; +const STATS_CONCURRENCY = 5; export interface DockerError extends Error { statusCode?: number; @@ -18,13 +28,11 @@ export interface DockerError extends Error { export class DockerProvider implements IStatusProvider { readonly type = "docker"; - private docker: Dockerode; + constructor( private logger: ILogger, private DockerLib: DockerodeType - ) { - this.docker = new this.DockerLib(); - } + ) {} supports(type: MonitorType): boolean { return type === "docker"; @@ -34,141 +42,161 @@ export class DockerProvider implements IStatusProvider { return error instanceof Error && ("statusCode" in error || "reason" in error || "json" in error); } - async handle(monitor: Monitor): Promise> { - const { url: containerInput } = monitor; + private toDockerOptions = (monitor: Monitor): Dockerode.DockerOptions => { + const url = monitor.url?.trim(); + if (!url) throw new AppError({ message: "Docker host URL is required", status: 422, service: SERVICE_NAME, method: "toDockerOptions" }); + if (url.startsWith("unix://")) { + const socketPath = url.slice("unix://".length); + if (!socketPath.startsWith("/")) throw new Error(`Invalid Docker host URL: ${url}`); + return { socketPath }; + } + if (url.startsWith("/")) return { socketPath: url }; + const match = url.match(/^ssh:\/\/(?:([^@\s]+)@)?([^\s/:@]+)(?::(\d{1,5}))?\/?$/); + if (!match) throw new AppError({ message: "Invalid Docker host URL", status: 422, service: SERVICE_NAME, method: "toDockerOptions" }); + const [, username, host, port] = match; + if (!username) + throw new AppError({ + message: "SSH Docker host URL requires a user: ssh://user@host", + status: 422, + service: SERVICE_NAME, + method: "ToDockerOptions", + }); + return { + protocol: "ssh", + host, + port: port ? Number(port) : 22, + username, + sshOptions: { privateKey: monitor.sshPrivateKey }, + }; + }; + + private async mapWithConcurrency(items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = []; + for (let i = 0; i < items.length; i += limit) { + results.push(...(await Promise.all(items.slice(i, i + limit).map(fn)))); + } + return results; + } + + private toContainerState(state: string): DockerContainerState { + return (DockerContainerStates as readonly string[]).includes(state) ? (state as DockerContainerState) : "created"; + } + + private toHealthStatus(status: string | undefined): DockerHealthStatus { + return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none"; + } + + private computeCpuPct(stats: Dockerode.ContainerStats): number { + const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage; + const systemDelta = stats.cpu_stats.system_cpu_usage - (stats.precpu_stats.system_cpu_usage ?? 0); + const onlineCpus = stats.cpu_stats.online_cpus ?? stats.cpu_stats.cpu_usage.percpu_usage?.length ?? 1; + if (!(systemDelta > 0) || cpuDelta < 0) return 0; + return (cpuDelta / systemDelta) * onlineCpus * 100; + } + + private computeMemory(stats: Dockerode.ContainerStats): Pick { + const raw = stats.memory_stats.usage ?? 0; + const pageCache = stats.memory_stats.stats?.inactive_file ?? stats.memory_stats.stats?.cache ?? 0; + const memoryUsedBytes = Math.max(0, raw - pageCache); + const memoryLimitBytes = stats.memory_stats.limit ?? 0; + return { + memoryUsedBytes, + memoryLimitBytes, + memoryPct: memoryLimitBytes > 0 ? (memoryUsedBytes / memoryLimitBytes) * 100 : 0, + }; + } + + private async toContainerInfo(docker: Dockerode, summary: Dockerode.ContainerInfo): Promise { + const info: DockerContainerInfo = { + id: summary.Id, + name: summary.Names?.[0]?.replace(/^\//, "") ?? summary.Id.slice(0, 12), + image: summary.Image, + state: this.toContainerState(summary.State), + status: summary.Status, + health: "none", + }; + + const container = docker.getContainer(summary.Id); + const [inspectResult, statsResult] = await Promise.allSettled([ + container.inspect(), + summary.State === "running" ? container.stats({ stream: false }) : Promise.resolve(null), + ]); + + if (inspectResult.status === "fulfilled") { + info.restartCount = inspectResult.value.RestartCount; + info.startedAt = inspectResult.value.State?.StartedAt; + info.health = this.toHealthStatus(inspectResult.value.State?.Health?.Status); + } else { + this.logger.warn({ + message: `Failed to inspect container ${info.name}`, + service: SERVICE_NAME, + method: "toContainerInfo", + details: { containerId: summary.Id }, + }); + } + + if (statsResult.status === "fulfilled" && statsResult.value) { + info.cpuPct = this.computeCpuPct(statsResult.value); + Object.assign(info, this.computeMemory(statsResult.value)); + } + + return info; + } + + handle = async (monitor: Monitor): Promise> => { try { - if (!containerInput) { - throw new Error("Container name or ID is required for Docker monitor"); - } - - const containers = await this.docker.listContainers({ all: true }); - const normalizedInput = containerInput.replace(/^\/+/, "").toLowerCase(); - - // Priority-based matching to avoid ambiguity: - // 1. Exact full ID match (64-char) - const exactIdMatch = containers.find((c) => c.Id.toLowerCase() === normalizedInput); - - // 2. Exact container name match (case-insensitive) - const exactNameMatch = containers.find((c) => - c.Names.some((name: string) => { - const cleanName = name.replace(/^\/+/, "").toLowerCase(); - return cleanName === normalizedInput; - }) - ); - - // 3. Partial ID match (fallback for backwards compatibility) - const partialIdMatch = containers.find((c) => c.Id.toLowerCase().startsWith(normalizedInput)); - - // Select container based on priority - const targetContainer = exactIdMatch || exactNameMatch || partialIdMatch; - - // Handle no match - if (!targetContainer) { - this.logger.warn({ - message: `No container found for "${monitor.url}".`, - service: SERVICE_NAME, - method: "handle", - details: { url: monitor.url }, - }); - - return { - monitorId: monitor.id, - teamId: monitor.teamId, - type: monitor.type, - status: false, - code: 404, - message: "Docker container not found", - responseTime: 0, - payload: null, - }; - } - - // 5. Handle Ambiguity Check - const matchTypes: string[] = []; - if (exactIdMatch) matchTypes.push("exact ID"); - if (exactNameMatch) matchTypes.push("exact name"); - if (partialIdMatch && !exactIdMatch) matchTypes.push("partial ID"); - - if (matchTypes.length > 1) { - const matchUsed = exactIdMatch ? "exact ID" : "exact name"; - const message = `Ambiguous container match for "${containerInput}". Matched by: ${matchTypes.join(", ")}. Using ${matchUsed} match.`; - - this.logger.warn({ - message, - service: SERVICE_NAME, - method: "handle", - details: { url: containerInput }, - }); - - return { - monitorId: monitor.id, - teamId: monitor.teamId, - type: monitor.type, - status: false, - code: NETWORK_ERROR, - message, - responseTime: 0, - payload: null, - }; - } - - // 6. Inspect Container Status - const container = this.docker.getContainer(targetContainer.Id); - const { response, responseTime, error } = await timeRequest(() => container.inspect()); - + const docker = new this.DockerLib(this.toDockerOptions(monitor)); + // Host reachability is the monitor's status; ping latency is the responseTime + const { responseTime, error } = await timeRequest(() => docker.ping()); if (error) { - let message = "Failed to fetch Docker container information"; + let message = "Docker host is unreachable"; let code = NETWORK_ERROR; - if (this.isDockerError(error)) { code = error.statusCode ?? NETWORK_ERROR; message = error.json?.message ?? error.reason ?? error.message; } else if (error instanceof Error) { message = error.message; } - return { monitorId: monitor.id, teamId: monitor.teamId, type: monitor.type, status: false, - code: code, - message: message, - responseTime, - payload: null, - }; - } - - if (!response) { - return { - monitorId: monitor.id, - teamId: monitor.teamId, - type: monitor.type, - status: false, - code: NETWORK_ERROR, - message: "No response from Docker container inspect", + code, + message, responseTime, payload: null, }; } + const summaries = await docker.listContainers({ all: true }); + const containers = await this.mapWithConcurrency(summaries, STATS_CONCURRENCY, (s) => this.toContainerInfo(docker, s)); + const summary: DockerContainerSummary = { + total: containers.length, + running: containers.filter((c) => c.state === "running").length, + stopped: containers.filter((c) => c.state === "exited" || c.state === "dead").length, + unhealthy: containers.filter((c) => c.health === "unhealthy").length, + }; return { monitorId: monitor.id, teamId: monitor.teamId, type: monitor.type, - status: response.State?.Status === "running", + status: true, code: 200, - message: "Docker container status fetched successfully", + message: "Docker host is reachable", responseTime, - payload: response as unknown as DockerStatusPayload, + payload: { containers, summary }, }; - } catch (err: unknown) { + } catch (error: unknown) { + if (error instanceof AppError) throw error; throw new AppError({ - message: err instanceof Error ? err.message : "Error performing Docker request", + message: error instanceof Error ? error.message : "Error performing Docker request", service: SERVICE_NAME, method: "handle", - details: { url: containerInput }, + details: { + url: monitor.url, + }, }); } - } + }; } diff --git a/server/src/types/network.ts b/server/src/types/network.ts index 0c8dd7cae..f92268f42 100644 --- a/server/src/types/network.ts +++ b/server/src/types/network.ts @@ -96,7 +96,39 @@ export interface HardwareStatusPayload { [key: string]: unknown; } -export type DockerStatusPayload = Record; +// Docker host monitoring +export const DockerContainerStates = ["created", "running", "paused", "restarting", "removing", "exited", "dead"]; +export type DockerContainerState = (typeof DockerContainerStates)[number]; + +export const DockerHealthStatuses = ["healthy", "unhealthy", "starting", "none"] as const; +export type DockerHealthStatus = (typeof DockerHealthStatuses)[number]; + +export interface DockerContainerSummary { + total: number; + running: number; + stopped: number; + unhealthy: number; +} + +export interface DockerContainerInfo { + id: string; + name: string; + image: string; + state: DockerContainerState; + status: string; + health: DockerHealthStatus; + cpuPct?: number; + memoryUsedBytes?: number; + memoryLimitBytes?: number; + memoryPct?: number; + restartCount?: number; + startedAt?: string; // ISO date +} + +export interface DockerStatusPayload { + containers: DockerContainerInfo[]; + summary: DockerContainerSummary; +} export interface PortStatusPayload { success: boolean;