Support Capture-backed Docker monitors
This commit is contained in:
1 parent
da8b230cec
commit
862ff15eef
14 files changed
+399
-33
No files matched your search
@@ -15,7 +15,7 @@ import {
|
||||
import { DateRanges, SortOrders } from "@/types/query.js";
|
||||
import { DockerContainerStates, DockerHealthStatuses, DockerLogStreams, DockerPortProtocols } from "@/domain/docker/docker.type.js";
|
||||
import { DOCKER_LOG_PAGE_DEFAULT, DOCKER_LOG_PAGE_MAX } from "@/domain/docker/docker-log.type.js";
|
||||
import { isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { isCaptureDockerUrl, isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { X509Certificate } from "node:crypto";
|
||||
import { keyMatchesCertificate, parseCertificates, parsePrivateKey } from "@/utils/pem.js";
|
||||
|
||||
@@ -125,15 +125,25 @@ const refineProxySelection = (body: { proxyMode?: string; proxyId?: string }, ct
|
||||
|
||||
const refineDockerUrl = (data: { type?: string; url?: string }, ctx: z.RefinementCtx) => {
|
||||
if (data.type !== "docker" || data.url === undefined) return;
|
||||
if (!isDockerSocketUrl(data.url) && !isDockerTlsUrl(data.url)) {
|
||||
if (!isDockerSocketUrl(data.url) && !isDockerTlsUrl(data.url) && !isCaptureDockerUrl(data.url)) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: ["url"],
|
||||
message: "Docker host must be unix:///path, an absolute socket path, or tcp://host[:port]",
|
||||
message: "Docker host must be a socket path, a TLS daemon URL, or a Capture /metrics/docker endpoint",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const refineCaptureDocker =
|
||||
(mode: "create" | "edit") => (data: { type?: string; url?: string; secret?: string; dockerLogsEnabled?: boolean }, ctx: z.RefinementCtx) => {
|
||||
if (data.type !== "docker" || !isCaptureDockerUrl(data.url)) return;
|
||||
const secretMissing = mode === "create" ? !data.secret?.trim() : data.secret !== undefined && !data.secret.trim();
|
||||
if (secretMissing) ctx.addIssue({ code: "custom", path: ["secret"], message: "Capture API secret is required" });
|
||||
if (data.dockerLogsEnabled) {
|
||||
ctx.addIssue({ code: "custom", path: ["dockerLogsEnabled"], message: "Capture-backed Docker monitors do not support container logs" });
|
||||
}
|
||||
};
|
||||
|
||||
type DockerTlsFields = {
|
||||
type?: string;
|
||||
url?: string;
|
||||
@@ -226,6 +236,7 @@ export const createMonitorBodyValidation = z
|
||||
.superRefine(refineRegexPattern)
|
||||
.superRefine(refineProxySelection)
|
||||
.superRefine(refineDockerUrl)
|
||||
.superRefine(refineCaptureDocker("create"))
|
||||
.superRefine(refineDockerTls("create"));
|
||||
|
||||
export const editMonitorBodyValidation = z
|
||||
@@ -275,6 +286,7 @@ export const editMonitorBodyValidation = z
|
||||
.superRefine(refineRegexPattern)
|
||||
.superRefine(refineProxySelection)
|
||||
.superRefine(refineDockerUrl)
|
||||
.superRefine(refineCaptureDocker("edit"))
|
||||
.superRefine(refineDockerTls("edit"));
|
||||
|
||||
export const pauseMonitorParamValidation = z.object({
|
||||
@@ -353,7 +365,8 @@ const importedMonitorSchema = z
|
||||
.superRefine(refineHeadMatching)
|
||||
.superRefine(refineRegexPattern)
|
||||
.superRefine(refineProxySelection)
|
||||
.superRefine(refineDockerUrl);
|
||||
.superRefine(refineDockerUrl)
|
||||
.superRefine(refineCaptureDocker("create"));
|
||||
|
||||
export const importMonitorsBodyValidation = z.object({
|
||||
monitors: z.array(importedMonitorSchema).min(1, "At least one monitor is required"),
|
||||
|
||||
@@ -82,7 +82,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
|
||||
const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
|
||||
const hardwareProvider = new HardwareProvider(httpProvider);
|
||||
const dockerProvider = new DockerProvider(logger, Docker, encryptionService);
|
||||
const dockerProvider = new DockerProvider(logger, Docker, encryptionService, httpProvider);
|
||||
const portProvider = new PortProvider(net);
|
||||
const gameProvider = new GameProvider(logger, GameDig);
|
||||
const grpcProvider = new GrpcProvider(grpc, protoLoader);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import https from "node:https";
|
||||
|
||||
import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import { HttpProvider } from "@/service/network/HttpProvider.js";
|
||||
import { CheckContext, DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
|
||||
import {
|
||||
@@ -23,10 +24,11 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import Dockerode from "dockerode";
|
||||
import { z } from "zod";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
|
||||
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { DOCKER_TLS_URL, isCaptureDockerUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { splitCertificateBundle } from "@/utils/pem.js";
|
||||
|
||||
type DockerodeType = typeof Dockerode;
|
||||
@@ -40,6 +42,54 @@ const DOCKER_LOG_MAX_LINE_BYTES = 4096;
|
||||
const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
|
||||
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
|
||||
const DOCKER_TLS_DEFAULT_PORT = 2376;
|
||||
const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]);
|
||||
|
||||
const captureDockerResponseSchema = z.object({
|
||||
data: z
|
||||
.array(
|
||||
z.object({
|
||||
container_id: z.string(),
|
||||
container_name: z.string(),
|
||||
status: z.string(),
|
||||
health: z
|
||||
.object({
|
||||
healthy: z.boolean(),
|
||||
source: z.string(),
|
||||
})
|
||||
.nullish(),
|
||||
running: z.boolean(),
|
||||
base_image: z.string(),
|
||||
exposed_ports: z
|
||||
.array(
|
||||
z.object({
|
||||
port: z.string(),
|
||||
protocol: z.string(),
|
||||
})
|
||||
)
|
||||
.optional(),
|
||||
started_at: z.number().optional(),
|
||||
stats: z
|
||||
.object({
|
||||
cpu_percent: z.number(),
|
||||
memory_usage: z.number(),
|
||||
memory_limit: z.number(),
|
||||
memory_percent: z.number(),
|
||||
})
|
||||
.nullish(),
|
||||
})
|
||||
)
|
||||
.nullable(),
|
||||
errors: z
|
||||
.array(
|
||||
z.object({
|
||||
metric: z.array(z.string()),
|
||||
err: z.string(),
|
||||
})
|
||||
)
|
||||
.nullish(),
|
||||
});
|
||||
|
||||
type CaptureContainer = NonNullable<z.infer<typeof captureDockerResponseSchema>["data"]>[number];
|
||||
|
||||
export interface DockerError extends Error {
|
||||
statusCode?: number;
|
||||
@@ -53,7 +103,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
constructor(
|
||||
private logger: ILogger,
|
||||
private DockerLib: DockerodeType,
|
||||
private encryptionService: IEncryptionService
|
||||
private encryptionService: IEncryptionService,
|
||||
private httpProvider: HttpProvider
|
||||
) {}
|
||||
|
||||
supports(type: MonitorType): boolean {
|
||||
@@ -137,6 +188,83 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none";
|
||||
}
|
||||
|
||||
private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary {
|
||||
return {
|
||||
total: containers.length,
|
||||
running: containers.filter((container) => container.state === "running").length,
|
||||
stopped: containers.filter((container) => container.state === "exited" || container.state === "dead").length,
|
||||
unhealthy: containers.filter((container) => container.health === "unhealthy").length,
|
||||
};
|
||||
}
|
||||
|
||||
private toCaptureContainer(container: CaptureContainer): DockerContainerInfo {
|
||||
const startedAt = container.started_at && container.started_at > 0 ? new Date(container.started_at * 1000).toISOString() : undefined;
|
||||
const stats = container.stats
|
||||
? {
|
||||
cpuPct: container.stats.cpu_percent / 100,
|
||||
memoryUsedBytes: container.stats.memory_usage,
|
||||
memoryLimitBytes: container.stats.memory_limit,
|
||||
memoryPct: container.stats.memory_percent / 100,
|
||||
}
|
||||
: {};
|
||||
return {
|
||||
id: container.container_id,
|
||||
name: container.container_name || container.container_id.slice(0, 12),
|
||||
image: container.base_image,
|
||||
state: this.toContainerState(container.status),
|
||||
status: container.status,
|
||||
// Capture derives health from state when no Docker healthcheck exists.
|
||||
// Keep Checkmate's `none` value in that case so summaries retain their existing semantics.
|
||||
health: container.health?.source === "container_health_check" ? (container.health.healthy ? "healthy" : "unhealthy") : "none",
|
||||
startedAt,
|
||||
ports: (container.exposed_ports ?? []).flatMap((port) => {
|
||||
const privatePort = Number(port.port);
|
||||
return Number.isInteger(privatePort) ? [{ privatePort, protocol: this.toPortProtocol(port.protocol) }] : [];
|
||||
}),
|
||||
...stats,
|
||||
};
|
||||
}
|
||||
|
||||
private handleCapture = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
|
||||
const captureUrl = new URL(monitor.url.trim());
|
||||
captureUrl.searchParams.set("all", "true");
|
||||
const response = await this.httpProvider.handle<unknown>({ ...monitor, url: captureUrl.toString() }, ctx);
|
||||
if (!response.status) return this.failed(monitor, response.code, response.message, response.responseTime ?? 0);
|
||||
|
||||
const parsed = captureDockerResponseSchema.safeParse(response.payload);
|
||||
if (!parsed.success) {
|
||||
this.logger.warn({
|
||||
message: "Capture returned an invalid Docker metrics payload",
|
||||
service: SERVICE_NAME,
|
||||
method: "handleCapture",
|
||||
});
|
||||
return this.failed(monitor, NETWORK_ERROR, "Capture returned an invalid Docker metrics payload", response.responseTime ?? 0);
|
||||
}
|
||||
|
||||
const errors = parsed.data.errors ?? [];
|
||||
const fatalError = errors.find((error) => error.metric.some((metric) => CAPTURE_FATAL_ERROR_METRICS.has(metric)));
|
||||
if (fatalError) return this.failed(monitor, NETWORK_ERROR, fatalError.err, response.responseTime ?? 0);
|
||||
if (errors.length > 0) {
|
||||
this.logger.warn({
|
||||
message: `Capture returned ${errors.length} partial Docker collection error(s)`,
|
||||
service: SERVICE_NAME,
|
||||
method: "handleCapture",
|
||||
});
|
||||
}
|
||||
|
||||
const containers = (parsed.data.data ?? []).map((container) => this.toCaptureContainer(container));
|
||||
return {
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: response.code,
|
||||
message: "Docker host is reachable through Capture",
|
||||
responseTime: response.responseTime,
|
||||
payload: { containers, summary: this.toContainerSummary(containers) },
|
||||
};
|
||||
};
|
||||
|
||||
private computeCpuPct(stats: Dockerode.ContainerStats): number {
|
||||
const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage;
|
||||
const systemDelta = stats.cpu_stats.system_cpu_usage - (stats.precpu_stats.system_cpu_usage ?? 0);
|
||||
@@ -328,6 +456,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
|
||||
handle = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
|
||||
try {
|
||||
if (isCaptureDockerUrl(monitor.url)) return await this.handleCapture(monitor, ctx);
|
||||
|
||||
const credentials = this.resolveTlsCredentials(monitor, ctx);
|
||||
if (credentials && !credentials.ok) return this.failed(monitor, NETWORK_ERROR, credentials.message, 0);
|
||||
|
||||
@@ -350,12 +480,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
const containers = await this.mapWithConcurrency(summaries, STATS_CONCURRENCY, (s) => this.toContainerInfo(docker, s));
|
||||
const logs = monitor.dockerLogsEnabled ? await this.collectLogs(docker, summaries) : undefined;
|
||||
|
||||
const summary: DockerContainerSummary = {
|
||||
total: containers.length,
|
||||
running: containers.filter((c) => c.state === "running").length,
|
||||
stopped: containers.filter((c) => c.state === "exited" || c.state === "dead").length,
|
||||
unhealthy: containers.filter((c) => c.health === "unhealthy").length,
|
||||
};
|
||||
const summary = this.toContainerSummary(containers);
|
||||
return {
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
|
||||
@@ -3,3 +3,13 @@ export const DOCKER_SOCKET_URL = /^(?:unix:\/\/\/\S+|\/\S+)$/;
|
||||
|
||||
export const isDockerTlsUrl = (url: string | undefined): boolean => DOCKER_TLS_URL.test(url?.trim() ?? "");
|
||||
export const isDockerSocketUrl = (url: string | undefined): boolean => DOCKER_SOCKET_URL.test(url?.trim() ?? "");
|
||||
|
||||
export const isCaptureDockerUrl = (url: string | undefined): boolean => {
|
||||
try {
|
||||
const parsed = new URL(url?.trim() ?? "");
|
||||
const pathname = parsed.pathname.replace(/\/+$/, "");
|
||||
return (parsed.protocol === "http:" || parsed.protocol === "https:") && pathname.endsWith("/metrics/docker");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
Reference in new issue
Block a user