Support Capture-backed Docker monitors

This commit is contained in:
fabiangigler committed 2026-09-18 19:54:33 +02:00
1 parent da8b230cec
commit 862ff15eef
14 files changed
+399 -33

No files matched your search

+17 -4
View File
@@ -15,7 +15,7 @@ import {
import { DateRanges, SortOrders } from "@/types/query.js";
import { DockerContainerStates, DockerHealthStatuses, DockerLogStreams, DockerPortProtocols } from "@/domain/docker/docker.type.js";
import { DOCKER_LOG_PAGE_DEFAULT, DOCKER_LOG_PAGE_MAX } from "@/domain/docker/docker-log.type.js";
import { isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
import { isCaptureDockerUrl, isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
import { X509Certificate } from "node:crypto";
import { keyMatchesCertificate, parseCertificates, parsePrivateKey } from "@/utils/pem.js";
@@ -125,15 +125,25 @@ const refineProxySelection = (body: { proxyMode?: string; proxyId?: string }, ct
const refineDockerUrl = (data: { type?: string; url?: string }, ctx: z.RefinementCtx) => {
if (data.type !== "docker" || data.url === undefined) return;
if (!isDockerSocketUrl(data.url) && !isDockerTlsUrl(data.url)) {
if (!isDockerSocketUrl(data.url) && !isDockerTlsUrl(data.url) && !isCaptureDockerUrl(data.url)) {
ctx.addIssue({
code: "custom",
path: ["url"],
message: "Docker host must be unix:///path, an absolute socket path, or tcp://host[:port]",
message: "Docker host must be a socket path, a TLS daemon URL, or a Capture /metrics/docker endpoint",
});
}
};
const refineCaptureDocker =
(mode: "create" | "edit") => (data: { type?: string; url?: string; secret?: string; dockerLogsEnabled?: boolean }, ctx: z.RefinementCtx) => {
if (data.type !== "docker" || !isCaptureDockerUrl(data.url)) return;
const secretMissing = mode === "create" ? !data.secret?.trim() : data.secret !== undefined && !data.secret.trim();
if (secretMissing) ctx.addIssue({ code: "custom", path: ["secret"], message: "Capture API secret is required" });
if (data.dockerLogsEnabled) {
ctx.addIssue({ code: "custom", path: ["dockerLogsEnabled"], message: "Capture-backed Docker monitors do not support container logs" });
}
};
type DockerTlsFields = {
type?: string;
url?: string;
@@ -226,6 +236,7 @@ export const createMonitorBodyValidation = z
.superRefine(refineRegexPattern)
.superRefine(refineProxySelection)
.superRefine(refineDockerUrl)
.superRefine(refineCaptureDocker("create"))
.superRefine(refineDockerTls("create"));
export const editMonitorBodyValidation = z
@@ -275,6 +286,7 @@ export const editMonitorBodyValidation = z
.superRefine(refineRegexPattern)
.superRefine(refineProxySelection)
.superRefine(refineDockerUrl)
.superRefine(refineCaptureDocker("edit"))
.superRefine(refineDockerTls("edit"));
export const pauseMonitorParamValidation = z.object({
@@ -353,7 +365,8 @@ const importedMonitorSchema = z
.superRefine(refineHeadMatching)
.superRefine(refineRegexPattern)
.superRefine(refineProxySelection)
.superRefine(refineDockerUrl);
.superRefine(refineDockerUrl)
.superRefine(refineCaptureDocker("create"));
export const importMonitorsBodyValidation = z.object({
monitors: z.array(importedMonitorSchema).min(1, "At least one monitor is required"),
+1 -1
View File
@@ -82,7 +82,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
const hardwareProvider = new HardwareProvider(httpProvider);
const dockerProvider = new DockerProvider(logger, Docker, encryptionService);
const dockerProvider = new DockerProvider(logger, Docker, encryptionService, httpProvider);
const portProvider = new PortProvider(net);
const gameProvider = new GameProvider(logger, GameDig);
const grpcProvider = new GrpcProvider(grpc, protoLoader);
+133 -8
View File
@@ -1,6 +1,7 @@
import https from "node:https";
import { IStatusProvider } from "@/service/network/IStatusProvider.js";
import { HttpProvider } from "@/service/network/HttpProvider.js";
import { CheckContext, DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import {
@@ -23,10 +24,11 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { ILogger } from "@/utils/logger.js";
import { AppError } from "@/utils/AppError.js";
import Dockerode from "dockerode";
import { z } from "zod";
import { timeRequest } from "@/service/network/utils.js";
import { NETWORK_ERROR } from "@/types/network.js";
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
import { DOCKER_TLS_URL, isCaptureDockerUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
import { splitCertificateBundle } from "@/utils/pem.js";
type DockerodeType = typeof Dockerode;
@@ -40,6 +42,54 @@ const DOCKER_LOG_MAX_LINE_BYTES = 4096;
const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
const DOCKER_TLS_DEFAULT_PORT = 2376;
const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]);
const captureDockerResponseSchema = z.object({
data: z
.array(
z.object({
container_id: z.string(),
container_name: z.string(),
status: z.string(),
health: z
.object({
healthy: z.boolean(),
source: z.string(),
})
.nullish(),
running: z.boolean(),
base_image: z.string(),
exposed_ports: z
.array(
z.object({
port: z.string(),
protocol: z.string(),
})
)
.optional(),
started_at: z.number().optional(),
stats: z
.object({
cpu_percent: z.number(),
memory_usage: z.number(),
memory_limit: z.number(),
memory_percent: z.number(),
})
.nullish(),
})
)
.nullable(),
errors: z
.array(
z.object({
metric: z.array(z.string()),
err: z.string(),
})
)
.nullish(),
});
type CaptureContainer = NonNullable<z.infer<typeof captureDockerResponseSchema>["data"]>[number];
export interface DockerError extends Error {
statusCode?: number;
@@ -53,7 +103,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
constructor(
private logger: ILogger,
private DockerLib: DockerodeType,
private encryptionService: IEncryptionService
private encryptionService: IEncryptionService,
private httpProvider: HttpProvider
) {}
supports(type: MonitorType): boolean {
@@ -137,6 +188,83 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none";
}
private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary {
return {
total: containers.length,
running: containers.filter((container) => container.state === "running").length,
stopped: containers.filter((container) => container.state === "exited" || container.state === "dead").length,
unhealthy: containers.filter((container) => container.health === "unhealthy").length,
};
}
private toCaptureContainer(container: CaptureContainer): DockerContainerInfo {
const startedAt = container.started_at && container.started_at > 0 ? new Date(container.started_at * 1000).toISOString() : undefined;
const stats = container.stats
? {
cpuPct: container.stats.cpu_percent / 100,
memoryUsedBytes: container.stats.memory_usage,
memoryLimitBytes: container.stats.memory_limit,
memoryPct: container.stats.memory_percent / 100,
}
: {};
return {
id: container.container_id,
name: container.container_name || container.container_id.slice(0, 12),
image: container.base_image,
state: this.toContainerState(container.status),
status: container.status,
// Capture derives health from state when no Docker healthcheck exists.
// Keep Checkmate's `none` value in that case so summaries retain their existing semantics.
health: container.health?.source === "container_health_check" ? (container.health.healthy ? "healthy" : "unhealthy") : "none",
startedAt,
ports: (container.exposed_ports ?? []).flatMap((port) => {
const privatePort = Number(port.port);
return Number.isInteger(privatePort) ? [{ privatePort, protocol: this.toPortProtocol(port.protocol) }] : [];
}),
...stats,
};
}
private handleCapture = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
const captureUrl = new URL(monitor.url.trim());
captureUrl.searchParams.set("all", "true");
const response = await this.httpProvider.handle<unknown>({ ...monitor, url: captureUrl.toString() }, ctx);
if (!response.status) return this.failed(monitor, response.code, response.message, response.responseTime ?? 0);
const parsed = captureDockerResponseSchema.safeParse(response.payload);
if (!parsed.success) {
this.logger.warn({
message: "Capture returned an invalid Docker metrics payload",
service: SERVICE_NAME,
method: "handleCapture",
});
return this.failed(monitor, NETWORK_ERROR, "Capture returned an invalid Docker metrics payload", response.responseTime ?? 0);
}
const errors = parsed.data.errors ?? [];
const fatalError = errors.find((error) => error.metric.some((metric) => CAPTURE_FATAL_ERROR_METRICS.has(metric)));
if (fatalError) return this.failed(monitor, NETWORK_ERROR, fatalError.err, response.responseTime ?? 0);
if (errors.length > 0) {
this.logger.warn({
message: `Capture returned ${errors.length} partial Docker collection error(s)`,
service: SERVICE_NAME,
method: "handleCapture",
});
}
const containers = (parsed.data.data ?? []).map((container) => this.toCaptureContainer(container));
return {
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
status: true,
code: response.code,
message: "Docker host is reachable through Capture",
responseTime: response.responseTime,
payload: { containers, summary: this.toContainerSummary(containers) },
};
};
private computeCpuPct(stats: Dockerode.ContainerStats): number {
const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage;
const systemDelta = stats.cpu_stats.system_cpu_usage - (stats.precpu_stats.system_cpu_usage ?? 0);
@@ -328,6 +456,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
handle = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
try {
if (isCaptureDockerUrl(monitor.url)) return await this.handleCapture(monitor, ctx);
const credentials = this.resolveTlsCredentials(monitor, ctx);
if (credentials && !credentials.ok) return this.failed(monitor, NETWORK_ERROR, credentials.message, 0);
@@ -350,12 +480,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
const containers = await this.mapWithConcurrency(summaries, STATS_CONCURRENCY, (s) => this.toContainerInfo(docker, s));
const logs = monitor.dockerLogsEnabled ? await this.collectLogs(docker, summaries) : undefined;
const summary: DockerContainerSummary = {
total: containers.length,
running: containers.filter((c) => c.state === "running").length,
stopped: containers.filter((c) => c.state === "exited" || c.state === "dead").length,
unhealthy: containers.filter((c) => c.health === "unhealthy").length,
};
const summary = this.toContainerSummary(containers);
return {
monitorId: monitor.id,
teamId: monitor.teamId,
+10
View File
@@ -3,3 +3,13 @@ export const DOCKER_SOCKET_URL = /^(?:unix:\/\/\/\S+|\/\S+)$/;
export const isDockerTlsUrl = (url: string | undefined): boolean => DOCKER_TLS_URL.test(url?.trim() ?? "");
export const isDockerSocketUrl = (url: string | undefined): boolean => DOCKER_SOCKET_URL.test(url?.trim() ?? "");
export const isCaptureDockerUrl = (url: string | undefined): boolean => {
try {
const parsed = new URL(url?.trim() ?? "");
const pathname = parsed.pathname.replace(/\/+$/, "");
return (parsed.protocol === "http:" || parsed.protocol === "https:") && pathname.endsWith("/metrics/docker");
} catch {
return false;
}
};
@@ -76,6 +76,33 @@ const makeStats = (overrides?: Record<string, any>) => ({
...overrides,
});
const makeCaptureContainer = (overrides?: Record<string, any>) => ({
container_id: "capture-container-id",
container_name: "capture-container",
status: "running",
health: { healthy: true, source: "container_health_check" },
running: true,
base_image: "nginx:latest",
exposed_ports: [{ port: "80", protocol: "tcp" }],
started_at: 1_788_000_000,
stats: {
cpu_percent: 80,
memory_usage: 256,
memory_limit: 1024,
memory_percent: 25,
},
...overrides,
});
const makeCaptureResponse = (overrides?: Record<string, any>) => ({
status: true,
code: 200,
message: "OK",
responseTime: 42,
payload: { data: [makeCaptureContainer()], errors: [] },
...overrides,
});
const makeLogFrame = (streamType: number, text: string): Buffer => {
const payload = Buffer.from(text);
const header = Buffer.alloc(8);
@@ -92,6 +119,7 @@ const setup = (
inspect: any;
stats: any;
logs: any;
httpHandle: any;
encryptionService: IEncryptionService;
}> = {}
) => {
@@ -107,12 +135,14 @@ const setup = (
const DockerLib = jest.fn().mockReturnValue(instance) as any;
const logger = createMockLogger();
const encryptionService = overrides.encryptionService ?? makeEncryptionService();
const provider = new DockerProvider(logger as any, DockerLib, encryptionService);
const httpHandle = overrides.httpHandle ?? jest.fn();
const provider = new DockerProvider(logger as any, DockerLib, encryptionService, { handle: httpHandle } as any);
return {
provider,
logger,
DockerLib,
encryptionService,
httpHandle,
ping: instance.ping,
listContainers: instance.listContainers,
getContainer,
@@ -162,6 +192,105 @@ describe("DockerProvider", () => {
});
});
// ── Capture urls ─────────────────────────────────────────────────────
describe("Capture urls", () => {
it("requests all containers and normalizes Capture metrics", async () => {
const httpHandle = jest.fn().mockResolvedValue(makeCaptureResponse());
const { provider, DockerLib } = setup({ httpHandle });
const monitor = makeMonitor({
url: "https://capture.example.com/api/v1/metrics/docker?site=west&all=false",
secret: "capture-secret",
});
const result = await provider.handle(monitor);
expect(httpHandle).toHaveBeenCalledWith(
expect.objectContaining({
url: "https://capture.example.com/api/v1/metrics/docker?site=west&all=true",
secret: "capture-secret",
}),
undefined
);
expect(DockerLib).not.toHaveBeenCalled();
expect(result).toMatchObject({
status: true,
payload: { summary: { total: 1, running: 1, stopped: 0, unhealthy: 0 } },
});
expect(result.payload?.containers[0]).toMatchObject({
id: "capture-container-id",
state: "running",
health: "healthy",
cpuPct: 0.8,
memoryUsedBytes: 256,
memoryPct: 0.25,
startedAt: new Date(1_788_000_000 * 1000).toISOString(),
ports: [{ privatePort: 80, protocol: "tcp" }],
});
});
it("keeps state-derived Capture health separate from Docker healthchecks", async () => {
const response = makeCaptureResponse({
payload: {
data: [makeCaptureContainer({ health: { healthy: false, source: "state_based_health_check" } })],
errors: [],
},
});
const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) });
const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" }));
expect(result.payload?.containers[0]?.health).toBe("none");
expect(result.payload?.summary.unhealthy).toBe(0);
});
it("keeps a partial Capture response up", async () => {
const response = makeCaptureResponse({
code: 207,
payload: {
data: [makeCaptureContainer()],
errors: [{ metric: ["docker.container.stats"], err: "stats unavailable" }],
},
});
const { provider, logger } = setup({ httpHandle: jest.fn().mockResolvedValue(response) });
const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" }));
expect(result.status).toBe(true);
expect(result.code).toBe(207);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Capture returned 1 partial Docker collection error(s)" }));
});
it("marks fatal Capture Docker errors down", async () => {
const response = makeCaptureResponse({
code: 207,
payload: { data: null, errors: [{ metric: ["docker.client"], err: "Docker daemon unavailable" }] },
});
const { provider } = setup({ httpHandle: jest.fn().mockResolvedValue(response) });
const result = await provider.handle(makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" }));
expect(result).toMatchObject({ status: false, code: NETWORK_ERROR, message: "Docker daemon unavailable", payload: null });
});
it("accepts an empty Capture host and rejects malformed payloads", async () => {
const emptyHandle = jest.fn().mockResolvedValue(makeCaptureResponse({ payload: { data: [], errors: null } }));
const { provider } = setup({ httpHandle: emptyHandle });
const monitor = makeMonitor({ url: "http://capture:59232/api/v1/metrics/docker", secret: "secret" });
const emptyResult = await provider.handle(monitor);
expect(emptyResult).toMatchObject({ status: true, payload: { containers: [], summary: { total: 0 } } });
emptyHandle.mockResolvedValueOnce(makeCaptureResponse({ payload: { data: "not-an-array" } }));
const malformedResult = await provider.handle(monitor);
expect(malformedResult).toMatchObject({
status: false,
code: NETWORK_ERROR,
message: "Capture returned an invalid Docker metrics payload",
});
});
});
// ── TLS host urls ────────────────────────────────────────────────────
describe("tls host urls", () => {
+17 -1
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from "@jest/globals";
import { DOCKER_TLS_URL, isDockerSocketUrl, isDockerTlsUrl } from "../../../src/utils/dockerHost.ts";
import { DOCKER_TLS_URL, isCaptureDockerUrl, isDockerSocketUrl, isDockerTlsUrl } from "../../../src/utils/dockerHost.ts";
describe("isDockerTlsUrl", () => {
it.each(["tcp://host", "tcp://host:2376", "https://host", "https://host:2377", "tcp://host/", "tcp://10.0.0.1:2376", " tcp://host "])(
@@ -31,6 +31,22 @@ describe("isDockerTlsUrl", () => {
});
});
describe("isCaptureDockerUrl", () => {
it.each(["http://host:59232/api/v1/metrics/docker", "https://example.com/prefix/metrics/docker/?all=false"])("accepts %s", (url) => {
expect(isCaptureDockerUrl(url)).toBe(true);
});
it.each([
"https://host:2376",
"tcp://host:2376/api/v1/metrics/docker",
"https://example.com/api/v1/metrics/docker/containers",
"https://example.com/?path=/metrics/docker",
undefined,
])("rejects %s", (url) => {
expect(isCaptureDockerUrl(url)).toBe(false);
});
});
describe("isDockerSocketUrl", () => {
it.each(["unix:///var/run/docker.sock", "/var/run/docker.sock", " /run/docker.sock "])("accepts %s", (url) => {
expect(isDockerSocketUrl(url)).toBe(true);
@@ -629,6 +629,18 @@ describe("monitorValidation — Docker host url", () => {
}
});
it("accepts a Capture Docker metric endpoint with a secret", () => {
const url = "http://capture:59232/api/v1/metrics/docker";
const parsed = createMonitorBodyValidation.parse({ ...baseDockerBody, url, secret: "capture-secret" });
expect(parsed.url).toBe(url);
});
it("requires a secret for a Capture Docker endpoint", () => {
expect(() => createMonitorBodyValidation.parse({ ...baseDockerBody, url: "https://capture.example.com/api/v1/metrics/docker" })).toThrow(
"Capture API secret is required"
);
});
it("rejects container names and unsupported engine urls", () => {
for (const badUrl of [
"my-container",
@@ -671,6 +683,13 @@ describe("monitorValidation — Docker host url", () => {
expect(parsed.monitors[0].url).toBe("unix:///var/run/docker.sock");
});
it("accepts an imported Capture Docker monitor with a secret", () => {
const parsed = importMonitorsBodyValidation.parse({
monitors: [{ ...baseDockerBody, url: "http://capture:59232/api/v1/metrics/docker", secret: "capture-secret" }],
});
expect(parsed.monitors[0].secret).toBe("capture-secret");
});
it("rejects an imported docker monitor with a container-name url", () => {
expect(() => importMonitorsBodyValidation.parse({ monitors: [{ ...baseDockerBody, url: "my-container" }] })).toThrow();
});