Merge pull request #3973 from fabiangigler/feat/capture-docker-monitors
Support Capture-backed Docker monitors
This commit is contained in:
15 files changed
+380
-34
No files matched your search
@@ -1,7 +1,8 @@
|
||||
import https from "node:https";
|
||||
|
||||
import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import { CheckContext, DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { HttpProvider } from "@/service/network/HttpProvider.js";
|
||||
import { CaptureDockerContainer, CaptureDockerStatusPayload, CheckContext, DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
|
||||
import {
|
||||
DOCKER_LOG_TAIL_LINES,
|
||||
@@ -26,7 +27,7 @@ import Dockerode from "dockerode";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
|
||||
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { DOCKER_TLS_URL, isCaptureDockerUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { splitCertificateBundle } from "@/utils/pem.js";
|
||||
|
||||
type DockerodeType = typeof Dockerode;
|
||||
@@ -40,6 +41,7 @@ const DOCKER_LOG_MAX_LINE_BYTES = 4096;
|
||||
const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
|
||||
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
|
||||
const DOCKER_TLS_DEFAULT_PORT = 2376;
|
||||
const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]);
|
||||
|
||||
export interface DockerError extends Error {
|
||||
statusCode?: number;
|
||||
@@ -53,7 +55,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
constructor(
|
||||
private logger: ILogger,
|
||||
private DockerLib: DockerodeType,
|
||||
private encryptionService: IEncryptionService
|
||||
private encryptionService: IEncryptionService,
|
||||
private httpProvider: HttpProvider
|
||||
) {}
|
||||
|
||||
supports(type: MonitorType): boolean {
|
||||
@@ -137,6 +140,83 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none";
|
||||
}
|
||||
|
||||
private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary {
|
||||
return {
|
||||
total: containers.length,
|
||||
running: containers.filter((container) => container.state === "running").length,
|
||||
stopped: containers.filter((container) => container.state === "exited" || container.state === "dead").length,
|
||||
unhealthy: containers.filter((container) => container.health === "unhealthy").length,
|
||||
};
|
||||
}
|
||||
|
||||
private toCaptureContainer(container: CaptureDockerContainer): DockerContainerInfo {
|
||||
const startedAt = container.started_at && container.started_at > 0 ? new Date(container.started_at * 1000).toISOString() : undefined;
|
||||
const stats = container.stats
|
||||
? {
|
||||
cpuPct: container.stats.cpu_percent / 100,
|
||||
memoryUsedBytes: container.stats.memory_usage,
|
||||
memoryLimitBytes: container.stats.memory_limit,
|
||||
memoryPct: container.stats.memory_percent / 100,
|
||||
}
|
||||
: {};
|
||||
return {
|
||||
id: container.container_id,
|
||||
name: container.container_name || container.container_id.slice(0, 12),
|
||||
image: container.base_image,
|
||||
state: this.toContainerState(container.status),
|
||||
status: container.status,
|
||||
// Capture derives health from state when no Docker healthcheck exists.
|
||||
// Keep Checkmate's `none` value in that case so summaries retain their existing semantics.
|
||||
health: container.health?.source === "container_health_check" ? (container.health.healthy ? "healthy" : "unhealthy") : "none",
|
||||
startedAt,
|
||||
ports: (container.exposed_ports ?? []).flatMap((port) => {
|
||||
const privatePort = Number(port.port);
|
||||
return Number.isInteger(privatePort) ? [{ privatePort, protocol: this.toPortProtocol(port.protocol) }] : [];
|
||||
}),
|
||||
...stats,
|
||||
};
|
||||
}
|
||||
|
||||
private handleCapture = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
|
||||
const captureUrl = new URL(monitor.url.trim());
|
||||
captureUrl.searchParams.set("all", "true");
|
||||
const response = await this.httpProvider.handle<CaptureDockerStatusPayload>({ ...monitor, url: captureUrl.toString() }, ctx);
|
||||
if (!response.status) return this.failed(monitor, response.code, response.message, response.responseTime ?? 0);
|
||||
|
||||
const payload = response.payload;
|
||||
if (!payload || typeof payload === "string" || (!Array.isArray(payload.data) && payload.data !== null)) {
|
||||
this.logger.warn({
|
||||
message: "Capture returned an invalid Docker metrics payload",
|
||||
service: SERVICE_NAME,
|
||||
method: "handleCapture",
|
||||
});
|
||||
return this.failed(monitor, NETWORK_ERROR, "Capture returned an invalid Docker metrics payload", response.responseTime ?? 0);
|
||||
}
|
||||
|
||||
const errors = payload.errors ?? [];
|
||||
const fatalError = errors.find((error) => error.metric.some((metric) => CAPTURE_FATAL_ERROR_METRICS.has(metric)));
|
||||
if (fatalError) return this.failed(monitor, NETWORK_ERROR, fatalError.err, response.responseTime ?? 0);
|
||||
if (errors.length > 0) {
|
||||
this.logger.warn({
|
||||
message: `Capture returned ${errors.length} partial Docker collection error(s)`,
|
||||
service: SERVICE_NAME,
|
||||
method: "handleCapture",
|
||||
});
|
||||
}
|
||||
|
||||
const containers = (payload.data ?? []).map((container) => this.toCaptureContainer(container));
|
||||
return {
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: response.code,
|
||||
message: "Docker host is reachable through Capture",
|
||||
responseTime: response.responseTime,
|
||||
payload: { containers, summary: this.toContainerSummary(containers) },
|
||||
};
|
||||
};
|
||||
|
||||
private computeCpuPct(stats: Dockerode.ContainerStats): number {
|
||||
const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage;
|
||||
const systemDelta = stats.cpu_stats.system_cpu_usage - (stats.precpu_stats.system_cpu_usage ?? 0);
|
||||
@@ -335,6 +415,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
|
||||
handle = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
|
||||
try {
|
||||
if (isCaptureDockerUrl(monitor.url)) return await this.handleCapture(monitor, ctx);
|
||||
|
||||
const credentials = this.resolveTlsCredentials(monitor, ctx);
|
||||
if (credentials && !credentials.ok) return this.failed(monitor, NETWORK_ERROR, credentials.message, 0);
|
||||
|
||||
@@ -361,12 +443,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
const containers = await this.mapWithConcurrency(summaries, STATS_CONCURRENCY, (s) => this.toContainerInfo(docker, s));
|
||||
const logs = monitor.dockerLogsEnabled ? await this.collectLogs(docker, summaries) : undefined;
|
||||
|
||||
const summary: DockerContainerSummary = {
|
||||
total: containers.length,
|
||||
running: containers.filter((c) => c.state === "running").length,
|
||||
stopped: containers.filter((c) => c.state === "exited" || c.state === "dead").length,
|
||||
unhealthy: containers.filter((c) => c.health === "unhealthy").length,
|
||||
};
|
||||
const summary = this.toContainerSummary(containers);
|
||||
return {
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
|
||||
Reference in new issue
Block a user