Merge pull request #3920 from bluewave-labs/feat/docker-container-logs

feat: docker container logs
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-09-04 14:42:27 -07:00
commit 7fcdcc0568
49 files changed
+1710 -215

No files matched your search

+2 -1
View File
@@ -12,7 +12,8 @@
"lint-strict": "eslint . --ext js,jsx,ts,tsx --report-unused-disable-directives --max-warnings 0",
"preview": "vite preview",
"format": "prettier --write .",
"format-check": "prettier --check ."
"format-check": "prettier --check .",
"compile": "tsc -b"
},
"lint-staged": {
"**/*": "prettier --write --ignore-unknown"
@@ -1,10 +1,12 @@
import Box, { type BoxProps } from "@mui/material/Box";
import { useTheme } from "@mui/material/styles";
import { forwardRef } from "react";
export const BaseBox = ({ sx, ...rest }: BoxProps) => {
export const BaseBox = forwardRef<HTMLDivElement, BoxProps>(({ sx, ...rest }, ref) => {
const theme = useTheme();
return (
<Box
ref={ref}
{...rest}
sx={{
backgroundColor: theme.palette.background.paper,
@@ -16,4 +18,6 @@ export const BaseBox = ({ sx, ...rest }: BoxProps) => {
}}
/>
);
};
});
BaseBox.displayName = "BaseBox";
+1 -1
View File
@@ -6,7 +6,7 @@ import { useState } from "react";
import { useToast } from "@/Hooks/UseToast";
import { logger } from "@/Utils/logger";
interface ApiResponse<T> {
export interface ApiResponse<T> {
success: boolean;
msg: string;
data: T;
+23 -20
View File
@@ -1,27 +1,30 @@
import { toast, type ToastOptions } from "react-toastify";
import { useMemo } from "react";
export const useToast = () => {
const showToast = (message: string, options?: ToastOptions) => {
toast.dismiss();
const baseStyle: React.CSSProperties = {
whiteSpace: "nowrap",
overflow: "hidden",
textOverflow: "ellipsis",
return useMemo(() => {
const showToast = (message: string, options?: ToastOptions) => {
toast.dismiss();
const baseStyle: React.CSSProperties = {
whiteSpace: "nowrap",
overflow: "hidden",
textOverflow: "ellipsis",
};
toast(message, {
...options,
style: { ...baseStyle, ...(options?.style || {}) },
});
};
toast(message, {
...options,
style: { ...baseStyle, ...(options?.style || {}) },
});
};
const toastSuccess = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "success" });
const toastError = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "error" });
const toastInfo = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "info" });
const toastWarning = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "warning" });
const toastSuccess = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "success" });
const toastError = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "error" });
const toastInfo = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "info" });
const toastWarning = (msg: string, opts?: ToastOptions) =>
showToast(msg, { ...opts, type: "warning" });
return { showToast, toastSuccess, toastError, toastInfo, toastWarning };
return { showToast, toastSuccess, toastError, toastInfo, toastWarning };
}, []);
};
+1
View File
@@ -77,6 +77,7 @@ export const getMonitorDefaults = (
...base,
type: "docker",
url: data?.url || "",
dockerLogsEnabled: data?.dockerLogsEnabled ?? false,
};
break;
case "game":
+21 -3
View File
@@ -118,8 +118,8 @@ const getGeneralSettingsConfig = (
showDnsRecordType: false,
},
docker: {
urlLabel: t("pages.createMonitor.form.general.option.container.label"),
urlPlaceholder: t("pages.createMonitor.form.general.option.container.placeholder"),
urlLabel: t("pages.createMonitor.form.general.option.dockerHost.label"),
urlPlaceholder: t("pages.createMonitor.form.general.option.dockerHost.placeholder"),
namePlaceholder: t("pages.createMonitor.form.general.option.name.placeholder"),
showUrl: true,
showProxy: false,
@@ -255,6 +255,7 @@ const CreateMonitorPage = () => {
const firstSegment = pathSegments[0];
if (firstSegment === "pagespeed") return "pagespeed";
if (firstSegment === "infrastructure") return "hardware";
if (firstSegment === "docker") return "docker";
return "uptime";
}, [location.pathname]);
@@ -264,7 +265,9 @@ const CreateMonitorPage = () => {
? "pagespeed"
: pageType === "hardware"
? "hardware"
: "http";
: pageType === "docker"
? "docker"
: "http";
const { data: existingMonitor, refetch: refetchMonitor } = useGet<Monitor>(
isEditMode ? `/monitors/${monitorId}` : null
@@ -373,6 +376,8 @@ const CreateMonitorPage = () => {
navigate("/pagespeed");
} else if (pageType === "hardware") {
navigate("/infrastructure");
} else if (pageType === "docker") {
navigate("/docker");
} else {
navigate("/uptime");
}
@@ -823,6 +828,19 @@ const CreateMonitorPage = () => {
/>
)}
{showStep(1) && watchedType === "docker" && (
<ConfigBox
title={t("pages.createMonitor.form.dockerLogs.title")}
subtitle={t("pages.createMonitor.form.dockerLogs.description")}
rightContent={
<FormSwitchField
name="dockerLogsEnabled"
label={t("pages.createMonitor.form.dockerLogs.option.enabled.label")}
/>
}
/>
)}
{showStep(2) &&
(watchedType === "http" ||
watchedType === "grpc" ||
@@ -0,0 +1,69 @@
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { Dot } from "@/Components/design-elements";
// Hooks
import { useEffect, useState } from "react";
import { useTheme } from "@mui/material";
import { useTranslation } from "react-i18next";
// Utils
import { LAYOUT, SPACING } from "@/Utils/Theme/constants";
import { MS_PER_SECOND, MS_PER_MINUTE } from "@/Utils/TimeUtils";
interface HeaderLogsProps {
lines: number;
loading: boolean;
lastUpdatedAt: number | null;
}
export const HeaderLogs = ({ lines, loading, lastUpdatedAt }: HeaderLogsProps) => {
const theme = useTheme();
const { t } = useTranslation();
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (lastUpdatedAt === null) return;
setNow(Date.now());
const id = window.setInterval(() => setNow(Date.now()), MS_PER_SECOND);
return () => window.clearInterval(id);
}, [lastUpdatedAt]);
const renderStatus = () => {
if (loading) return t("pages.docker.container.logs.loading");
if (lastUpdatedAt === null) return null;
const ageMs = Math.max(0, now - lastUpdatedAt);
return ageMs < MS_PER_MINUTE
? t("pages.docker.container.logs.updatedSecondsAgo", {
count: Math.floor(ageMs / MS_PER_SECOND),
})
: t("pages.docker.container.logs.updatedMinutesAgo", {
count: Math.floor(ageMs / MS_PER_MINUTE),
});
};
return (
<Stack
direction="row"
alignItems="center"
gap={LAYOUT.MD}
>
<Typography>
{t("pages.docker.container.logs.recentLines", { count: lines })}
</Typography>
<Stack
direction="row"
alignItems="center"
gap={SPACING.SM}
>
{lastUpdatedAt !== null && !loading && (
<Dot
color={theme.palette.success.main}
size="md"
/>
)}
<Typography color={theme.palette.text.secondary}>{renderStatus()}</Typography>
</Stack>
</Stack>
);
};
@@ -0,0 +1,78 @@
import Grid from "@mui/material/Grid";
import Typography from "@mui/material/Typography";
import Box from "@mui/material/Box";
import Stack from "@mui/material/Stack";
import Divider from "@mui/material/Divider";
import { Fragment } from "react";
// Hooks
import { useTheme } from "@mui/material";
import { useSelector } from "react-redux";
import { useTranslation } from "react-i18next";
// Utils
import type { DockerLogRow } from "@/Utils/MonitorUtils";
import { LAYOUT, SPACING } from "@/Utils/Theme/constants";
import { formatDateWithTz } from "@/Utils/TimeUtils";
import type { RootState } from "@/store";
export const RowLog = ({ row }: { row: DockerLogRow }) => {
const theme = useTheme();
const uiTimezone = useSelector((state: RootState) => state.ui.timezone);
const { t } = useTranslation();
return (
<Fragment>
<Grid
container
spacing={{ xs: 0, md: LAYOUT.MD }}
py={{ xs: LAYOUT.LG, md: 0 }}
>
<Grid size={{ xs: 12, md: "auto" }}>
<Typography
fontFamily="monospace"
color={theme.palette.text.secondary}
>
{formatDateWithTz(row.ts, "MMM D HH:mm:ss", uiTimezone)}
</Typography>
</Grid>
<Grid size={{ xs: 12, md: "grow" }}>
<Typography
fontFamily={"monospace"}
whiteSpace={{ xs: "pre-wrap", md: "nowrap" }}
sx={{
overflowWrap: { xs: "anywhere", md: "normal" },
}}
>
<Box
component={"span"}
color={
row.stream === "stderr"
? theme.palette.error.main
: theme.palette.success.main
}
>
{row.stream}
</Box>
: {row.text}
</Typography>
</Grid>
</Grid>
{row.gapBefore ? (
<Stack
width="100%"
direction="row"
alignItems={"center"}
py={SPACING.MD}
>
<Typography pr={SPACING.MD}>
{t("pages.docker.container.logs.skipped")}
</Typography>
<Divider sx={{ flex: 1 }} />
</Stack>
) : (
<Divider sx={{ display: { xs: "block", md: "none" } }} />
)}
</Fragment>
);
};
@@ -1,3 +1,241 @@
export const TabLogs = () => {
return "This is the logs tab";
// Components
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { BaseBox, EmptyState } from "@/Components/design-elements";
import { Button } from "@/Components/inputs";
import { HeaderLogs } from "@/Pages/Docker/ContainerDetails/Components/HeaderLogs";
import { RowLog } from "@/Pages/Docker/ContainerDetails/Components/RowLog";
// Types
import type { DockerContainerLogsResponse } from "@/Types/Monitor";
import type { DockerLog } from "@/Types/Check";
import type { ApiResponse } from "@/Hooks/UseApi";
// Hooks
import {
useState,
useEffect,
useLayoutEffect,
useMemo,
useRef,
useCallback,
} from "react";
import { useTheme } from "@mui/material";
import { useTranslation } from "react-i18next";
import { useToast } from "@/Hooks/UseToast";
// Utils
import { get } from "@/Utils/ApiClient";
import { LAYOUT } from "@/Utils/Theme/constants";
import { flattenDockerLogs } from "@/Utils/MonitorUtils";
interface TabLogsProps {
monitorId?: string;
containerName?: string;
enabled: boolean;
}
const SCROLL_PIN_THRESHOLD_PX = 8;
const TAIL_POLL_MS = 10_000;
const isScrolledToBottom = (el: HTMLElement) =>
el.scrollHeight - el.scrollTop - el.clientHeight <= SCROLL_PIN_THRESHOLD_PX;
const fetchLogs = async (url: string, params: { before?: string; after?: string }) => {
const res = await get<ApiResponse<DockerContainerLogsResponse>>(url, { params });
return res.data.data;
};
export const TabLogs = ({ monitorId, containerName, enabled }: TabLogsProps) => {
const theme = useTheme();
const { t } = useTranslation();
const { toastError } = useToast();
// logs is newest-first, matching the API and what flattenDockerLogs expects.
const [logs, setLogs] = useState<DockerLog[]>([]);
const [olderExhausted, setOlderExhausted] = useState(false);
const [loadingInitial, setLoadingInitial] = useState(false);
const [loadingOlder, setLoadingOlder] = useState(false);
const [lastUpdatedAt, setLastUpdatedAt] = useState<number | null>(null);
const logsUrl =
enabled && monitorId && containerName
? `/monitors/docker/details/${encodeURIComponent(monitorId)}/containers/${encodeURIComponent(containerName)}/logs`
: null;
const newestCursor = logs[0]?.checkedAt;
const oldestCursor = logs[logs.length - 1]?.checkedAt;
// Initial page. Resets everything when the container changes.
useEffect(() => {
setLogs([]);
setOlderExhausted(false);
if (!logsUrl) return;
let cancelled = false;
setLoadingInitial(true);
fetchLogs(logsUrl, {})
.then((page) => {
if (cancelled) return;
setLogs(page.logs);
setOlderExhausted(page.nextCursor === null);
setLastUpdatedAt(Date.now());
})
.catch(() => {
if (cancelled) return;
toastError(t("pages.docker.container.logs.loadError"));
})
.finally(() => {
if (!cancelled) setLoadingInitial(false);
});
return () => {
cancelled = true;
};
}, [logsUrl, t, toastError]);
const newestCursorRef = useRef(newestCursor);
newestCursorRef.current = newestCursor;
useEffect(() => {
if (!logsUrl) return;
let inFlight = false;
const tick = async () => {
if (inFlight || document.hidden) return;
inFlight = true;
try {
const cursor = newestCursorRef.current;
const page = await fetchLogs(logsUrl, cursor ? { after: cursor } : {});
if (page.logs.length > 0) setLogs((prev) => [...page.logs, ...prev]);
setLastUpdatedAt(Date.now());
} catch (err: unknown) {
console.error(err);
} finally {
inFlight = false;
}
};
const id = window.setInterval(tick, TAIL_POLL_MS);
document.addEventListener("visibilitychange", tick);
return () => {
window.clearInterval(id);
document.removeEventListener("visibilitychange", tick);
};
}, [logsUrl]);
const rows = useMemo(() => flattenDockerLogs(logs), [logs]);
const logBoxRef = useRef<HTMLDivElement>(null);
const pinnedToBottom = useRef(true);
const prependAnchor = useRef<{ scrollHeight: number; scrollTop: number } | null>(null);
// Always pin to bottom when container/monitor changes
useLayoutEffect(() => {
pinnedToBottom.current = true;
}, [monitorId, containerName]);
// Rows changed: restore position after a prepend, otherwise honour the pin
useLayoutEffect(() => {
const el = logBoxRef.current;
if (!el) return;
const anchor = prependAnchor.current;
if (anchor) {
prependAnchor.current = null;
el.scrollTop = el.scrollHeight - anchor.scrollHeight + anchor.scrollTop;
return;
}
if (pinnedToBottom.current) el.scrollTop = el.scrollHeight;
}, [rows]);
const handleScroll = useCallback((event: React.UIEvent<HTMLDivElement>) => {
pinnedToBottom.current = isScrolledToBottom(event.currentTarget);
}, []);
const loadOlder = useCallback(async () => {
if (!logsUrl || !oldestCursor || loadingOlder) return;
setLoadingOlder(true);
try {
const page = await fetchLogs(logsUrl, { before: oldestCursor });
const el = logBoxRef.current;
if (el)
prependAnchor.current = {
scrollHeight: el.scrollHeight,
scrollTop: el.scrollTop,
};
setLogs((prev) => [...prev, ...page.logs]);
setOlderExhausted(page.nextCursor === null);
} catch {
toastError(t("pages.docker.container.logs.loadOlderError"));
} finally {
setLoadingOlder(false);
}
}, [logsUrl, oldestCursor, loadingOlder, t, toastError]);
if (!enabled) {
return (
<EmptyState
title={t("pages.docker.container.logs.disabled.title")}
description={t("pages.docker.container.logs.disabled.description")}
actionText={t("pages.docker.container.logs.disabled.action")}
actionTo={`/docker/configure/${monitorId}`}
/>
);
}
if (lastUpdatedAt !== null && rows.length === 0) {
return (
<EmptyState
title={t("pages.docker.container.logs.empty.title")}
description={t("pages.docker.container.logs.empty.description")}
/>
);
}
return (
<Stack gap={LAYOUT.MD}>
<HeaderLogs
lines={rows.length}
loading={loadingInitial}
lastUpdatedAt={lastUpdatedAt}
/>
<BaseBox
ref={logBoxRef}
onScroll={handleScroll}
minHeight={240}
maxHeight="50vh"
overflow="auto"
padding={LAYOUT.MD}
sx={{ overflowAnchor: "none" }}
>
{rows.length > 0 && (
<Stack
direction="row"
justifyContent="center"
pb={LAYOUT.MD}
>
{olderExhausted ? (
<Typography color={theme.palette.text.secondary}>
{t("pages.docker.container.logs.noOlder")}
</Typography>
) : (
<Button
variant="contained"
disabled={loadingOlder}
onClick={loadOlder}
>
{loadingOlder
? t("pages.docker.container.logs.loadingOlder")
: t("pages.docker.container.logs.loadOlder")}
</Button>
)}
</Stack>
)}
{rows.map((row) => {
return (
<RowLog
key={row.key}
row={row}
/>
);
})}
</BaseBox>
</Stack>
);
};
@@ -21,7 +21,7 @@ const TAB_MAP = {
logs: 1,
} as const;
const DockerDetailsPage = ({ open = "overview" }: DockerContainerDetailsProps) => {
const DockerDetailsPage = ({ open = "logs" }: DockerContainerDetailsProps) => {
const [activeTab, setActiveTab] = useState<number>(TAB_MAP[open]);
const [dateRange, setDateRange] = useState<DateRange>("recent");
const { monitorId, containerName } = useParams<{
@@ -70,7 +70,13 @@ const DockerDetailsPage = ({ open = "overview" }: DockerContainerDetailsProps) =
<Tab label={t("pages.docker.container.tabs.logs")} />
</Tabs>
{activeTab === 0 && <TabOverview stats={stats} />}
{activeTab === 1 && <TabLogs />}
{activeTab === 1 && (
<TabLogs
monitorId={monitorId}
containerName={containerName}
enabled={monitor.dockerLogsEnabled ?? false}
/>
)}
</BasePage>
);
};
+25
View File
@@ -161,6 +161,31 @@ export interface DockerContainerMount {
rw: boolean;
}
export const DockerLogStreams = ["stdout", "stderr"] as const;
export type DockerLogStream = (typeof DockerLogStreams)[number];
export interface DockerLogLine {
ts: string;
stream: DockerLogStream;
text: string;
}
export interface DockerLog {
id: string;
metadata: {
monitorId: string;
teamId: string;
containerId: string;
containerName: string;
};
lines: DockerLogLine[];
gap: boolean;
checkedAt: string;
expiry: string;
createdAt: string;
updatedAt: string;
}
export interface DockerContainerInfo {
id: string;
name: string;
+6
View File
@@ -5,6 +5,7 @@ import type {
DailyCheckBucket,
DockerContainerInfo,
DockerContainerSummary,
DockerLog,
} from "@/Types/Check";
import type { PageSpeedGroupedCheck } from "@/Types/Check";
import type { GeoContinent } from "@/Types/GeoCheck";
@@ -158,6 +159,7 @@ export interface Monitor {
geoCheckEnabled?: boolean;
geoCheckLocations?: GeoContinent[];
geoCheckInterval?: number;
dockerLogsEnabled?: boolean;
dnsServer?: string;
dnsRecordType?: DnsRecordType;
recentChecks: CheckSnapshot[];
@@ -322,6 +324,10 @@ export interface DockerContainerResponse {
stats: DockerContainerStats;
}
export interface DockerContainerLogsResponse {
logs: DockerLog[];
nextCursor: string | null;
}
export interface Game {
name: string;
options?: {
+39 -4
View File
@@ -1,10 +1,13 @@
import type { MonitorStatus, MonitorType } from "@/Types/Monitor";
import type { PaletteKey } from "@/Utils/Theme/Theme";
import type { ValueType } from "@/Components/design-elements/StatusLabel";
import type {
DockerContainerMount,
DockerContainerPort,
DockerContainerState,
import {
DockerLogStreams,
type DockerContainerMount,
type DockerContainerPort,
type DockerContainerState,
type DockerLog,
type DockerLogLine,
} from "@/Types/Check";
export const getMonitorPath = (type: MonitorType): string => {
@@ -144,3 +147,35 @@ export const getDockerMountLabel = (mount: DockerContainerMount) => {
if (mount.source) return mount.source;
return mount.type;
};
export const DockerLogStreamFilters = ["all", ...DockerLogStreams] as const;
export type DockerLogStreamFilter = (typeof DockerLogStreamFilters)[number];
export interface DockerLogRow extends DockerLogLine {
key: string;
gapBefore: boolean; // This signifies lines skipped
}
export const flattenDockerLogs = (logs: DockerLog[]): DockerLogRow[] => {
const rows: DockerLogRow[] = [];
for (const log of [...logs].reverse()) {
log.lines.forEach((line: DockerLogLine, idx: number) => {
rows.push({ ...line, key: `${log.id}:${idx}`, gapBefore: idx === 0 && log.gap });
});
}
return rows;
};
export const filterDockerLogRows = (
rows: DockerLogRow[],
stream: DockerLogStreamFilter,
query: string
): DockerLogRow[] => {
const normalizedQuery = query.trim().toLowerCase();
return rows.filter((row) => {
return (
(stream === "all" || row.stream === stream) &&
(normalizedQuery === "" || row.text.toLowerCase().includes(normalizedQuery))
);
});
};
+2 -1
View File
@@ -111,7 +111,8 @@ const portSchema = baseSchema.extend({
// Docker monitor schema
const dockerSchema = baseSchema.extend({
type: z.literal("docker"),
url: z.string().min(1, "Container ID is required"),
url: z.string().min(1, "Docker host URL is required"),
dockerLogsEnabled: z.boolean().register(monitorStepRegistry, { step: 1 }),
});
// Game server monitor schema
+35 -5
View File
@@ -632,9 +632,9 @@
},
"general": {
"option": {
"container": {
"label": "Container name/ID",
"placeholder": "my-app or abcd1234"
"dockerHost": {
"label": "Docker host",
"placeholder": "unix:///var/run/docker.sock"
},
"host": {
"label": "Host",
@@ -716,7 +716,7 @@
"http": "Enter the URL or IP to monitor (e.g., https://example.com/ or 192.168.1.100) and add a clear display name that appears on the dashboard. Set an optional proxy server to connect through.",
"ping": "Enter the IP address or hostname to ping (e.g., 192.168.1.100 or example.com) and add a clear display name that appears on the dashboard.",
"port": "Enter the URL or IP of the server, the port number and a clear display name that appears on the dashboard.",
"docker": "Enter the Docker container name or ID. You can use either the container name (e.g., my-app) or the container ID (full 64-char ID or short ID).",
"docker": "Enter the Docker socket to monitor, as unix:///var/run/docker.sock or an absolute path such as /var/run/docker.sock. Every container on the host is monitored.",
"game": "Enter the IP address or hostname and the port number to ping (e.g., 192.168.1.100 or example.com) and choose game type. Some games require enabling server queries in their config file — check your game server's documentation. See the <gamedigLink>supported games list</gamedigLink> for protocol details.",
"pagespeed": "Track page load performance, Core Web Vitals, and optimization scores for your website.",
"grpc": "Enter the hostname and port of the gRPC server, optionally specify a service name for the health check, and add a display name.",
@@ -756,7 +756,7 @@
},
"type": {
"description": "Select the type of check to perform",
"optionDockerDescription": "Use Docker to monitor if a container is running.",
"optionDockerDescription": "Monitor a Docker host and every container running on it.",
"optionGameDescription": "Monitor if a specific game server is online.",
"optionGrpcDescription": "Monitor gRPC services using the standard Health Checking Protocol.",
"optionWebSocketDescription": "Monitor WebSocket endpoints for connection health and response time.",
@@ -814,6 +814,15 @@
}
}
},
"dockerLogs": {
"title": "Container logs",
"description": "Collect the most recent log lines from every container on this host on each check. Logs are kept for 7 days. Leave this off for hosts with chatty containers to save database space.",
"option": {
"enabled": {
"label": "Collect container logs"
}
}
},
"stats": {
"title": "Monitor history",
"description": "Clear all monitoring history and statistics for your team. This action is irreversible.",
@@ -956,6 +965,27 @@
"tabs": {
"overview": "Overview",
"logs": "Logs"
},
"logs": {
"recentLines": "{{count}} recent lines",
"loading": "Loading...",
"updatedSecondsAgo": "Updated {{count}}s ago",
"updatedMinutesAgo": "Updated {{count}}m ago",
"skipped": "Some log entries were skipped",
"loadOlder": "Load older logs",
"loadingOlder": "Loading...",
"noOlder": "Beginning of retained logs",
"loadError": "Could not load logs",
"loadOlderError": "Could not load older logs",
"empty": {
"title": "No logs collected yet",
"description": "Lines will appear here after the next check. Only output written since collection was turned on is stored."
},
"disabled": {
"title": "Log collection is off for this host",
"description": "Turn on container logs in the monitor settings to start collecting.",
"action": "Configure monitor"
}
}
},
"host": {
+12
View File
@@ -24,6 +24,8 @@ import {
dockerContainerDetailsResponseSchema,
monitorResponseSchema,
uptimeDetailsResponseSchema,
getDockerContainerLogsQueryValidation,
dockerContainerLogsResponseSchema,
} from "@/api/validation/monitorValidation.js";
import { updateNotificationsValidation } from "@/api/validation/notificationValidation.js";
@@ -131,6 +133,16 @@ registry.registerPath({
responses: { "200": okJson(dockerContainerDetailsResponseSchema), ...standardErrors },
});
registry.registerPath({
method: "get",
path: "/monitors/docker/details/{monitorId}/containers/{containerName}/logs",
tags,
summary: "Get stored log lines for one container on a Docker host monitor, newest first",
security: bearer,
request: { params: getDockerContainerNameParamValidation, query: getDockerContainerLogsQueryValidation },
responses: { "200": okJson(dockerContainerLogsResponseSchema), ...standardErrors },
});
registry.registerPath({
method: "get",
path: "/monitors/pagespeed/details/{monitorId}",
@@ -21,6 +21,7 @@ import {
getDockerDetailsByIdQueryValidation,
getDockerContainerNameParamValidation,
getDockerContainerByNameQueryValidation,
getDockerContainerLogsQueryValidation,
} from "@/api/validation/monitorValidation.js";
import sslChecker from "ssl-checker";
import * as whoiser from "whoiser";
@@ -37,6 +38,7 @@ export interface IMonitorController {
getPageSpeedDetailsById: RequestHandler;
getDockerDetailsById: RequestHandler;
getDockerContainerByName: RequestHandler;
getDockerContainerLogs: RequestHandler;
getGeoChecksByMonitorId: RequestHandler;
getMonitorById: RequestHandler;
createMonitor: RequestHandler;
@@ -198,6 +200,30 @@ class MonitorController implements IMonitorController {
});
});
getDockerContainerLogs = catchAsync(async (req: Request, res: Response) => {
const validatedParams = getDockerContainerNameParamValidation.parse(req.params);
const validatedQuery = getDockerContainerLogsQueryValidation.parse(req.query);
const { monitorId, containerName } = validatedParams;
const { before, after, limit } = validatedQuery;
const teamId = requireTeamId(req.user?.teamId);
const data = await this.monitorService.getDockerContainerLogs({
teamId,
monitorId,
containerName,
before: before ? new Date(before) : undefined,
after: after ? new Date(after) : undefined,
limit,
});
return res.status(200).json({
success: true,
msg: "Docker container logs retrieved successfully",
data,
});
});
getGeoChecksByMonitorId = catchAsync(async (req: Request, res: Response) => {
const validatedParams = getMonitorByIdParamValidation.parse(req.params);
const validatedQuery = getMonitorByIdQueryValidation.parse(req.query);
+1
View File
@@ -21,6 +21,7 @@ export const createMonitorRoutes = (monitorController: IMonitorController): Rout
// Docker routes
router.get("/docker/details/:monitorId", monitorController.getDockerDetailsById);
router.get("/docker/details/:monitorId/containers/:containerName", monitorController.getDockerContainerByName);
router.get("/docker/details/:monitorId/containers/:containerName/logs", monitorController.getDockerContainerLogs);
// Geo checks routes
router.get("/:monitorId/geo-checks", monitorController.getGeoChecksByMonitorId);
+48 -15
View File
@@ -13,7 +13,8 @@ import {
ProxyModes,
} from "@/domain/monitors/monitor.type.js";
import { DateRanges, SortOrders } from "@/types/query.js";
import { DockerContainerStates, DockerHealthStatuses, DockerPortProtocols } from "@/types/network.js";
import { DockerContainerStates, DockerHealthStatuses, DockerLogStreams, DockerPortProtocols } from "@/domain/docker/docker.type.js";
import { DOCKER_LOG_PAGE_DEFAULT, DOCKER_LOG_PAGE_MAX } from "@/domain/docker/docker-log.type.js";
const httpStatusCode = z.number().refine((code) => HttpStatusCodeSet.has(code), { message: "Must be a valid HTTP status code" });
@@ -119,22 +120,15 @@ const refineProxySelection = (body: { proxyMode?: string; proxyId?: string }, ct
}
};
const dockerUrlRegex = /^(?:ssh:\/\/[^@\s]+@[^\s/:@]+(?::\d{1,5})?\/?|unix:\/\/\/\S+|\/\S+)$/;
const dockerUrlRegex = /^(?:unix:\/\/\/\S+|\/\S+)$/;
const refineDockerUrl = (data: { type?: string; url?: string; sshPrivateKey?: string }, ctx: z.RefinementCtx) => {
const refineDockerUrl = (data: { type?: string; url?: string }, ctx: z.RefinementCtx) => {
if (data.type !== "docker" || data.url === undefined) return;
if (!dockerUrlRegex.test(data.url)) {
ctx.addIssue({
code: "custom",
path: ["url"],
message: "Docker host must be ssh://user@host[:port], unix:///path, or an absolute socket path",
});
}
if (data.url.startsWith("ssh://") && !data.sshPrivateKey) {
ctx.addIssue({
code: "custom",
path: ["sshPrivateKey"],
message: "SSH Docker hosts require a private key",
message: "Docker host must be unix:///path or an absolute socket path",
});
}
};
@@ -163,7 +157,6 @@ export const createMonitorBodyValidation = z
tags: z.array(z.string()).optional(),
customUpCodes: z.array(httpStatusCode).default([]),
secret: z.string().optional(),
sshPrivateKey: z.string().optional(),
jsonPath: z.union([z.string(), z.literal("")]).optional(),
expectedValue: z.union([z.string(), z.literal("")]).optional(),
matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(),
@@ -176,6 +169,7 @@ export const createMonitorBodyValidation = z
geoCheckEnabled: z.boolean().optional(),
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
geoCheckInterval: z.number().min(300000).optional(),
dockerLogsEnabled: z.boolean().optional(),
dnsServer: dnsServerValidation.optional(),
dnsRecordType: z.enum(DnsRecordTypes).optional(),
})
@@ -199,7 +193,6 @@ export const editMonitorBodyValidation = z
tags: z.array(z.string()).optional(),
customUpCodes: z.array(httpStatusCode).optional(),
secret: z.string().optional(),
sshPrivateKey: z.string().optional(),
ignoreTlsErrors: z.boolean().optional(),
proxyMode: z.enum(ProxyModes).optional(),
proxyId: proxyIdValidation,
@@ -221,6 +214,7 @@ export const editMonitorBodyValidation = z
geoCheckEnabled: z.boolean().optional(),
geoCheckLocations: z.array(z.enum(GeoContinents)).optional(),
geoCheckInterval: z.number().min(300000).optional(),
dockerLogsEnabled: z.boolean().optional(),
dnsServer: dnsServerValidation.optional(),
dnsRecordType: z.enum(DnsRecordTypes).optional(),
})
@@ -280,7 +274,6 @@ const importedMonitorSchema = z
tags: z.array(z.string()).default([]),
customUpCodes: z.array(httpStatusCode).default([]),
secret: z.string().optional(),
sshPrivateKey: z.string().optional(),
cpuAlertThreshold: z.number().default(100),
cpuAlertCounter: z.number().default(5),
memoryAlertThreshold: z.number().default(100),
@@ -297,6 +290,8 @@ const importedMonitorSchema = z
geoCheckEnabled: z.boolean().default(false),
geoCheckLocations: z.array(z.enum(GeoContinents)).default([]),
geoCheckInterval: z.number().min(300000).default(300000),
dockerLogsEnabled: z.boolean().default(false),
dnsServer: dnsServerValidation.optional(),
dnsRecordType: z.enum(DnsRecordTypes).optional(),
createdAt: z.string().optional(),
@@ -332,6 +327,21 @@ export const getDockerContainerNameParamValidation = z.object({
});
export const getDockerContainerByNameQueryValidation = z.object({ dateRange: z.enum(DateRanges).optional() });
export const getDockerContainerLogsQueryValidation = z
.object({
before: z.iso.datetime().optional(),
after: z.iso.datetime().optional(),
limit: z.coerce.number().int().min(1).max(DOCKER_LOG_PAGE_MAX).default(DOCKER_LOG_PAGE_DEFAULT),
})
.superRefine((query, ctx) => {
if (query.before && query.after) {
ctx.addIssue({
code: "custom",
message: "Specify either before or after, not both",
path: ["after"],
});
}
});
// Canonical monitor shape returned by /monitors endpoints. Keep aligned with
// what the controllers actually serialize.
export const monitorResponseSchema = z
@@ -359,7 +369,6 @@ export const monitorResponseSchema = z
tags: z.array(z.string()),
customUpCodes: z.array(httpStatusCode).optional(),
secret: z.string().optional(),
sshPrivateKey: z.string().optional(),
cpuAlertThreshold: z.number(),
memoryAlertThreshold: z.number(),
diskAlertThreshold: z.number(),
@@ -371,6 +380,7 @@ export const monitorResponseSchema = z
geoCheckEnabled: z.boolean(),
geoCheckLocations: z.array(z.enum(GeoContinents)),
geoCheckInterval: z.number(),
dockerLogsEnabled: z.boolean(),
dnsServer: z.string().optional(),
dnsRecordType: z.enum(DnsRecordTypes).optional(),
teamId: z.string(),
@@ -529,3 +539,26 @@ export const dockerContainerDetailsResponseSchema = z.object({
.nullable(),
}),
});
export const dockerLogLineResponseSchema = z.object({
ts: z.string(),
stream: z.enum(DockerLogStreams),
text: z.string(),
});
export const dockerLogResponseSchema = z.object({
id: z.string(),
metadata: z.object({ monitorId: z.string(), teamId: z.string(), containerId: z.string(), containerName: z.string() }),
lines: z.array(dockerLogLineResponseSchema),
gap: z.boolean(),
checkedAt: z.string(),
expiry: z.string(),
createdAt: z.string(),
updatedAt: z.string(),
});
// Response of GET /monitors/docker/details/{monitorId}/containers/{containerName}/logs.
export const dockerContainerLogsResponseSchema = z.object({
logs: z.array(dockerLogResponseSchema),
nextCursor: z.string().nullable(),
});
+2
View File
@@ -35,6 +35,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
monitorsRepository,
checksRepository,
geoChecksRepository,
dockerLogsRepository,
monitorStatsRepository,
statusPagesRepository,
usersRepository,
@@ -75,6 +76,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
monitorsRepository,
checksRepository,
geoChecksRepository,
dockerLogsRepository,
monitorStatsRepository,
statusPagesRepository,
incidentsRepository,
+9
View File
@@ -13,6 +13,7 @@ import { IDb } from "@/db/db.interface.js";
import { ISettingsService, EnvConfig, SettingsService } from "@/domain/app-settings/app-settings.service.js";
import { ICheckService, CheckService } from "@/domain/checks/check.service.js";
import { IGeoChecksService, GeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
import { IDockerLogsService, DockerLogsService } from "@/domain/docker/docker-log.service.js";
import { IIncidentService, IncidentService } from "@/domain/incidents/incident.service.js";
import { INotificationMessageBuilder, NotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
import { INotificationsService, NotificationsService } from "@/domain/notifications/notification.service.js";
@@ -58,6 +59,7 @@ import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.
import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js";
import MongoChecksRepository from "@/domain/checks/check.repository.mongo.js";
import MongoGeoChecksRepository from "@/domain/geo-checks/geo-check.repository.mongo.js";
import MongoDockerLogsRepository from "@/domain/docker/docker-log.repository.mongo.js";
import MongoIncidentsRepository from "@/domain/incidents/incident.repository.mongo.js";
import MongoInvitesRepository from "@/domain/invites/invite.repository.mongo.js";
import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js";
@@ -72,6 +74,7 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js";
import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js";
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
// Shared infrastructure + business services that both the API and the worker process construct.
export interface SharedServices {
@@ -83,6 +86,7 @@ export interface SharedServices {
incidentService: IIncidentService;
checkService: ICheckService;
geoChecksService: IGeoChecksService;
dockerLogsService: IDockerLogsService;
notificationsService: INotificationsService;
// Queue identity (one per process; the worker reuses these)
@@ -94,6 +98,7 @@ export interface SharedServices {
monitorsRepository: IMonitorsRepository;
checksRepository: IChecksRepository;
geoChecksRepository: IGeoChecksRepository;
dockerLogsRepository: IDockerLogsRepository;
monitorStatsRepository: IMonitorStatsRepository;
statusPagesRepository: IStatusPagesRepository;
usersRepository: IUsersRepository;
@@ -131,6 +136,7 @@ export const buildShared = async ({
const incidentsRepository = new MongoIncidentsRepository();
const usersRepository = new MongoUsersRepository();
const geoChecksRepository = new MongoGeoChecksRepository();
const dockerLogsRepository = new MongoDockerLogsRepository();
const notificationsRepository = new MongoNotificationsRepository();
const jobsRepository = new MongoJobsRepository(workerId);
const queueWorkersRepository = new MongoQueueWorkersRepository();
@@ -161,6 +167,7 @@ export const buildShared = async ({
monitorsRepository,
});
const dockerLogsService = new DockerLogsService({ logger, dockerLogsRepository });
const webhookProvider = new WebhookProvider(logger);
const rocketChatProvider = new RocketChatProvider(logger);
const slackProvider = new SlackProvider(logger);
@@ -207,6 +214,7 @@ export const buildShared = async ({
incidentService,
checkService,
geoChecksService,
dockerLogsService,
notificationsService,
workerId,
jobsRepository,
@@ -214,6 +222,7 @@ export const buildShared = async ({
queueWorkersRepository,
checksRepository,
geoChecksRepository,
dockerLogsRepository,
monitorStatsRepository,
statusPagesRepository,
usersRepository,
+6 -2
View File
@@ -52,6 +52,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
settingsService,
checkService,
geoChecksService,
dockerLogsService,
notificationsService,
incidentService,
workerId,
@@ -60,6 +61,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
monitorsRepository,
checksRepository,
geoChecksRepository,
dockerLogsRepository,
monitorStatsRepository,
incidentsRepository,
teamsRepository,
@@ -95,7 +97,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
]);
const proxyResolver = new ProxyResolver(proxiesRepository, settingsService, logger);
const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository);
const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
const monitorStatusPolicy = new MonitorStatusPolicy();
@@ -119,6 +121,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
networkService,
proxyResolver,
bufferService,
dockerLogsService,
logger
);
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
@@ -138,7 +141,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
monitorStatsRepository,
checksRepository,
incidentsRepository,
geoChecksRepository
geoChecksRepository,
dockerLogsRepository
);
const worker = await DBQueueWorker.create({
@@ -7,8 +7,8 @@ import type { ILogger } from "@/utils/logger.js";
* The old docker provider interpreted monitor.url as a container name/ID and only
* ever talked to the local socket, so every pre-existing docker monitor's host is
* definitionally unix:///var/run/docker.sock. The container name is preserved in
* the description. Urls already in host form (unix://, ssh://, or an absolute
* socket path) are left untouched, which makes the migration idempotent.
* the description. Urls already in host form (unix:// or an absolute socket
* path) are left untouched, which makes the migration idempotent.
*/
export async function migrateDockerMonitorUrls(logger: ILogger): Promise<void> {
const SERVICE_NAME = "Migration:MigrateDockerMonitorUrls";
@@ -16,7 +16,7 @@ export async function migrateDockerMonitorUrls(logger: ILogger): Promise<void> {
try {
logger.info({ service: SERVICE_NAME, message: "Starting docker monitor url migration" });
const result = await MonitorModel.updateMany({ type: "docker", url: { $not: /^((unix|ssh):\/\/|\/)/ } }, [
const result = await MonitorModel.updateMany({ type: "docker", url: { $not: /^(unix:\/\/|\/)/ } }, [
{
$set: {
description: {
+1 -1
View File
@@ -22,7 +22,7 @@ import {
DockerContainerSummary,
DockerHealthStatuses,
DockerPortProtocols,
} from "@/types/network.js";
} from "@/domain/docker/docker.type.js";
type CheckMetadataDocument = Omit<CheckMetadata, "monitorId" | "teamId"> & {
monitorId: Types.ObjectId;
+1 -1
View File
@@ -11,7 +11,7 @@ import type {
SnapshotHostInfo,
SnapshotMemoryInfo,
} from "@/domain/checks/check.type.js";
import { DockerContainerSummary } from "@/types/network.js";
import { DockerContainerSummary } from "@/domain/docker/docker.type.js";
import { toDateString } from "@/utils/mongoMappers.js";
export type CheckSnapshotSource = Pick<
+1 -1
View File
@@ -1,5 +1,5 @@
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { DockerContainerInfo, DockerContainerSummary } from "@/types/network.js";
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { Response } from "got";
export const CHECK_TTL_SENTINEL = 366;
@@ -0,0 +1,60 @@
import { Schema, model, Types } from "mongoose";
import { DockerLogStreams, type DockerLogLine } from "@/domain/docker/docker.type.js";
import type { DockerLog, DockerLogMetadata } from "@/domain/docker/docker-log.type.js";
type DockerLogMetadataDocument = Omit<DockerLogMetadata, "monitorId" | "teamId"> & {
monitorId: Types.ObjectId;
teamId: Types.ObjectId;
};
type DockerLogDocumentBase = Omit<DockerLog, "id" | "metadata" | "checkedAt" | "expiry" | "createdAt" | "updatedAt"> & {
metadata: DockerLogMetadataDocument;
checkedAt: Date;
expiry: Date;
createdAt: Date;
updatedAt: Date;
};
export interface DockerLogDocument extends DockerLogDocumentBase {
_id: Types.ObjectId;
}
const dockerLogMetadataSchema = new Schema<DockerLogMetadataDocument>(
{
monitorId: { type: Schema.Types.ObjectId, required: true },
teamId: { type: Schema.Types.ObjectId, required: true },
containerId: { type: String, required: true },
containerName: { type: String, required: true },
},
{ _id: false }
);
const dockerLogLineSchema = new Schema<DockerLogLine>(
{
ts: { type: String, required: true },
stream: { type: String, enum: DockerLogStreams, required: true },
text: { type: String, default: "" },
},
{ _id: false }
);
const DockerLogSchema = new Schema<DockerLogDocument>(
{
metadata: { type: dockerLogMetadataSchema, required: true },
lines: { type: [dockerLogLineSchema], required: true, default: [] },
gap: { type: Boolean, default: false },
checkedAt: { type: Date, required: true },
expiry: { type: Date, required: true, index: { expires: 0 } }, // TTL: Mongo deletes the doc once `expiry` passes
},
{ timestamps: true }
);
DockerLogSchema.index({ "metadata.monitorId": 1, "metadata.containerName": 1, checkedAt: -1 }); // API paging
DockerLogSchema.index({ "metadata.monitorId": 1, "metadata.containerId": 1, checkedAt: -1 }); // cursor seed
DockerLogSchema.index({ "metadata.teamId": 1 });
const DockerLogModel = model<DockerLogDocument>("DockerLog", DockerLogSchema);
export type { DockerLogMetadataDocument };
export { DockerLogModel };
export default DockerLogModel;
@@ -0,0 +1,10 @@
import { DockerLog } from "@/domain/docker/docker-log.type.js";
export interface IDockerLogsRepository {
createDockerLogs(logs: Omit<DockerLog, "id" | "createdAt" | "updatedAt">[]): Promise<number>;
findLastLineTimestamp(monitorId: string, containerId: string): Promise<string | null>;
findByContainerName(args: { monitorId: string; containerName: string; before?: Date; after?: Date; limit: number }): Promise<DockerLog[]>;
deleteByMonitorId(monitorId: string): Promise<number>;
deleteByTeamId(teamId: string): Promise<number>;
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
}
@@ -0,0 +1,99 @@
import DockerLogModel, { DockerLogDocument } from "@/domain/docker/docker-log.model.js";
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
import { DockerLog } from "@/domain/docker/docker-log.type.js";
import { toDateString, toStringId } from "@/utils/mongoMappers.js";
import mongoose from "mongoose";
const SERVICE_NAME = "DockerLogsRepository";
class MongoDockerLogsRepository implements IDockerLogsRepository {
static SERVICE_NAME = SERVICE_NAME;
private toEntity = (doc: DockerLogDocument): DockerLog => {
return {
id: toStringId(doc._id),
metadata: {
monitorId: toStringId(doc.metadata.monitorId),
teamId: toStringId(doc.metadata.teamId),
containerId: doc.metadata.containerId,
containerName: doc.metadata.containerName,
},
lines: (doc.lines ?? []).map((line) => ({ ts: line.ts, stream: line.stream, text: line.text ?? "" })),
gap: doc.gap ?? false,
checkedAt: toDateString(doc.checkedAt),
expiry: toDateString(doc.expiry),
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
};
};
createDockerLogs = async (logs: Omit<DockerLog, "id" | "createdAt" | "updatedAt">[]): Promise<number> => {
if (logs.length === 0) return 0;
const docs = await DockerLogModel.insertMany(
logs.map((log) => {
return {
metadata: {
monitorId: new mongoose.Types.ObjectId(log.metadata.monitorId),
teamId: new mongoose.Types.ObjectId(log.metadata.teamId),
containerId: log.metadata.containerId,
containerName: log.metadata.containerName,
},
lines: log.lines,
gap: log.gap,
checkedAt: new Date(log.checkedAt),
expiry: new Date(log.expiry),
};
}),
{ ordered: false }
);
return docs.length;
};
findLastLineTimestamp = async (monitorId: string, containerId: string): Promise<string | null> => {
const doc = await DockerLogModel.findOne({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.containerId": containerId,
})
.sort({ checkedAt: -1 })
.select({ lines: { $slice: -1 } })
.lean<Pick<DockerLogDocument, "lines">>();
return doc?.lines?.[0]?.ts ?? null;
};
findByContainerName = async ({
monitorId,
containerName,
before,
after,
limit,
}: {
monitorId: string;
containerName: string;
before?: Date;
after?: Date;
limit: number;
}): Promise<DockerLog[]> => {
const docs = await DockerLogModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
"metadata.containerName": containerName,
...(before ? { checkedAt: { $lt: before } } : {}),
...(after ? { checkedAt: { $gt: after } } : {}),
})
.sort({ checkedAt: after ? 1 : -1 })
.limit(limit)
.lean<DockerLogDocument[]>();
const entities = docs.map(this.toEntity);
return after ? entities.reverse() : entities;
};
deleteByMonitorId = async (monitorId: string) =>
(await DockerLogModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) })).deletedCount ?? 0;
deleteByTeamId = async (teamId: string) =>
(await DockerLogModel.deleteMany({ "metadata.teamId": new mongoose.Types.ObjectId(teamId) })).deletedCount ?? 0;
deleteByMonitorIdsNotIn = async (monitorIds: string[]) =>
(await DockerLogModel.deleteMany({ "metadata.monitorId": { $nin: monitorIds.map((id) => new mongoose.Types.ObjectId(id)) } })).deletedCount ?? 0;
}
export default MongoDockerLogsRepository;
@@ -0,0 +1,100 @@
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
import { DOCKER_LOG_RETENTION_DAYS, DockerLog } from "@/domain/docker/docker-log.type.js";
import { DOCKER_LOG_TAIL_LINES, DockerLogLine } from "@/domain/docker/docker.type.js";
import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { ILogger } from "@/utils/logger.js";
import mongoose from "mongoose";
const SERVICE_NAME = "DockerLogsService";
const MS_PER_DAY = 24 * 60 * 60 * 1000;
export interface IDockerLogsService {
buildDockerLogs(status: MonitorStatusResponse<DockerStatusPayload>): Promise<DockerLog[]>;
createDockerLogs(logs: DockerLog[]): Promise<number>;
}
export class DockerLogsService implements IDockerLogsService {
static SERVICE_NAME = SERVICE_NAME;
private logger: ILogger;
private dockerLogsRepository: IDockerLogsRepository;
private lastTimestampByContainer = new Map<string, string>();
constructor({ logger, dockerLogsRepository }: { logger: ILogger; dockerLogsRepository: IDockerLogsRepository }) {
this.logger = logger;
this.dockerLogsRepository = dockerLogsRepository;
}
private generateCursorKey = (monitorId: string, containerId: string) => `${monitorId}:${containerId}`;
private getLastTimestamp = async (monitorId: string, containerId: string) => {
const key = this.generateCursorKey(monitorId, containerId);
const cached = this.lastTimestampByContainer.get(key);
if (cached !== undefined) return cached || null;
const stored = await this.dockerLogsRepository.findLastLineTimestamp(monitorId, containerId);
this.lastTimestampByContainer.set(key, stored ?? "");
return stored;
};
private newLines = (lines: DockerLogLine[], lastTimestamp: string | null): DockerLogLine[] => {
return lastTimestamp
? lines.filter((line) => {
return line.ts > lastTimestamp;
})
: lines;
};
private maxTimestamp = (lines: DockerLogLine[]) => {
return lines.reduce((max, line) => (line.ts > max ? line.ts : max), "");
};
buildDockerLogs = async (status: MonitorStatusResponse<DockerStatusPayload>): Promise<DockerLog[]> => {
const payload = typeof status.payload === "object" ? status.payload : null;
const containerLogs = payload?.logs ?? [];
if (containerLogs.length === 0) return [];
const now = new Date();
const checkedAt = now.toISOString();
const expiry = new Date(now.getTime() + DOCKER_LOG_RETENTION_DAYS * MS_PER_DAY).toISOString();
const results: DockerLog[] = [];
for (const dockerContainerLog of containerLogs) {
const { containerId, containerName, lines } = dockerContainerLog;
try {
const lastTimestamp = await this.getLastTimestamp(status.monitorId, containerId);
const fresh = this.newLines(lines, lastTimestamp);
if (fresh.length === 0) continue;
this.lastTimestampByContainer.set(this.generateCursorKey(status.monitorId, containerId), this.maxTimestamp(fresh));
results.push({
id: new mongoose.Types.ObjectId().toString(),
metadata: {
monitorId: status.monitorId,
teamId: status.teamId,
containerId,
containerName,
},
lines: fresh,
gap: fresh.length === DOCKER_LOG_TAIL_LINES,
// We only fetch DOCKER_LOG_TAIL_LINES worth of logs.
// If we have 200 fresh logs, there _could_ be more than 200 lines, so set gap = true
// Client can display that some logs may be missing
checkedAt,
expiry,
createdAt: checkedAt,
updatedAt: checkedAt,
});
} catch (error: unknown) {
this.logger.warn({
message: `Failed to build docker logs for container ${containerName}`,
service: SERVICE_NAME,
method: "buildDockerLogs",
details: { monitorId: status.monitorId, containerId, error: error instanceof Error ? error.message : String(error) },
});
}
}
return results;
};
createDockerLogs = async (logs: DockerLog[]): Promise<number> => this.dockerLogsRepository.createDockerLogs(logs);
}
@@ -0,0 +1,28 @@
import { DockerLogLine } from "@/domain/docker/docker.type.js";
export const DOCKER_LOG_RETENTION_DAYS = 7;
export const DOCKER_LOG_PAGE_DEFAULT = 20;
export const DOCKER_LOG_PAGE_MAX = 50;
export interface DockerLogMetadata {
monitorId: string;
teamId: string;
containerId: string;
containerName: string;
}
export interface DockerLog {
id: string;
metadata: DockerLogMetadata;
lines: DockerLogLine[];
gap: boolean;
checkedAt: string;
expiry: string;
createdAt: string;
updatedAt: string;
}
export interface DockerLogPage {
logs: DockerLog[];
nextCursor: string | null;
}
+65
View File
@@ -0,0 +1,65 @@
export const DockerContainerStates = ["created", "running", "paused", "restarting", "removing", "exited", "dead"] as const;
export type DockerContainerState = (typeof DockerContainerStates)[number];
export const DockerHealthStatuses = ["healthy", "unhealthy", "starting", "none"] as const;
export type DockerHealthStatus = (typeof DockerHealthStatuses)[number];
export const DockerPortProtocols = ["tcp", "udp", "sctp"] as const;
export type DockerPortProtocol = (typeof DockerPortProtocols)[number];
export const DockerLogStreams = ["stdout", "stderr"] as const;
export type DockerLogStream = (typeof DockerLogStreams)[number];
export const DOCKER_LOG_TAIL_LINES = 200;
export interface DockerLogLine {
ts: string;
stream: DockerLogStream;
text: string;
}
export interface DockerContainerLogs {
containerId: string;
containerName: string;
lines: DockerLogLine[];
}
export interface DockerContainerPort {
privatePort: number;
protocol: DockerPortProtocol;
publicPort?: number;
hostIp?: string;
}
export interface DockerContainerMount {
type: string; // open set: bind, volume, tmpfs, image, npipe, cluster, …
name?: string;
source: string;
destination: string;
mode: string;
rw: boolean;
}
export interface DockerContainerInfo {
id: string;
name: string;
image: string;
state: DockerContainerState;
status: string;
health: DockerHealthStatus;
cpuPct?: number; // fraction of one core (docker stats convention / 100); exceeds 1 when using multiple cores
memoryUsedBytes?: number;
memoryLimitBytes?: number;
memoryPct?: number; // 0-1 fraction of the memory limit
restartCount?: number;
startedAt?: string; // ISO date
ports?: DockerContainerPort[];
mounts?: DockerContainerMount[];
}
export interface DockerContainerSummary {
total: number;
running: number;
stopped: number;
unhealthy: number;
}
+4 -3
View File
@@ -235,9 +235,6 @@ const MonitorSchema = new Schema<MonitorDocument>(
secret: {
type: String,
},
sshPrivateKey: {
type: String,
},
cpuAlertThreshold: {
type: Number,
default: 100,
@@ -306,6 +303,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
type: Number,
default: 300000,
},
dockerLogsEnabled: {
type: Boolean,
default: false,
},
dnsServer: {
type: String,
},
@@ -436,7 +436,6 @@ class MongoMonitorsRepository implements IMonitorsRepository {
tags: tagIds,
customUpCodes: doc.customUpCodes ?? [],
secret: doc.secret ?? undefined,
sshPrivateKey: doc.sshPrivateKey ?? undefined,
cpuAlertThreshold: doc.cpuAlertThreshold,
cpuAlertCounter: doc.cpuAlertCounter,
memoryAlertThreshold: doc.memoryAlertThreshold,
@@ -454,6 +453,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
geoCheckEnabled: doc.geoCheckEnabled ?? false,
geoCheckLocations: doc.geoCheckLocations ?? [],
geoCheckInterval: doc.geoCheckInterval ?? 300000,
dockerLogsEnabled: doc.dockerLogsEnabled ?? false,
dnsServer: doc.dnsServer ?? undefined,
dnsRecordType: doc.dnsRecordType ?? undefined,
createdAt: toDateString(doc.createdAt),
@@ -9,6 +9,7 @@ import type {
GroupedGeoCheckResult,
DockerDetailsResult,
DockerContainerDetailsResult,
DockerContainerLogsResult,
} from "@/domain/monitors/monitor.type.js";
import { supportsGeoCheck, supportsUptimeDetails } from "@/domain/monitors/monitor.type.js";
import type {
@@ -31,6 +32,7 @@ import type { ImportedMonitor } from "@/api/validation/monitorValidation.js";
import { ILogger } from "@/utils/logger.js";
import { IJobScheduler } from "@/worker/worker.interface.js";
import { DateRange } from "@/types/query.js";
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
const SERVICE_NAME = "MonitorService";
@@ -73,6 +75,14 @@ export interface IMonitorService {
containerName: string;
dateRange: DateRange;
}): Promise<DockerContainerDetailsResult>;
getDockerContainerLogs(args: {
teamId: string;
monitorId: string;
containerName: string;
before?: Date;
after?: Date;
limit: number;
}): Promise<DockerContainerLogsResult>;
getGeoChecksByMonitorId(args: {
teamId: string;
monitorId: string;
@@ -130,6 +140,7 @@ export class MonitorService implements IMonitorService {
private monitorsRepository: IMonitorsRepository;
private checksRepository: IChecksRepository;
private geoChecksRepository: IGeoChecksRepository;
private dockerLogsRepository: IDockerLogsRepository;
private monitorStatsRepository: IMonitorStatsRepository;
private statusPagesRepository: IStatusPagesRepository;
private incidentsRepository: IIncidentsRepository;
@@ -141,6 +152,7 @@ export class MonitorService implements IMonitorService {
monitorsRepository,
checksRepository,
geoChecksRepository,
dockerLogsRepository,
monitorStatsRepository,
statusPagesRepository,
incidentsRepository,
@@ -151,6 +163,7 @@ export class MonitorService implements IMonitorService {
monitorsRepository: IMonitorsRepository;
checksRepository: IChecksRepository;
geoChecksRepository: IGeoChecksRepository;
dockerLogsRepository: IDockerLogsRepository;
monitorStatsRepository: IMonitorStatsRepository;
statusPagesRepository: IStatusPagesRepository;
incidentsRepository: IIncidentsRepository;
@@ -161,6 +174,7 @@ export class MonitorService implements IMonitorService {
this.monitorsRepository = monitorsRepository;
this.checksRepository = checksRepository;
this.geoChecksRepository = geoChecksRepository;
this.dockerLogsRepository = dockerLogsRepository;
this.monitorStatsRepository = monitorStatsRepository;
this.statusPagesRepository = statusPagesRepository;
this.incidentsRepository = incidentsRepository;
@@ -385,6 +399,41 @@ export class MonitorService implements IMonitorService {
};
};
getDockerContainerLogs = async ({
teamId,
monitorId,
containerName,
before,
after,
limit,
}: {
teamId: string;
monitorId: string;
containerName: string;
before?: Date;
after?: Date;
limit: number;
}): Promise<DockerContainerLogsResult> => {
const monitor = await this.monitorsRepository.findById(monitorId, teamId);
if (!monitor) {
throw new AppError({
message: `Monitor with ID ${monitorId} not found`,
status: 404,
});
}
if (monitor.type !== "docker") {
throw new AppError({
message: `${monitor.type} monitors are not supported for docker container logs`,
status: 400,
});
}
const logs = await this.dockerLogsRepository.findByContainerName({ monitorId, containerName, before, after, limit });
const nextCursor = !after && logs.length === limit ? (logs[logs.length - 1]?.checkedAt ?? null) : null;
return { logs, nextCursor };
};
getGeoChecksByMonitorId = async ({
teamId,
monitorId,
@@ -593,6 +642,14 @@ export class MonitorService implements IMonitorService {
});
});
await this.dockerLogsRepository.deleteByMonitorId(monitor.id).catch((err: unknown) => {
this.logger.warn({
message: `Error deleting docker logs for monitor ${monitor.id} with name ${monitor.name}`,
service: SERVICE_NAME,
stack: err instanceof Error ? err.stack : undefined,
});
});
await this.monitorStatsRepository.deleteByMonitorId(monitor.id).catch((err: unknown) => {
this.logger.warn({
message: `Error deleting monitor stats for monitor ${monitor.id} with name ${monitor.name}`,
+4 -1
View File
@@ -5,6 +5,7 @@ export type { GeoContinent } from "@/domain/geo-checks/geo-check.type.js";
import http from "node:http";
import { HardwareStats } from "@/domain/checks/check.type.js";
import { MonitorStats } from "@/domain/monitor-stats/monitor-stats.type.js";
import { DockerLogPage } from "@/domain/docker/docker-log.type.js";
export const HttpStatusCodes = [
...Object.keys(http.STATUS_CODES).map(Number),
@@ -91,7 +92,6 @@ export interface Monitor {
tags: string[];
customUpCodes: HttpStatusCode[];
secret?: string;
sshPrivateKey?: string;
cpuAlertThreshold: number;
cpuAlertCounter: number;
memoryAlertThreshold: number;
@@ -108,6 +108,7 @@ export interface Monitor {
geoCheckEnabled?: boolean;
geoCheckLocations?: GeoContinent[];
geoCheckInterval?: number;
dockerLogsEnabled?: boolean;
dnsServer?: string;
dnsRecordType?: DnsRecordType;
recentChecks: CheckSnapshot[];
@@ -189,3 +190,5 @@ export interface Game {
export type GamesMap = Record<string, Game>;
export type MonitorScheduleFields = Pick<Monitor, "id" | "type" | "isActive" | "interval" | "geoCheckEnabled" | "geoCheckInterval">;
export type DockerContainerLogsResult = DockerLogPage;
+48
View File
@@ -5,14 +5,18 @@ import type { ILogger } from "@/utils/logger.js";
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ICheckService } from "../domain/checks/check.service.js";
import { DockerLog } from "@/domain/docker/docker-log.type.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
const SERVICE_NAME = "BufferService";
export interface IBufferService {
addToBuffer(check: Check): void;
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
addDockerLogToBuffer(dockerLog: DockerLog): void;
scheduleNextFlush(): void;
flushBuffer(): Promise<void>;
flushGeoBuffer(): Promise<void>;
flushDockerLogsBuffer(): Promise<void>;
shutdown(): Promise<void>;
}
@@ -23,15 +27,18 @@ export class BufferService implements IBufferService {
private SERVICE_NAME: string;
private buffer: Check[];
private geoBuffer: GeoCheck[];
private dockerLogBuffer: DockerLog[];
private bufferTimer: NodeJS.Timeout | null = null;
private checksService: ICheckService;
private geoChecksService: IGeoChecksService;
private dockerLogsService: IDockerLogsService;
private jobsRepository: IJobsRepository;
constructor(
logger: ILogger,
checkService: ICheckService,
geoChecksService: IGeoChecksService,
dockerLogsService: IDockerLogsService,
settingsService: ISettingsService,
jobsRepository: IJobsRepository
) {
@@ -39,10 +46,12 @@ export class BufferService implements IBufferService {
this.logger = logger;
this.checksService = checkService;
this.geoChecksService = geoChecksService;
this.dockerLogsService = dockerLogsService;
this.jobsRepository = jobsRepository;
this.SERVICE_NAME = SERVICE_NAME;
this.buffer = [];
this.geoBuffer = [];
this.dockerLogBuffer = [];
this.scheduleNextFlush();
this.logger.info({
message: `Buffer service initialized, flushing every ${this.BUFFER_TIMEOUT / 1000}s`,
@@ -77,6 +86,19 @@ export class BufferService implements IBufferService {
}
}
addDockerLogToBuffer(dockerLog: DockerLog): void {
try {
this.dockerLogBuffer.push(dockerLog);
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : "Unknown error",
service: this.SERVICE_NAME,
method: "addDockerLogToBuffer",
stack: error instanceof Error ? error.stack : undefined,
});
}
}
scheduleNextFlush() {
if (this.bufferTimer) {
clearTimeout(this.bufferTimer);
@@ -85,6 +107,7 @@ export class BufferService implements IBufferService {
try {
await this.flushBuffer();
await this.flushGeoBuffer();
await this.flushDockerLogsBuffer();
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : "Unknown error",
@@ -150,6 +173,30 @@ export class BufferService implements IBufferService {
}
}
async flushDockerLogsBuffer() {
if (this.dockerLogBuffer.length === 0) {
return;
}
// Take the batch first so a write that drains it can't be appended to mid-flush
const batch = this.dockerLogBuffer;
this.dockerLogBuffer = [];
try {
this.logger.debug({
message: `Flushing ${batch.length} docker logs to database`,
service: this.SERVICE_NAME,
method: "flushDockerLogsBuffer",
});
await this.dockerLogsService.createDockerLogs(batch);
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : "Unknown error",
service: this.SERVICE_NAME,
method: "flushDockerLogsBuffer",
stack: error instanceof Error ? error.stack : undefined,
});
}
}
async shutdown() {
if (this.bufferTimer) {
clearTimeout(this.bufferTimer);
@@ -157,5 +204,6 @@ export class BufferService implements IBufferService {
}
await this.flushBuffer();
await this.flushGeoBuffer();
await this.flushDockerLogsBuffer();
}
}
+94 -30
View File
@@ -1,6 +1,10 @@
import { IStatusProvider } from "@/service/network/IStatusProvider.js";
import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import {
DOCKER_LOG_TAIL_LINES,
DockerContainerInfo,
DockerContainerLogs,
DockerContainerMount,
DockerContainerPort,
DockerContainerState,
@@ -8,11 +12,11 @@ import {
DockerContainerSummary,
DockerHealthStatus,
DockerHealthStatuses,
DockerLogLine,
DockerLogStream,
DockerPortProtocol,
DockerPortProtocols,
DockerStatusPayload,
MonitorStatusResponse,
} from "@/types/network.js";
} from "@/domain/docker/docker.type.js";
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
import { ILogger } from "@/utils/logger.js";
import { AppError } from "@/utils/AppError.js";
@@ -23,6 +27,9 @@ type DockerodeType = typeof Dockerode;
const SERVICE_NAME = "DockerProvider";
const STATS_CONCURRENCY = 5;
const DOCKER_LOG_MAX_LINE_BYTES = 4096;
const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
export interface DockerError extends Error {
statusCode?: number;
@@ -56,31 +63,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
return { socketPath };
}
if (url.startsWith("/")) return { socketPath: url };
const match = url.match(/^ssh:\/\/(?:([^@\s]+)@)?([^\s/:@]+)(?::(\d{1,5}))?\/?$/);
if (!match) throw new AppError({ message: `Invalid Docker host URL: ${url}`, status: 422, service: SERVICE_NAME, method: "toDockerOptions" });
const [, username, host, port] = match;
if (!username)
throw new AppError({
message: "SSH Docker host URL requires a user: ssh://user@host",
status: 422,
service: SERVICE_NAME,
method: "toDockerOptions",
});
if (!monitor.sshPrivateKey)
throw new AppError({
message: "SSH Docker host requires a private key",
status: 422,
service: SERVICE_NAME,
method: "toDockerOptions",
});
return {
protocol: "ssh",
host,
port: port ? Number(port) : 22,
username,
sshOptions: { privateKey: monitor.sshPrivateKey },
};
throw new AppError({ message: `Invalid Docker host URL: ${url}`, status: 422, service: SERVICE_NAME, method: "toDockerOptions" });
};
private async mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]> {
@@ -119,10 +102,14 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
};
}
private toContainerName = (summary: Dockerode.ContainerInfo): string => {
return summary.Names?.[0]?.replace(/^\//, "") ?? summary.Id.slice(0, 12);
};
private async toContainerInfo(docker: Dockerode, summary: Dockerode.ContainerInfo): Promise<DockerContainerInfo> {
const info: DockerContainerInfo = {
id: summary.Id,
name: summary.Names?.[0]?.replace(/^\//, "") ?? summary.Id.slice(0, 12),
name: this.toContainerName(summary),
image: summary.Image,
state: this.toContainerState(summary.State),
status: summary.Status,
@@ -198,6 +185,81 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
}));
}
// Log parsing
private isMultiplexed(buffer: Buffer): boolean {
return buffer.length >= 8 && buffer.readUInt8(0) <= 2 && buffer.readUInt8(1) === 0 && buffer.readUInt8(2) === 0 && buffer.readUInt8(3) === 0;
}
// Docker will will merge stdout and stderr if stared with `-t`.
// Most containers will _not_ be run with `-t` though, in which case logs come in frames of [header][payload]
private demuxLogBuffer = (buffer: Buffer): { stream: DockerLogStream; chunk: string }[] => {
if (!this.isMultiplexed(buffer)) return [{ stream: "stdout", chunk: buffer.toString("utf8") }]; // `-t` case, log is just a string, return it
const frames: { stream: DockerLogStream; chunk: string }[] = [];
let offset = 0; // Start reading at the first byte of the first header
while (offset + 8 <= buffer.length) {
// Every 8 bytes is a header, if < 8 bytes at the end, invalid frame, stop
const streamType = buffer.readUint8(offset); // First byte of the header is the stream type: 0 = stdin, 1 = stdout, 2 = stderr
const payloadSize = buffer.readUint32BE(offset + 4); // Byte 4 to 7 describe how many payload bytes follow the header
const payloadStart = offset + 8; // Advance to the payload (header 8 bytes long)
const payloadEnd = Math.min(payloadStart + payloadSize, buffer.length); // End is the start of the payload + payload size.
frames.push({ stream: streamType === 2 ? "stderr" : "stdout", chunk: buffer.toString("utf-8", payloadStart, payloadEnd) });
offset = payloadEnd;
}
return frames;
};
private truncateLog = (text: string): string => {
if (Buffer.byteLength(text, "utf8") <= DOCKER_LOG_MAX_LINE_BYTES) return text;
const cut = Buffer.from(text, "utf8").subarray(0, DOCKER_LOG_MAX_LINE_BYTES).toString("utf8").replace(/�$/, "");
return cut + DOCKER_LOG_TRUNCATION_MARKER;
};
// Docker drops trailing zeroes form timestamps. Pad out timestamps for correct string comparison
private toLogLine(stream: DockerLogStream, raw: string): DockerLogLine | null {
const match = raw.match(DOCKER_LOG_TS_REGEX);
if (!match) return null; // If there's no time stamp, something is wrong with this line, drop it
const ts = `${match[1]}.${(match[2] ?? "").padEnd(9, "0")}Z`;
const text = raw.slice(match[0].length).replace(/\r$/, "");
return { ts, stream, text: this.truncateLog(text) };
}
private parseLogBuffer = (buffer: Buffer): DockerLogLine[] => {
const lines: DockerLogLine[] = [];
const demuxedBuffer = this.demuxLogBuffer(buffer);
for (const byteStream of demuxedBuffer) {
const { stream, chunk } = byteStream;
for (const raw of chunk.split("\n")) {
if (raw.length === 0) continue;
const line = this.toLogLine(stream, raw);
if (line) lines.push(line);
}
}
return lines;
};
private toContainerLogs = async (docker: Dockerode, summary: Dockerode.ContainerInfo): Promise<DockerContainerLogs | null> => {
const containerName = this.toContainerName(summary);
try {
const buffer = await docker
.getContainer(summary.Id)
.logs({ follow: false, stdout: true, stderr: true, timestamps: true, tail: DOCKER_LOG_TAIL_LINES });
return { containerId: summary.Id, containerName, lines: this.parseLogBuffer(buffer) };
} catch (error: unknown) {
this.logger.warn({
message: `Failed to read logs for container ${containerName}`,
service: SERVICE_NAME,
method: "toContainerLogs",
details: { containerId: summary.Id, error: error instanceof Error ? error.message : String(error) },
});
return null;
}
};
private collectLogs = async (docker: Dockerode, summaries: Dockerode.ContainerInfo[]): Promise<DockerContainerLogs[]> => {
const rawLogs = await this.mapWithConcurrency(summaries, STATS_CONCURRENCY, (s) => this.toContainerLogs(docker, s));
return rawLogs.filter((log) => log !== null);
};
handle = async (monitor: Monitor): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
try {
const docker = new this.DockerLib(this.toDockerOptions(monitor));
@@ -226,6 +288,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
const summaries = await docker.listContainers({ all: true });
const containers = await this.mapWithConcurrency(summaries, STATS_CONCURRENCY, (s) => this.toContainerInfo(docker, s));
const logs = monitor.dockerLogsEnabled ? await this.collectLogs(docker, summaries) : undefined;
const summary: DockerContainerSummary = {
total: containers.length,
running: containers.filter((c) => c.state === "running").length,
@@ -240,7 +304,7 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
code: 200,
message: "Docker host is reachable",
responseTime,
payload: { containers, summary },
payload: { containers, summary, logs },
};
} catch (error: unknown) {
if (error instanceof AppError) throw error;
+2 -48
View File
@@ -9,6 +9,7 @@ import type {
GotTimings,
ILighthouseAudit,
} from "@/domain/checks/check.type.js";
import { DockerContainerInfo, DockerContainerLogs, DockerContainerSummary } from "@/domain/docker/docker.type.js";
import type { DnsRecordType, Monitor, MonitorMatchMethod, MonitorStatus, MonitorType } from "@/domain/monitors/monitor.type.js";
import type { QueryResult } from "gamedig";
@@ -97,58 +98,11 @@ export interface HardwareStatusPayload {
}
// Docker host monitoring
export const DockerContainerStates = ["created", "running", "paused", "restarting", "removing", "exited", "dead"] as const;
export type DockerContainerState = (typeof DockerContainerStates)[number];
export const DockerHealthStatuses = ["healthy", "unhealthy", "starting", "none"] as const;
export type DockerHealthStatus = (typeof DockerHealthStatuses)[number];
export interface DockerContainerSummary {
total: number;
running: number;
stopped: number;
unhealthy: number;
}
export interface DockerContainerInfo {
id: string;
name: string;
image: string;
state: DockerContainerState;
status: string;
health: DockerHealthStatus;
cpuPct?: number; // fraction of one core (docker stats convention / 100); exceeds 1 when using multiple cores
memoryUsedBytes?: number;
memoryLimitBytes?: number;
memoryPct?: number; // 0-1 fraction of the memory limit
restartCount?: number;
startedAt?: string; // ISO date
ports?: DockerContainerPort[];
mounts?: DockerContainerMount[];
}
export const DockerPortProtocols = ["tcp", "udp", "sctp"] as const;
export type DockerPortProtocol = (typeof DockerPortProtocols)[number];
export interface DockerContainerPort {
privatePort: number;
protocol: DockerPortProtocol;
publicPort?: number;
hostIp?: string;
}
export interface DockerContainerMount {
type: string; // open set: bind, volume, tmpfs, image, npipe, cluster, …
name?: string;
source: string;
destination: string;
mode: string;
rw: boolean;
}
export interface DockerStatusPayload {
containers: DockerContainerInfo[];
summary: DockerContainerSummary;
logs?: DockerContainerLogs[];
}
export interface PortStatusPayload {
+11 -1
View File
@@ -1,6 +1,6 @@
import { Monitor } from "@/domain/monitors/monitor.type.js";
import { Check } from "@/domain/checks/check.type.js";
import { MonitorStatusResponse } from "@/types/network.js";
import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import { ILogger } from "@/utils/logger.js";
import { IBufferService } from "@/service/bufferService.js";
@@ -10,6 +10,7 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
import { isWindowActive } from "@/utils/maintenanceWindow.js";
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
export interface ICheckProducer {
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
@@ -25,6 +26,7 @@ export class CheckProducer implements ICheckProducer {
private networkService: INetworkService,
private proxyResolver: IProxyResolver,
private bufferService: IBufferService,
private dockerLogsService: IDockerLogsService,
private logger: ILogger
) {}
@@ -81,6 +83,14 @@ export class CheckProducer implements ICheckProducer {
}
// Step 2b: Add to buffer
this.bufferService.addToBuffer(check);
// Step 2c: Handle docker logs
if (status.type === "docker") {
const dockerLogs = await this.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
for (const dockerLog of dockerLogs) {
this.bufferService.addDockerLogToBuffer(dockerLog);
}
}
return { status, check };
};
}
+14 -1
View File
@@ -10,6 +10,7 @@ import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interf
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
export interface IWorkerHelper {
getCleanupOrphanedJob(): () => Promise<void>;
@@ -43,6 +44,7 @@ export class WorkerHelper implements IWorkerHelper {
private checksRepository: IChecksRepository;
private incidentsRepository: IIncidentsRepository;
private geoChecksRepository: IGeoChecksRepository;
private dockerLogsRepository: IDockerLogsRepository;
constructor(
logger: ILogger,
@@ -54,7 +56,8 @@ export class WorkerHelper implements IWorkerHelper {
monitorStatsRepository: IMonitorStatsRepository,
checksRepository: IChecksRepository,
incidentsRepository: IIncidentsRepository,
geoChecksRepository: IGeoChecksRepository
geoChecksRepository: IGeoChecksRepository,
dockerLogsRepository: IDockerLogsRepository
) {
this.logger = logger;
this.checkService = checkService;
@@ -66,6 +69,7 @@ export class WorkerHelper implements IWorkerHelper {
this.checksRepository = checksRepository;
this.incidentsRepository = incidentsRepository;
this.geoChecksRepository = geoChecksRepository;
this.dockerLogsRepository = dockerLogsRepository;
}
getCleanupOrphanedJob = () => {
@@ -142,6 +146,15 @@ export class WorkerHelper implements IWorkerHelper {
});
}
const deletedDockerLogsCount = await this.dockerLogsRepository.deleteByMonitorIdsNotIn(allMonitorIds);
if (deletedDockerLogsCount > 0) {
this.logger.info({
message: `Deleted ${deletedDockerLogsCount} orphaned docker logs`,
service: SERVICE_NAME,
method: "getCleanupOrphanedJob",
});
}
// Remove orphaned jobs
const deletedJobsCount = await this.jobsRepository.deleteByMonitorIdsNotIn(allMonitorIds);
if (deletedJobsCount > 0) {
@@ -128,6 +128,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
const checkProducer = new CheckProducer(
monitorsRepo as any,
@@ -136,6 +137,7 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
networkService as any,
proxyResolver as any,
bufferStub as any,
dockerLogsService as any,
logger
);
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
@@ -5,11 +5,10 @@ import { createMockLogger } from "../../../helpers/createMockLogger.ts";
import { NETWORK_ERROR } from "../../../../src/types/network.ts";
import { AppError } from "../../../../src/utils/AppError.ts";
import type { Monitor } from "../../../../src/domain/monitors/monitor.type.ts";
import { DOCKER_LOG_TAIL_LINES } from "../../../../src/domain/docker/docker.type.ts";
// ── Fixtures ─────────────────────────────────────────────────────────────────
const PRIVATE_KEY = "-----BEGIN OPENSSH PRIVATE KEY-----\nfake\n-----END OPENSSH PRIVATE KEY-----";
const makeMonitor = (overrides?: Partial<Monitor>): Monitor =>
({
id: "mon-1",
@@ -53,10 +52,19 @@ const makeStats = (overrides?: Record<string, any>) => ({
...overrides,
});
const setup = (overrides: Partial<{ ping: any; listContainers: any; getContainer: any; inspect: any; stats: any }> = {}) => {
const makeLogFrame = (streamType: number, text: string): Buffer => {
const payload = Buffer.from(text);
const header = Buffer.alloc(8);
header.writeUInt8(streamType, 0);
header.writeUInt32BE(payload.length, 4);
return Buffer.concat([header, payload]);
};
const setup = (overrides: Partial<{ ping: any; listContainers: any; getContainer: any; inspect: any; stats: any; logs: any }> = {}) => {
const inspect = overrides.inspect ?? jest.fn().mockResolvedValue(makeInspect());
const stats = overrides.stats ?? jest.fn().mockResolvedValue(makeStats());
const getContainer = overrides.getContainer ?? jest.fn().mockReturnValue({ inspect, stats });
const logs = overrides.logs ?? jest.fn().mockResolvedValue(Buffer.alloc(0));
const getContainer = overrides.getContainer ?? jest.fn().mockReturnValue({ inspect, stats, logs });
const instance = {
ping: overrides.ping ?? jest.fn().mockResolvedValue("OK"),
listContainers: overrides.listContainers ?? jest.fn().mockResolvedValue([makeContainer()]),
@@ -65,7 +73,7 @@ const setup = (overrides: Partial<{ ping: any; listContainers: any; getContainer
const DockerLib = jest.fn().mockReturnValue(instance) as any;
const logger = createMockLogger();
const provider = new DockerProvider(logger as any, DockerLib);
return { provider, logger, DockerLib, ping: instance.ping, listContainers: instance.listContainers, getContainer, inspect, stats };
return { provider, logger, DockerLib, ping: instance.ping, listContainers: instance.listContainers, getContainer, inspect, stats, logs };
};
// ── Contract ─────────────────────────────────────────────────────────────────
@@ -106,28 +114,6 @@ describe("DockerProvider", () => {
expect(DockerLib).toHaveBeenCalledWith({ socketPath: "/var/run/docker.sock" });
});
it("parses ssh urls with a private key, defaulting to port 22", async () => {
const { provider, DockerLib } = setup();
await provider.handle(makeMonitor({ url: "ssh://deploy@prod-swarm-01.internal", sshPrivateKey: PRIVATE_KEY }));
expect(DockerLib).toHaveBeenCalledWith({
protocol: "ssh",
host: "prod-swarm-01.internal",
port: 22,
username: "deploy",
sshOptions: { privateKey: PRIVATE_KEY },
});
});
it("parses an explicit ssh port", async () => {
const { provider, DockerLib } = setup();
await provider.handle(makeMonitor({ url: "ssh://deploy@host:2222", sshPrivateKey: PRIVATE_KEY }));
expect(DockerLib).toHaveBeenCalledWith(expect.objectContaining({ port: 2222 }));
});
});
// ── Fail-loudly url validation ───────────────────────────────────────
@@ -141,7 +127,7 @@ describe("DockerProvider", () => {
["a container name (old semantics)", "my-container", "Invalid Docker host URL"],
["tcp engine urls (unsupported)", "tcp://host:2375", "Invalid Docker host URL"],
["https engine urls (unsupported)", "https://host", "Invalid Docker host URL"],
["ssh without a user", "ssh://host", "SSH Docker host URL requires a user: ssh://user@host"],
["ssh engine urls (unsupported)", "ssh://deploy@host", "Invalid Docker host URL"],
];
it.each(invalidUrls)("throws AppError for %s and never constructs a client", async (_label, url, message) => {
@@ -151,15 +137,6 @@ describe("DockerProvider", () => {
expect(DockerLib).not.toHaveBeenCalled();
});
it("throws AppError for an ssh url with no private key", async () => {
const { provider, DockerLib } = setup();
await expect(provider.handle(makeMonitor({ url: "ssh://deploy@host", sshPrivateKey: undefined }))).rejects.toThrow(
"SSH Docker host requires a private key"
);
expect(DockerLib).not.toHaveBeenCalled();
});
it("preserves the toDockerOptions AppError through handle's catch", async () => {
const { provider } = setup();
@@ -278,6 +255,83 @@ describe("DockerProvider", () => {
});
});
describe("container logs", () => {
it("parses multiplexed stdout and stderr with normalized timestamps", async () => {
const logs = jest
.fn()
.mockResolvedValue(
Buffer.concat([makeLogFrame(1, "2026-01-01T00:00:37.1Z ready\n"), makeLogFrame(2, "2026-01-01T00:00:38.123Z failed\r\n")])
);
const { provider } = setup({ logs });
const result = await provider.handle(makeMonitor({ dockerLogsEnabled: true }));
expect(result.payload?.logs?.[0]?.lines).toEqual([
{ ts: "2026-01-01T00:00:37.100000000Z", stream: "stdout", text: "ready" },
{ ts: "2026-01-01T00:00:38.123000000Z", stream: "stderr", text: "failed" },
]);
});
it("treats raw TTY output as stdout and drops lines without timestamps", async () => {
const logs = jest.fn().mockResolvedValue(Buffer.from("noise\n2026-01-01T00:00:00Z first\n2026-01-01T00:00:01.12Z second\n"));
const { provider } = setup({ logs });
const result = await provider.handle(makeMonitor({ dockerLogsEnabled: true }));
expect(result.payload?.logs?.[0]?.lines).toEqual([
{ ts: "2026-01-01T00:00:00.000000000Z", stream: "stdout", text: "first" },
{ ts: "2026-01-01T00:00:01.120000000Z", stream: "stdout", text: "second" },
]);
});
it("truncates oversized UTF-8 lines and appends the marker", async () => {
const logs = jest.fn().mockResolvedValue(Buffer.from(`2026-01-01T00:00:00Z ${"x".repeat(5000)}\n`));
const { provider } = setup({ logs });
const result = await provider.handle(makeMonitor({ dockerLogsEnabled: true }));
const text = result.payload?.logs?.[0]?.lines[0]?.text;
expect(text).toEqual(expect.stringMatching(/ …\[truncated\]$/));
expect(Buffer.byteLength(text?.replace(" …[truncated]", "") ?? "", "utf8")).toBeLessThanOrEqual(4096);
});
it("isolates a logs failure and warns", async () => {
const { provider, logger } = setup({ logs: jest.fn().mockRejectedValue(new Error("logs failed")) });
const result = await provider.handle(makeMonitor({ dockerLogsEnabled: true }));
expect(result.payload?.logs).toEqual([]);
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to read logs for container my-container" }));
});
it("requests a bounded timestamped tail for every container state", async () => {
const containers = [makeContainer({ Id: "a".repeat(64) }), makeContainer({ Id: "b".repeat(64), State: "exited" })];
const { provider, logs } = setup({ listContainers: jest.fn().mockResolvedValue(containers) });
await provider.handle(makeMonitor({ dockerLogsEnabled: true }));
expect(logs).toHaveBeenCalledTimes(2);
expect(logs).toHaveBeenCalledWith({ follow: false, stdout: true, stderr: true, timestamps: true, tail: DOCKER_LOG_TAIL_LINES });
});
it("does not read logs when dockerLogsEnabled is false", async () => {
const { provider, logs } = setup();
const result = await provider.handle(makeMonitor({ dockerLogsEnabled: false }));
expect(logs).not.toHaveBeenCalled();
expect(result.payload?.logs).toBeUndefined();
});
it("does not read logs when dockerLogsEnabled is unset", async () => {
const { provider, logs } = setup();
const result = await provider.handle(makeMonitor());
expect(logs).not.toHaveBeenCalled();
expect(result.payload?.logs).toBeUndefined();
});
});
// ── Enrichment fault isolation ───────────────────────────────────────
describe("enrichment fault isolation", () => {
+106 -7
View File
@@ -7,6 +7,8 @@ import type { ISettingsService } from "../../../src/domain/app-settings/app-sett
import type { IJobsRepository } from "../../../src/domain/jobs/job.repository.interface.ts";
import type { Check } from "../../../src/domain/checks/check.type.ts";
import type { GeoCheck } from "../../../src/domain/geo-checks/geo-check.type.ts";
import type { IDockerLogsService } from "../../../src/domain/docker/docker-log.service.ts";
import type { DockerLog } from "../../../src/domain/docker/docker-log.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -20,6 +22,11 @@ const createMockGeoChecksService = () =>
createGeoChecks: jest.fn().mockResolvedValue([]),
}) as unknown as jest.Mocked<IGeoChecksService>;
const createMockDockerLogsService = () =>
({
createDockerLogs: jest.fn().mockResolvedValue(0),
}) as unknown as jest.Mocked<IDockerLogsService>;
const createMockSettingsService = (nodeEnv: string = "development") =>
({
getSettings: jest.fn().mockReturnValue({ nodeEnv }),
@@ -48,14 +55,28 @@ const makeGeoCheck = (overrides?: Partial<GeoCheck>): GeoCheck =>
...overrides,
}) as GeoCheck;
const makeDockerLog = (overrides?: Partial<DockerLog>): DockerLog =>
({
id: "docker-log-1",
metadata: { monitorId: "mon-1", teamId: "team-1", containerId: "container-1", containerName: "web" },
lines: [{ ts: "2026-01-01T00:00:00.000000000Z", stream: "stdout", text: "ready" }],
gap: false,
checkedAt: "2026-01-01T00:00:01.000Z",
expiry: "2026-01-08T00:00:01.000Z",
createdAt: "2026-01-01T00:00:01.000Z",
updatedAt: "2026-01-01T00:00:01.000Z",
...overrides,
}) as DockerLog;
const createService = (nodeEnv: string = "development") => {
const logger = createMockLogger();
const checkService = createMockCheckService();
const geoChecksService = createMockGeoChecksService();
const dockerLogsService = createMockDockerLogsService();
const settingsService = createMockSettingsService(nodeEnv);
const jobsRepository = createMockJobsRepository();
const service = new BufferService(logger as any, checkService, geoChecksService, settingsService, jobsRepository);
return { service, logger, checkService, geoChecksService, jobsRepository };
const service = new BufferService(logger as any, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
return { service, logger, checkService, geoChecksService, dockerLogsService, jobsRepository };
};
// ── Tests ────────────────────────────────────────────────────────────────────
@@ -209,6 +230,27 @@ describe("BufferService", () => {
});
});
describe("addDockerLogToBuffer", () => {
it("adds a docker log to the buffer", async () => {
const { service, dockerLogsService } = createService();
const dockerLog = makeDockerLog();
service.addDockerLogToBuffer(dockerLog);
await service.flushDockerLogsBuffer();
expect(dockerLogsService.createDockerLogs).toHaveBeenCalledWith([dockerLog]);
});
it("logs an error if push throws", () => {
const { service, logger } = createService();
Object.defineProperty(service, "dockerLogBuffer", { value: Object.freeze([]) });
service.addDockerLogToBuffer(makeDockerLog());
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "addDockerLogToBuffer" }));
});
});
// ── scheduleNextFlush ────────────────────────────────────────────────
describe("scheduleNextFlush", () => {
@@ -223,15 +265,17 @@ describe("BufferService", () => {
expect(jest.getTimerCount()).toBe(initialTimerCount);
});
it("flushes buffer and geo buffer when timer fires", async () => {
const { service, checkService, geoChecksService } = createService();
it("flushes all buffers when timer fires", async () => {
const { service, checkService, geoChecksService, dockerLogsService } = createService();
service.addToBuffer(makeCheck());
service.addGeoCheckToBuffer(makeGeoCheck());
service.addDockerLogToBuffer(makeDockerLog());
await jest.advanceTimersByTimeAsync(1000);
expect(checkService.createChecks).toHaveBeenCalled();
expect(geoChecksService.createGeoChecks).toHaveBeenCalled();
expect(dockerLogsService.createDockerLogs).toHaveBeenCalled();
});
it("reschedules after flush completes", async () => {
@@ -472,18 +516,72 @@ describe("BufferService", () => {
});
});
describe("flushDockerLogsBuffer", () => {
it("does nothing when the docker log buffer is empty", async () => {
const { service, dockerLogsService } = createService();
await service.flushDockerLogsBuffer();
expect(dockerLogsService.createDockerLogs).not.toHaveBeenCalled();
});
it("flushes the batch and clears it", async () => {
const { service, dockerLogsService } = createService();
const dockerLog = makeDockerLog();
service.addDockerLogToBuffer(dockerLog);
await service.flushDockerLogsBuffer();
await service.flushDockerLogsBuffer();
expect(dockerLogsService.createDockerLogs).toHaveBeenCalledWith([dockerLog]);
expect(dockerLogsService.createDockerLogs).toHaveBeenCalledTimes(1);
});
it("does not drop logs added while a flush is in flight", async () => {
const { service, dockerLogsService } = createService();
let resolveWrite!: (value: number) => void;
(dockerLogsService.createDockerLogs as jest.Mock).mockImplementationOnce(() => new Promise<number>((resolve) => (resolveWrite = resolve)));
const first = makeDockerLog({ id: "docker-log-1" });
const second = makeDockerLog({ id: "docker-log-2" });
service.addDockerLogToBuffer(first);
const flush = service.flushDockerLogsBuffer();
service.addDockerLogToBuffer(second);
resolveWrite(1);
await flush;
await service.flushDockerLogsBuffer();
expect(dockerLogsService.createDockerLogs).toHaveBeenNthCalledWith(1, [first]);
expect(dockerLogsService.createDockerLogs).toHaveBeenNthCalledWith(2, [second]);
});
it("drops a failed batch and logs the error", async () => {
const { service, dockerLogsService, logger } = createService();
(dockerLogsService.createDockerLogs as jest.Mock).mockRejectedValueOnce(new Error("DB error"));
service.addDockerLogToBuffer(makeDockerLog());
await service.flushDockerLogsBuffer();
await service.flushDockerLogsBuffer();
expect(dockerLogsService.createDockerLogs).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "DB error", method: "flushDockerLogsBuffer" }));
});
});
// ── shutdown ─────────────────────────────────────────────────────────────
describe("shutdown", () => {
it("stops the flush timer and flushes both buffers", async () => {
const { service, checkService, geoChecksService } = createService();
it("stops the flush timer and flushes all buffers", async () => {
const { service, checkService, geoChecksService, dockerLogsService } = createService();
service.addToBuffer(makeCheck());
service.addGeoCheckToBuffer(makeGeoCheck());
service.addDockerLogToBuffer(makeDockerLog());
await service.shutdown();
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(geoChecksService.createGeoChecks).toHaveBeenCalledTimes(1);
expect(dockerLogsService.createDockerLogs).toHaveBeenCalledTimes(1);
// Timer is cleared: advancing past the flush interval triggers no further flush.
await jest.advanceTimersByTimeAsync(60 * 1000);
@@ -491,12 +589,13 @@ describe("BufferService", () => {
});
it("is safe to call with empty buffers", async () => {
const { service, checkService, geoChecksService } = createService();
const { service, checkService, geoChecksService, dockerLogsService } = createService();
await service.shutdown();
expect(checkService.createChecks).not.toHaveBeenCalled();
expect(geoChecksService.createGeoChecks).not.toHaveBeenCalled();
expect(dockerLogsService.createDockerLogs).not.toHaveBeenCalled();
});
});
});
@@ -28,6 +28,7 @@ const createProducer = (overrides?: Record<string, any>) => {
networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) },
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
buffer: { addToBuffer: jest.fn() },
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue([]) },
...overrides,
};
const producer = new CheckProducer(
@@ -37,6 +38,7 @@ const createProducer = (overrides?: Record<string, any>) => {
defaults.networkService as any,
defaults.proxyResolver as any,
defaults.buffer as any,
defaults.dockerLogsService as any,
defaults.logger as any
);
return { producer, defaults };
@@ -142,4 +144,28 @@ describe("CheckProducer", () => {
expect(defaults.buffer.addToBuffer).toHaveBeenCalledWith(check);
expect(result).toEqual({ status, check });
});
it("builds and buffers docker logs for a docker status", async () => {
const status = { type: "docker", monitorId: "m1", teamId: "team", status: true, code: 200, message: "OK", payload: {} };
const dockerLogs = [{ id: "docker-log-1" }, { id: "docker-log-2" }];
const { producer, defaults } = createProducer({
networkService: { requestStatus: jest.fn().mockResolvedValue(status) },
buffer: { addToBuffer: jest.fn(), addDockerLogToBuffer: jest.fn() },
dockerLogsService: { buildDockerLogs: jest.fn().mockResolvedValue(dockerLogs) },
});
await producer.produce(makeMonitor({ type: "docker" }));
expect(defaults.dockerLogsService.buildDockerLogs).toHaveBeenCalledWith(status);
expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(1, dockerLogs[0]);
expect(defaults.buffer.addDockerLogToBuffer).toHaveBeenNthCalledWith(2, dockerLogs[1]);
});
it("does not build docker logs for a non-docker status", async () => {
const { producer, defaults } = createProducer();
await producer.produce(makeMonitor());
expect(defaults.dockerLogsService.buildDockerLogs).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,87 @@
import { describe, expect, it, jest } from "@jest/globals";
import { DockerLogsService } from "../../../src/domain/docker/docker-log.service.ts";
import { DOCKER_LOG_TAIL_LINES, type DockerLogLine } from "../../../src/domain/docker/docker.type.ts";
import type { MonitorStatusResponse, DockerStatusPayload } from "../../../src/types/network.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
const line = (second: number): DockerLogLine => ({
ts: `2026-01-01T00:00:${String(second).padStart(2, "0")}.000000000Z`,
stream: "stdout",
text: `line ${second}`,
});
const status = (logs?: DockerStatusPayload["logs"]): MonitorStatusResponse<DockerStatusPayload> => ({
monitorId: "monitor-1",
teamId: "team-1",
type: "docker",
status: true,
code: 200,
message: "OK",
payload: { containers: [], summary: { total: 0, running: 0, stopped: 0, unhealthy: 0 }, logs },
});
const createService = () => {
const repository = {
findLastLineTimestamp: jest.fn().mockResolvedValue(null),
createDockerLogs: jest.fn().mockResolvedValue(0),
};
const logger = createMockLogger();
return { service: new DockerLogsService({ logger: logger as any, dockerLogsRepository: repository as any }), repository, logger };
};
describe("DockerLogsService", () => {
it("returns no documents for missing or empty logs", async () => {
const { service, repository } = createService();
expect(await service.buildDockerLogs(status())).toEqual([]);
expect(await service.buildDockerLogs(status([]))).toEqual([]);
expect(repository.findLastLineTimestamp).not.toHaveBeenCalled();
});
it("seeds the cursor, drops old lines, and sets seven-day expiry", async () => {
const { service, repository } = createService();
repository.findLastLineTimestamp.mockResolvedValue(line(1).ts);
const result = await service.buildDockerLogs(status([{ containerId: "container-1", containerName: "web", lines: [line(1), line(2)] }]));
expect(repository.findLastLineTimestamp).toHaveBeenCalledWith("monitor-1", "container-1");
expect(result[0]?.lines).toEqual([line(2)]);
expect(new Date(result[0]!.expiry).getTime() - new Date(result[0]!.checkedAt).getTime()).toBe(7 * 24 * 60 * 60 * 1000);
});
it("advances the cursor in memory and does not query again", async () => {
const { service, repository } = createService();
const logs = [{ containerId: "container-1", containerName: "web", lines: [line(1)] }];
expect(await service.buildDockerLogs(status(logs))).toHaveLength(1);
expect(await service.buildDockerLogs(status(logs))).toEqual([]);
expect(repository.findLastLineTimestamp).toHaveBeenCalledTimes(1);
});
it("sets gap only when every tail line is fresh", async () => {
const { service } = createService();
const lines = Array.from({ length: DOCKER_LOG_TAIL_LINES }, (_, index) => ({
ts: `2026-01-01T00:00:00.${String(index).padStart(9, "0")}Z`,
stream: "stdout" as const,
text: String(index),
}));
const [result] = await service.buildDockerLogs(status([{ containerId: "container-1", containerName: "web", lines }]));
expect(result?.gap).toBe(true);
});
it("skips a failed container and continues with the others", async () => {
const { service, repository, logger } = createService();
repository.findLastLineTimestamp.mockRejectedValueOnce(new Error("DB down")).mockResolvedValueOnce(null);
const result = await service.buildDockerLogs(
status([
{ containerId: "bad", containerName: "bad-container", lines: [line(1)] },
{ containerId: "good", containerName: "good-container", lines: [line(2)] },
])
);
expect(result).toHaveLength(1);
expect(result[0]?.metadata.containerId).toBe("good");
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: "Failed to build docker logs for container bad-container" }));
});
});
@@ -2,6 +2,7 @@ import { describe, expect, it, jest, beforeEach } from "@jest/globals";
import { MonitorService } from "../../../src/domain/monitors/monitor.service.ts";
import type { IChecksRepository } from "../../../src/domain/checks/check.repository.interface.ts";
import type { IGeoChecksRepository } from "../../../src/domain/geo-checks/geo-check.repository.interface.ts";
import type { IDockerLogsRepository } from "../../../src/domain/docker/docker-log.repository.interface.ts";
import type { IIncidentsRepository } from "../../../src/domain/incidents/incident.repository.interface.ts";
import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts";
import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts";
@@ -50,6 +51,12 @@ const createGeoChecksRepositoryMock = () =>
deleteByMonitorId: jest.fn(),
}) as unknown as IGeoChecksRepository;
const createDockerLogsRepositoryMock = () =>
({
findByContainerName: jest.fn(),
deleteByMonitorId: jest.fn().mockResolvedValue(0),
}) as unknown as IDockerLogsRepository;
const createIncidentsRepositoryMock = () =>
({
deleteByMonitorId: jest.fn(),
@@ -89,6 +96,7 @@ const createService = (
monitorStatsRepository?: IMonitorStatsRepository;
statusPagesRepository?: IStatusPagesRepository;
geoChecksRepository?: IGeoChecksRepository;
dockerLogsRepository?: IDockerLogsRepository;
incidentsRepository?: IIncidentsRepository;
jobQueue?: ReturnType<typeof createJobQueueMock>;
logger?: ReturnType<typeof createMockLogger>;
@@ -104,6 +112,7 @@ const createService = (
monitorsRepository: overrides.monitorsRepository ?? createMonitorsRepositoryMock(),
checksRepository: overrides.checksRepository ?? createChecksRepositoryMock(),
geoChecksRepository: overrides.geoChecksRepository ?? createGeoChecksRepositoryMock(),
dockerLogsRepository: overrides.dockerLogsRepository ?? createDockerLogsRepositoryMock(),
monitorStatsRepository: overrides.monitorStatsRepository ?? createMonitorStatsRepositoryMock(),
statusPagesRepository: overrides.statusPagesRepository ?? createStatusPagesRepositoryMock(),
incidentsRepository: overrides.incidentsRepository ?? createIncidentsRepositoryMock(),
@@ -594,6 +603,83 @@ describe("MonitorService", () => {
});
});
describe("getDockerContainerLogs", () => {
it("throws 404 when the monitor is not found", async () => {
const monitorsRepository = createMonitorsRepositoryMock();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(null);
const { service } = createService({ monitorsRepository });
await expect(service.getDockerContainerLogs({ teamId: TEAM_ID, monitorId: "missing", containerName: "web", limit: 20 })).rejects.toThrow(
"Monitor with ID missing not found"
);
});
it("throws 400 for a non-docker monitor", async () => {
const monitorsRepository = createMonitorsRepositoryMock();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(makeMonitor({ type: "http" }));
const { service } = createService({ monitorsRepository });
await expect(service.getDockerContainerLogs({ teamId: TEAM_ID, monitorId: MONITOR_ID, containerName: "web", limit: 20 })).rejects.toThrow(
"monitors are not supported for docker container logs"
);
});
it("passes pagination through and returns the last checkedAt as a full-page cursor", async () => {
const monitorsRepository = createMonitorsRepositoryMock();
const dockerLogsRepository = createDockerLogsRepositoryMock();
const before = new Date("2026-01-02T00:00:00.000Z");
const logs = [{ checkedAt: "2026-01-01T02:00:00.000Z" }, { checkedAt: "2026-01-01T01:00:00.000Z" }];
(monitorsRepository.findById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" }));
(dockerLogsRepository.findByContainerName as jest.Mock).mockResolvedValue(logs);
const { service } = createService({ monitorsRepository, dockerLogsRepository });
const result = await service.getDockerContainerLogs({
teamId: TEAM_ID,
monitorId: MONITOR_ID,
containerName: "web",
before,
limit: 2,
});
expect(dockerLogsRepository.findByContainerName).toHaveBeenCalledWith({ monitorId: MONITOR_ID, containerName: "web", before, limit: 2 });
expect(result.nextCursor).toBe("2026-01-01T01:00:00.000Z");
});
it("returns a null cursor for a short page", async () => {
const monitorsRepository = createMonitorsRepositoryMock();
const dockerLogsRepository = createDockerLogsRepositoryMock();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" }));
(dockerLogsRepository.findByContainerName as jest.Mock).mockResolvedValue([{ checkedAt: "2026-01-01T01:00:00.000Z" }]);
const { service } = createService({ monitorsRepository, dockerLogsRepository });
const result = await service.getDockerContainerLogs({ teamId: TEAM_ID, monitorId: MONITOR_ID, containerName: "web", limit: 2 });
expect(result.nextCursor).toBeNull();
});
it("passes after through and returns a null cursor even for a full page", async () => {
const monitorsRepository = createMonitorsRepositoryMock();
const dockerLogsRepository = createDockerLogsRepositoryMock();
const after = new Date("2026-01-01T00:00:00.000Z");
const logs = [{ checkedAt: "2026-01-01T02:00:00.000Z" }, { checkedAt: "2026-01-01T01:00:00.000Z" }];
(monitorsRepository.findById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" }));
(dockerLogsRepository.findByContainerName as jest.Mock).mockResolvedValue(logs);
const { service } = createService({ monitorsRepository, dockerLogsRepository });
const result = await service.getDockerContainerLogs({
teamId: TEAM_ID,
monitorId: MONITOR_ID,
containerName: "web",
after,
limit: 2,
});
expect(dockerLogsRepository.findByContainerName).toHaveBeenCalledWith({ monitorId: MONITOR_ID, containerName: "web", after, limit: 2 });
expect(result.logs).toBe(logs);
expect(result.nextCursor).toBeNull();
});
});
describe("getPageSpeedDetailsById", () => {
it("returns pagespeed details for a pagespeed monitor", async () => {
const monitorsRepository = createMonitorsRepositoryMock();
@@ -1802,6 +1888,7 @@ describe("MonitorService", () => {
monitorsRepository,
checksRepository,
geoChecksRepository,
dockerLogsRepository: createDockerLogsRepositoryMock(),
monitorStatsRepository,
statusPagesRepository,
incidentsRepository,
@@ -32,6 +32,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
const geoChecksRepository = {
deleteByMonitorIdsNotIn: jest.fn().mockResolvedValue(0),
};
const dockerLogsRepository = {
deleteByMonitorIdsNotIn: jest.fn().mockResolvedValue(0),
};
const settingsService = {
getDBSettings: jest.fn().mockResolvedValue({ checkTTL: 30 }),
};
@@ -50,6 +53,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
checksRepository,
incidentsRepository,
geoChecksRepository,
dockerLogsRepository,
...overrides,
};
@@ -63,7 +67,8 @@ const createHelper = (overrides?: Record<string, unknown>) => {
defaults.monitorStatsRepository as any,
defaults.checksRepository as any,
defaults.incidentsRepository as any,
defaults.geoChecksRepository as any
defaults.geoChecksRepository as any,
defaults.dockerLogsRepository as any
);
return { helper, defaults };
};
@@ -81,6 +86,7 @@ describe("WorkerHelper", () => {
(defaults.checksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(4);
(defaults.incidentsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(1);
(defaults.geoChecksRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(5);
(defaults.dockerLogsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(7);
(defaults.jobsRepository.deleteByMonitorIdsNotIn as jest.Mock).mockResolvedValue(6);
const job = helper.getCleanupOrphanedJob();
@@ -93,12 +99,14 @@ describe("WorkerHelper", () => {
expect(defaults.checksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.incidentsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.geoChecksRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.dockerLogsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.jobsRepository.deleteByMonitorIdsNotIn).toHaveBeenCalledWith(["m1"]);
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("2 orphaned monitors") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("3 orphaned monitor stats") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("4 orphaned checks") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("1 orphaned incidents") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("5 orphaned geo checks") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("7 orphaned docker logs") }));
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("6 orphaned jobs") }));
});
@@ -3,8 +3,31 @@ import {
createMonitorBodyValidation,
editMonitorBodyValidation,
importMonitorsBodyValidation,
getDockerContainerLogsQueryValidation,
} from "../../../src/api/validation/monitorValidation.ts";
describe("getDockerContainerLogsQueryValidation", () => {
it("defaults limit to 20", () => {
expect(getDockerContainerLogsQueryValidation.parse({})).toEqual({ limit: 20 });
});
it("rejects a limit above 50", () => {
expect(() => getDockerContainerLogsQueryValidation.parse({ limit: 51 })).toThrow();
});
it("rejects a non-ISO before cursor", () => {
expect(() => getDockerContainerLogsQueryValidation.parse({ before: "yesterday" })).toThrow();
});
it("rejects a non-ISO after cursor", () => {
expect(() => getDockerContainerLogsQueryValidation.parse({ after: "yesterday" })).toThrow();
});
it("rejects before and after together", () => {
expect(() => getDockerContainerLogsQueryValidation.parse({ before: "2026-01-02T00:00:00.000Z", after: "2026-01-01T00:00:00.000Z" })).toThrow();
});
});
const baseDnsBody = {
name: "DNS check",
type: "dns" as const,
@@ -210,6 +233,19 @@ describe("monitorValidation — strategy gating", () => {
expect(parsed.monitors[0].strategy).toBeUndefined();
});
it("defaults dockerLogsEnabled to false on imported docker monitors", () => {
const parsed = importMonitorsBodyValidation.parse({
monitors: [
{
name: "Imported Docker",
type: "docker",
url: "unix:///var/run/docker.sock",
},
],
});
expect(parsed.monitors[0].dockerLogsEnabled).toBe(false);
});
it("rejects strategy on imported non-pagespeed monitors", () => {
expect(() =>
importMonitorsBodyValidation.parse({
@@ -572,7 +608,6 @@ describe("monitorValidation — Docker host url", () => {
type: "docker" as const,
url: "unix:///var/run/docker.sock",
};
const PRIVATE_KEY = "-----BEGIN OPENSSH PRIVATE KEY-----\nfake\n-----END OPENSSH PRIVATE KEY-----";
describe("createMonitorBodyValidation", () => {
it("accepts the default unix socket url", () => {
@@ -585,32 +620,15 @@ describe("monitorValidation — Docker host url", () => {
expect(parsed.url).toBe("/run/docker.sock");
});
it("accepts an ssh url with a private key and retains the key", () => {
for (const url of ["ssh://deploy@host", "ssh://deploy@host:2222"]) {
const parsed = createMonitorBodyValidation.parse({ ...baseDockerBody, url, sshPrivateKey: PRIVATE_KEY });
expect(parsed.sshPrivateKey).toBe(PRIVATE_KEY);
}
});
it("rejects container names and unsupported engine urls", () => {
for (const badUrl of ["my-container", "tcp://host:2375", "https://host", "unix://relative/path", ""]) {
for (const badUrl of ["my-container", "tcp://host:2375", "https://host", "ssh://deploy@host", "unix://relative/path", ""]) {
expect(() => createMonitorBodyValidation.parse({ ...baseDockerBody, url: badUrl })).toThrow();
}
});
it("rejects an ssh url without a user", () => {
expect(() => createMonitorBodyValidation.parse({ ...baseDockerBody, url: "ssh://host", sshPrivateKey: PRIVATE_KEY })).toThrow();
});
it("rejects an ssh url without a private key, attributing the issue to sshPrivateKey", () => {
const result = createMonitorBodyValidation.safeParse({ ...baseDockerBody, url: "ssh://deploy@host" });
expect(result.success).toBe(false);
expect(result.error?.issues.some((issue) => issue.path.includes("sshPrivateKey"))).toBe(true);
});
it("does not apply docker url rules to other monitor types", () => {
const parsed = createMonitorBodyValidation.parse({ name: "HTTP check", type: "http", url: "https://example.com" });
expect(parsed.sshPrivateKey).toBeUndefined();
expect(parsed.url).toBe("https://example.com");
});
});
@@ -624,7 +642,7 @@ describe("monitorValidation — Docker host url", () => {
expect(() => editMonitorBodyValidation.parse({ type: "docker", url: "my-container" })).toThrow();
});
it("rejects a docker edit switching to ssh without a key", () => {
it("rejects a docker edit with an ssh url", () => {
expect(() => editMonitorBodyValidation.parse({ type: "docker", url: "ssh://deploy@host" })).toThrow();
});
});