Compare commits
48
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
10e127d0db | ||
|
|
a5e66318b5 | ||
|
|
7ebbb05482 | ||
|
|
99c62961cf | ||
|
|
40a9f3f1e1 | ||
|
|
b37e4d9083 | ||
|
|
37dd9fad26 | ||
|
|
0f51d68ff0 | ||
|
|
dfc048522d | ||
|
|
1f9072a905 | ||
|
|
780ed8147b | ||
|
|
b2cfc60387 | ||
|
|
c8f84e9e20 | ||
|
|
9434cd8929 | ||
|
|
55184724d0 | ||
|
|
de4c528d55 | ||
|
|
d5e887903e | ||
|
|
6196b81fe4 | ||
|
|
c3598a67fa | ||
|
|
2c896a5506 | ||
|
|
97e23e4b3a | ||
|
|
be39ad0bfe | ||
|
|
7f0e577dd2 | ||
|
|
f308325bd6 | ||
|
|
c5303add56 | ||
|
|
fe6f755451 | ||
|
|
4a2429d316 | ||
|
|
d7f8bcc9e1 | ||
|
|
d51fdbd772 | ||
|
|
199d68e8a2 | ||
|
|
461a5e35dd | ||
|
|
8dd57d45e2 | ||
|
|
01e9655f42 | ||
|
|
d4d2735d23 | ||
|
|
253642dde9 | ||
|
|
2f3de9797d | ||
|
|
1b3b8e5312 | ||
|
|
b55560e2f1 | ||
|
|
57ddbc474f | ||
|
|
862ff15eef | ||
|
|
13799af721 | ||
|
|
4fcb13694b | ||
|
|
3dc529733a | ||
|
|
d865103fb5 | ||
|
|
4335e6b7b6 | ||
|
|
4f5dba560d | ||
|
|
f52acb3f68 | ||
|
|
2632f98ad0 |
No files matched your search
@@ -134,14 +134,17 @@ If you need to monitor internal HTTPS endpoints with certificates from private C
|
||||
|
||||
### Docker monitors
|
||||
|
||||
A Docker monitor connects to a Docker daemon and reports on every container it runs. The daemon's ping response decides whether the monitor is up or down and its latency is the response time. Each check also records every container's state, health, CPU and memory usage, restart count, published ports and mounts. Enabling **Collect container logs** additionally stores the latest 200 log lines per container on every check; logs are kept for 7 days.
|
||||
A Docker monitor reports on every container running on a host. It can connect to the Docker daemon directly or read container metrics from a Capture agent. Direct monitors use the daemon's ping response for availability and record container state, health, CPU and memory usage, restart count, published ports and mounts. Enabling **Collect container logs** additionally stores the latest 200 log lines per container on every direct check; logs are kept for 7 days.
|
||||
|
||||
The **Docker host** field accepts two forms:
|
||||
The **Docker host** field accepts three forms:
|
||||
|
||||
| Host | Example | Notes |
|
||||
|---|---|---|
|
||||
| Local socket | `unix:///var/run/docker.sock` | Also accepts a bare absolute path such as `/var/run/docker.sock`. Use this for the daemon on the same machine Checkmate runs on. |
|
||||
| Remote daemon | `tcp://docker.example.com:2376` | Always uses mutual TLS; the port defaults to `2376`. Unencrypted daemons on `2375` are not supported. |
|
||||
| Capture agent | `https://capture.example.com/api/v1/metrics/docker` | Enter the full endpoint ending in `/metrics/docker` and its authorization secret. Checkmate requests `all=true` so stopped containers are included. |
|
||||
|
||||
Capture-backed monitors normalize Capture's container state, health, CPU, memory, start time and exposed ports into the same Docker pages. Capture does not currently provide restart counts, mounts, published host-port bindings or container logs, so those values are shown as unavailable. Capture also reports a container whose health check is still starting as healthy, so the `starting` health state only appears on direct monitors, and stopped containers carry no CPU or memory metrics.
|
||||
|
||||
**Monitoring the local socket.** The reference Compose file does not mount the socket, so add it and grant the container the host's `docker` group. The image runs as an unprivileged user and cannot read the socket otherwise. Find the group id with `stat -c %g /var/run/docker.sock`, then:
|
||||
|
||||
|
||||
Generated
+2
-22
@@ -93,7 +93,6 @@
|
||||
"resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz",
|
||||
"integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.29.7",
|
||||
"@babel/generator": "^7.29.7",
|
||||
@@ -409,7 +408,6 @@
|
||||
"node_modules/@emotion/react": {
|
||||
"version": "11.14.0",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.18.3",
|
||||
"@emotion/babel-plugin": "^11.13.5",
|
||||
@@ -447,7 +445,6 @@
|
||||
"node_modules/@emotion/styled": {
|
||||
"version": "11.14.1",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.18.3",
|
||||
"@emotion/babel-plugin": "^11.13.5",
|
||||
@@ -1236,7 +1233,6 @@
|
||||
"node_modules/@mui/material": {
|
||||
"version": "7.3.7",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.28.4",
|
||||
"@mui/core-downloads-tracker": "^7.3.7",
|
||||
@@ -1341,7 +1337,6 @@
|
||||
"node_modules/@mui/system": {
|
||||
"version": "7.3.7",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.28.4",
|
||||
"@mui/private-theming": "^7.3.7",
|
||||
@@ -2386,7 +2381,6 @@
|
||||
"version": "24.5.2",
|
||||
"devOptional": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~7.12.0"
|
||||
}
|
||||
@@ -2402,7 +2396,6 @@
|
||||
"node_modules/@types/react": {
|
||||
"version": "18.3.27",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@types/prop-types": "*",
|
||||
"csstype": "^3.2.2"
|
||||
@@ -2493,7 +2486,6 @@
|
||||
"integrity": "sha512-4Z+L8I2OqhZV8qA132M4wNL30ypZGYOQVBfMgxDH/K5UX0PNqTu1c6za9ST5r9+tavvHiTWmBnKzpCJ/GlVFtg==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/scope-manager": "7.18.0",
|
||||
"@typescript-eslint/types": "7.18.0",
|
||||
@@ -2774,7 +2766,6 @@
|
||||
"version": "8.15.0",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
@@ -3131,7 +3122,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"baseline-browser-mapping": "^2.10.42",
|
||||
"caniuse-lite": "^1.0.30001800",
|
||||
@@ -3575,8 +3565,7 @@
|
||||
},
|
||||
"node_modules/dayjs": {
|
||||
"version": "1.11.13",
|
||||
"license": "MIT",
|
||||
"peer": true
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
@@ -3975,7 +3964,6 @@
|
||||
"version": "8.57.1",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@eslint-community/eslint-utils": "^4.2.0",
|
||||
"@eslint-community/regexpp": "^4.6.1",
|
||||
@@ -4788,7 +4776,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.27.6"
|
||||
},
|
||||
@@ -5600,7 +5587,6 @@
|
||||
"resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-5.19.0.tgz",
|
||||
"integrity": "sha512-REhYUN8gNP3HlcIZS6QU2uy8iovl31cXsrNDkCcqWSQbCkcpdYLczqDz5PVIwNH42UQNyvukjes/RoHPDrOUmQ==",
|
||||
"license": "BSD-3-Clause",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@mapbox/geojson-rewind": "^0.5.2",
|
||||
"@mapbox/jsonlint-lines-primitives": "^2.0.2",
|
||||
@@ -6382,7 +6368,6 @@
|
||||
"node_modules/react-dom": {
|
||||
"version": "18.3.1",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"loose-envify": "^1.1.0",
|
||||
"scheduler": "^0.23.2"
|
||||
@@ -6394,7 +6379,6 @@
|
||||
"node_modules/react-hook-form": {
|
||||
"version": "7.71.1",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
@@ -6598,8 +6582,7 @@
|
||||
},
|
||||
"node_modules/redux": {
|
||||
"version": "5.0.1",
|
||||
"license": "MIT",
|
||||
"peer": true
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/redux-persist": {
|
||||
"version": "6.0.0",
|
||||
@@ -6749,7 +6732,6 @@
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz",
|
||||
"integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@types/estree": "1.0.9"
|
||||
},
|
||||
@@ -7591,7 +7573,6 @@
|
||||
"version": "5.9.3",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
@@ -7729,7 +7710,6 @@
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz",
|
||||
"integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"esbuild": "^0.25.0",
|
||||
"fdir": "^6.4.4",
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import { dashboardCardKeys, type DashboardCardKey } from "@/Types/Dashboard";
|
||||
import { createSlice, type PayloadAction } from "@reduxjs/toolkit";
|
||||
|
||||
type ThemeMode = "light" | "dark";
|
||||
type ChartType = "histogram" | "line";
|
||||
|
||||
export const dashboardCardKeys = ["monitorStatus", "monitorsByType"] as const;
|
||||
export type DashboardCardKey = (typeof dashboardCardKeys)[number];
|
||||
|
||||
type TableName =
|
||||
| "monitors"
|
||||
| "team"
|
||||
|
||||
@@ -77,6 +77,7 @@ export const getMonitorDefaults = (
|
||||
...base,
|
||||
type: "docker",
|
||||
url: data?.url || "",
|
||||
secret: data?.secret || "",
|
||||
ignoreTlsErrors: data?.ignoreTlsErrors || false,
|
||||
dockerTlsCa: data?.dockerTlsCa || "",
|
||||
dockerTlsCert: data?.dockerTlsCert || "",
|
||||
|
||||
@@ -51,6 +51,7 @@ import type { AppSettingsResponse } from "@/Types/Settings";
|
||||
import {
|
||||
stepFieldsFor,
|
||||
monitorStepCount,
|
||||
isCaptureDockerUrl,
|
||||
isDockerTlsUrl,
|
||||
type MonitorFormData,
|
||||
} from "@/Validation/monitor";
|
||||
@@ -350,6 +351,7 @@ const CreateMonitorPage = () => {
|
||||
|
||||
const watchedType = watch("type") as MonitorType;
|
||||
const watchedUrl = watch("url") as string;
|
||||
const isCaptureDocker = watchedType === "docker" && isCaptureDockerUrl(watchedUrl);
|
||||
const watchedMethod = watch("method") as HttpMethod | undefined;
|
||||
const watchedProxyMode = watch("proxyMode") as ProxyMode | undefined;
|
||||
const watchedUseAdvancedMatching = watch("useAdvancedMatching") as boolean;
|
||||
@@ -380,6 +382,13 @@ const CreateMonitorPage = () => {
|
||||
}
|
||||
}, [watchedType, showTypeSelector, reset, clearErrors]);
|
||||
|
||||
// The logs switch is hidden for Capture hosts, so a value left over from a socket host must not be submitted.
|
||||
useEffect(() => {
|
||||
if (isCaptureDocker) {
|
||||
setValue("dockerLogsEnabled", false);
|
||||
}
|
||||
}, [isCaptureDocker, setValue]);
|
||||
|
||||
const generalSettingsConfig = useMemo(
|
||||
() => getGeneralSettingsConfig(watchedType, t),
|
||||
[watchedType, t]
|
||||
@@ -709,8 +718,8 @@ const CreateMonitorPage = () => {
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Secret field - only for hardware type */}
|
||||
{generalSettingsConfig.showSecret && (
|
||||
{/* Capture API secret - used by hardware and Capture-backed Docker monitors */}
|
||||
{(generalSettingsConfig.showSecret || isCaptureDocker) && (
|
||||
<FormTextField
|
||||
name="secret"
|
||||
fieldLabel={t("pages.createMonitor.form.general.option.secret.label")}
|
||||
@@ -719,6 +728,12 @@ const CreateMonitorPage = () => {
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
{isCaptureDocker && (
|
||||
<FormSwitchField
|
||||
name="ignoreTlsErrors"
|
||||
label={t("pages.createMonitor.form.ignoreTls.option.tls.label")}
|
||||
/>
|
||||
)}
|
||||
{generalSettingsConfig.showDnsServer && (
|
||||
<FormTextField
|
||||
name="dnsServer"
|
||||
@@ -946,7 +961,7 @@ const CreateMonitorPage = () => {
|
||||
/>
|
||||
)}
|
||||
|
||||
{showStep(1) && watchedType === "docker" && (
|
||||
{showStep(1) && watchedType === "docker" && !isCaptureDocker && (
|
||||
<ConfigBox
|
||||
title={t("pages.createMonitor.form.dockerLogs.title")}
|
||||
subtitle={t("pages.createMonitor.form.dockerLogs.description")}
|
||||
|
||||
@@ -3,9 +3,7 @@ import Stack from "@mui/material/Stack";
|
||||
import FormControlLabel from "@mui/material/FormControlLabel";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Checkbox, Dialog } from "@/Components/inputs";
|
||||
import { type DashboardCardKey, dashboardCardKeys } from "@/Features/UI/uiSlice";
|
||||
|
||||
export type { DashboardCardKey };
|
||||
import { type DashboardCardKey, dashboardCardKeys } from "@/Types/Dashboard";
|
||||
|
||||
interface EditCardsModalProps {
|
||||
open: boolean;
|
||||
@@ -52,7 +50,7 @@ export const EditCardsModal = ({
|
||||
fullWidth
|
||||
>
|
||||
<Stack>
|
||||
{dashboardCardKeys.map((key) => (
|
||||
{dashboardCardKeys.map((key: DashboardCardKey) => (
|
||||
<FormControlLabel
|
||||
key={key}
|
||||
label={t(`pages.dashboard.cards.${key}`)}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useMemo, useState, type ReactNode } from "react";
|
||||
import { Fragment, useMemo, useState, type ReactNode } from "react";
|
||||
import Box from "@mui/material/Box";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
@@ -10,16 +10,13 @@ import { Button } from "@/Components/inputs";
|
||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||
import { useMonitorListController } from "@/Hooks/useMonitorListController";
|
||||
import { MonitorTypes, type MonitorTypeCount } from "@/Types/Monitor";
|
||||
import {
|
||||
setDashboardVisibleCards,
|
||||
dashboardCardKeys,
|
||||
type DashboardCardKey,
|
||||
} from "@/Features/UI/uiSlice";
|
||||
import { setDashboardVisibleCards } from "@/Features/UI/uiSlice";
|
||||
import type { RootState, AppDispatch } from "@/store";
|
||||
|
||||
import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard";
|
||||
import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard";
|
||||
import { EditCardsModal } from "@/Pages/Dashboard/components/EditCardsModal";
|
||||
import { type DashboardCardKey, dashboardCardKeys } from "@/Types/Dashboard";
|
||||
|
||||
const Dashboard = () => {
|
||||
const theme = useTheme();
|
||||
@@ -78,7 +75,7 @@ const Dashboard = () => {
|
||||
{dashboardCardKeys
|
||||
.filter((key) => visibleCards.has(key))
|
||||
.map((key) => (
|
||||
<div key={key}>{cardComponents[key]}</div>
|
||||
<Fragment key={key}>{cardComponents[key]}</Fragment>
|
||||
))}
|
||||
</Box>
|
||||
</Stack>
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { DockerContainerInfo } from "@/Types/Check";
|
||||
|
||||
// Utils
|
||||
import { formatDuration } from "@/Utils/TimeUtils";
|
||||
import { PLACEHOLDER } from "@/Utils/FormatUtils";
|
||||
import { getDockerPalette } from "@/Utils/MonitorUtils";
|
||||
|
||||
export const DockerContainerStatusBoxes = ({
|
||||
@@ -23,8 +24,9 @@ export const DockerContainerStatusBoxes = ({
|
||||
if (!container) return null;
|
||||
|
||||
const { state, restartCount, health } = container;
|
||||
const startedAt = container.startedAt ?? "0";
|
||||
const runningFor = Date.now() - new Date(startedAt).getTime();
|
||||
const runningFor = container.startedAt
|
||||
? Date.now() - new Date(container.startedAt).getTime()
|
||||
: undefined;
|
||||
const palette = getDockerPalette(state);
|
||||
return (
|
||||
<Stack
|
||||
@@ -39,11 +41,11 @@ export const DockerContainerStatusBoxes = ({
|
||||
/>
|
||||
<StatBox
|
||||
title={t("common.labels.uptime")}
|
||||
subtitle={formatDuration(runningFor)}
|
||||
subtitle={runningFor === undefined ? PLACEHOLDER : formatDuration(runningFor)}
|
||||
/>
|
||||
<StatBox
|
||||
title={t("common.labels.restarts")}
|
||||
subtitle={String(restartCount ?? 0)}
|
||||
subtitle={restartCount === undefined ? PLACEHOLDER : String(restartCount)}
|
||||
/>
|
||||
<StatBox
|
||||
title={t("common.labels.health")}
|
||||
|
||||
@@ -13,7 +13,7 @@ import type { Theme } from "@mui/material";
|
||||
import type { TFunction } from "i18next";
|
||||
import type { DockerContainerStats } from "@/Types/Monitor";
|
||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||
import { formatPercentage } from "@/Utils/FormatUtils";
|
||||
import { formatPercentage, PLACEHOLDER } from "@/Utils/FormatUtils";
|
||||
import prettyBytes from "pretty-bytes";
|
||||
import { BaseBox } from "@/Components/design-elements";
|
||||
import { dedupeDockerPorts, getDockerMountLabel } from "@/Utils/MonitorUtils";
|
||||
@@ -88,11 +88,17 @@ export const TabOverview = ({
|
||||
Ports
|
||||
</Typography>
|
||||
<Stack>
|
||||
{dedupedPorts?.map((port) => {
|
||||
{dedupedPorts.length === 0 && <Typography>{PLACEHOLDER}</Typography>}
|
||||
{dedupedPorts.map((port) => {
|
||||
const containerPort = `${port.privatePort}/${port.protocol}`;
|
||||
const label =
|
||||
port.publicPort === undefined
|
||||
? containerPort
|
||||
: `${port.hostIp ?? "0.0.0.0"}:${port.publicPort} -> ${containerPort}`;
|
||||
return (
|
||||
<Typography
|
||||
key={`${port.publicPort}/${port.privatePort}`}
|
||||
>{`${port.hostIp}:${port.publicPort} -> ${port.privatePort}/${port.protocol}`}</Typography>
|
||||
<Typography key={`${port.hostIp}/${port.publicPort}/${containerPort}`}>
|
||||
{label}
|
||||
</Typography>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
@@ -111,6 +117,7 @@ export const TabOverview = ({
|
||||
Volumes
|
||||
</Typography>
|
||||
<Stack>
|
||||
{!mounts?.length && <Typography>{PLACEHOLDER}</Typography>}
|
||||
{mounts?.map((mount) => {
|
||||
return (
|
||||
<Typography key={`${mount.source}/${mount.destination}`}>
|
||||
|
||||
@@ -12,7 +12,7 @@ import type { Header } from "@/Components/design-elements/Table";
|
||||
import type { DockerContainerInfo } from "@/Types/Check";
|
||||
|
||||
// Utils
|
||||
import { formatPercentage } from "@/Utils/FormatUtils";
|
||||
import { formatPercentage, PLACEHOLDER } from "@/Utils/FormatUtils";
|
||||
|
||||
interface DockerContainersTableProps {
|
||||
monitorId: string | undefined;
|
||||
@@ -91,7 +91,7 @@ export const DockerContainersTable = ({
|
||||
id: "restarts",
|
||||
content: t("common.labels.restarts"),
|
||||
render: (row) => {
|
||||
return <Typography>{row.restartCount}</Typography>;
|
||||
return <Typography>{row.restartCount ?? PLACEHOLDER}</Typography>;
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import Box from "@mui/material/Box";
|
||||
import { DetailGauge } from "@/Components/design-elements";
|
||||
|
||||
import { getPercentage, formatPercentageFromWhole } from "@/Utils/FormatUtils";
|
||||
import { getPercentage, formatPercentageFromWhole, PLACEHOLDER } from "@/Utils/FormatUtils";
|
||||
import prettyBytes from "pretty-bytes";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useTheme } from "@mui/material";
|
||||
@@ -11,7 +11,6 @@ interface StatGaugesProps {
|
||||
diagnostics: Diagnostics | null;
|
||||
}
|
||||
|
||||
const PLACEHOLDER = "—";
|
||||
|
||||
export const StatGauges = ({ diagnostics }: StatGaugesProps) => {
|
||||
const theme = useTheme();
|
||||
|
||||
@@ -6,12 +6,12 @@ import { useTheme } from "@mui/material";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import type { Diagnostics } from "@/Types/Diagnostics";
|
||||
import { formatDuration, formatMs } from "@/Utils/TimeUtils";
|
||||
import { PLACEHOLDER } from "@/Utils/FormatUtils";
|
||||
|
||||
interface StatsProps {
|
||||
diagnostics: Diagnostics | null;
|
||||
}
|
||||
|
||||
const PLACEHOLDER = "—";
|
||||
|
||||
export const Stats = ({ diagnostics }: StatsProps) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
import type { StatusPageThemeTokens } from "@/Pages/StatusPage/Status/themes/tokens";
|
||||
import { ThemedHeatmap } from "@/Pages/StatusPage/Status/themes/shared/ThemedHeatmap";
|
||||
import { ThemedHistogram } from "@/Pages/StatusPage/Status/themes/shared/ThemedHistogram";
|
||||
import { IncidentModal } from "@/Pages/StatusPage/Status/themes/shared/IncidentModal";
|
||||
import {
|
||||
ThemedInfrastructure,
|
||||
type GaugeFillLevel,
|
||||
@@ -116,6 +117,8 @@ export const BaseStatusPage = ({
|
||||
[config, tokens, mode]
|
||||
);
|
||||
const [chartMode, setChartMode] = useState<"heatmap" | "histogram">("heatmap");
|
||||
const [selectedMonitorId, setSelectedMonitorId] = useState<string | null>(null);
|
||||
const [selectedDate, setSelectedDate] = useState<string | null>(null);
|
||||
|
||||
const overall = resolveOverallStatus(monitors, t, config.overallStatusOptions);
|
||||
const logoSrc = statusPage.logo?.data
|
||||
@@ -306,6 +309,10 @@ export const BaseStatusPage = ({
|
||||
cells={cells}
|
||||
containerSx={styles.heatmap}
|
||||
cellSx={styles.heatmapCell}
|
||||
onCellClick={(date) => {
|
||||
setSelectedMonitorId(monitor.id);
|
||||
setSelectedDate(date);
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<ThemedHistogram
|
||||
@@ -313,6 +320,10 @@ export const BaseStatusPage = ({
|
||||
containerSx={styles.histogram}
|
||||
barSx={styles.bar}
|
||||
statsSx={styles.chartStats}
|
||||
onCellClick={(date) => {
|
||||
setSelectedMonitorId(monitor.id);
|
||||
setSelectedDate(date);
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</Box>
|
||||
@@ -333,6 +344,16 @@ export const BaseStatusPage = ({
|
||||
Checkmate
|
||||
</a>
|
||||
</Box>
|
||||
<IncidentModal
|
||||
url={statusPage.url}
|
||||
monitorId={selectedMonitorId}
|
||||
date={selectedDate}
|
||||
timezone={bucketTimezone}
|
||||
onClose={() => {
|
||||
setSelectedMonitorId(null);
|
||||
setSelectedDate(null);
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -25,6 +25,7 @@ export interface ChartCell {
|
||||
responseTime: number; // 0 for empty cells
|
||||
tooltip: React.ReactNode | null;
|
||||
ariaLabel: string;
|
||||
date: string | null;
|
||||
}
|
||||
|
||||
const DAY_DOWN_THRESHOLD = 0.5;
|
||||
@@ -61,7 +62,7 @@ export const classifyResponseTime = (
|
||||
return "fast";
|
||||
};
|
||||
|
||||
const emptyCell = (key: string): ChartCell => ({
|
||||
const emptyCell = (key: string, date: string | null = null): ChartCell => ({
|
||||
key,
|
||||
barKind: "empty",
|
||||
heatKind: "empty",
|
||||
@@ -70,6 +71,7 @@ const emptyCell = (key: string): ChartCell => ({
|
||||
tooltip: null,
|
||||
ariaLabel: "",
|
||||
severity: 0,
|
||||
date,
|
||||
});
|
||||
|
||||
const padCells = (cells: ChartCell[], length: number): ChartCell[] => [
|
||||
@@ -92,6 +94,7 @@ export const checksToCells = (checks: CheckSnapshot[]): ChartCell[] => {
|
||||
tooltip: <ThemedChartTooltip check={check} />,
|
||||
ariaLabel: `${formatMs(check.responseTime)}, ${check.status ? "up" : "down"}`,
|
||||
severity: 0,
|
||||
date: null,
|
||||
})
|
||||
);
|
||||
return padCells(cells, MAX_RECENT_CHECKS);
|
||||
@@ -108,7 +111,7 @@ export const dailyBucketsToCells = (
|
||||
const enumerated = enumerateDays(days, bucketTimezone);
|
||||
return enumerated.map((date) => {
|
||||
const bucket = byDate.get(date);
|
||||
if (!bucket) return emptyCell(`day-${date}`);
|
||||
if (!bucket) return emptyCell(`day-${date}`, date);
|
||||
const upRatio = bucket.upChecks / bucket.totalChecks;
|
||||
|
||||
const barKind = classifyDay(bucket, upRatio);
|
||||
@@ -132,6 +135,7 @@ export const dailyBucketsToCells = (
|
||||
date,
|
||||
uptime: formatPercentage(bucket.upChecks / bucket.totalChecks),
|
||||
}),
|
||||
date,
|
||||
};
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,111 @@
|
||||
import Dialog from "@mui/material/Dialog";
|
||||
import DialogTitle from "@mui/material/DialogTitle";
|
||||
import DialogContent from "@mui/material/DialogContent";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import IconButton from "@mui/material/IconButton";
|
||||
import { X as XIcon } from "lucide-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useGet } from "@/Hooks/UseApi";
|
||||
import { useTheme } from "@mui/material";
|
||||
import { SPACING } from "@/Utils/Theme/constants";
|
||||
import { formatDateWithTz } from "@/Utils/TimeUtils";
|
||||
|
||||
import type { Incident } from "@/Types/Incident";
|
||||
interface IncidentModalProps {
|
||||
url: string;
|
||||
monitorId: string | null;
|
||||
date: string | null;
|
||||
timezone: string;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export const IncidentModal = ({
|
||||
url,
|
||||
monitorId,
|
||||
date,
|
||||
timezone,
|
||||
onClose,
|
||||
}: IncidentModalProps) => {
|
||||
const theme = useTheme();
|
||||
const { t } = useTranslation();
|
||||
const open = Boolean(monitorId && date);
|
||||
|
||||
const { data, isLoading, error } = useGet<{ incidents: Incident[] }>(
|
||||
open ? `/status-page/${url}/incidents/${monitorId}?date=${date}` : null,
|
||||
{},
|
||||
{ revalidateOnFocus: false, keepPreviousData: true }
|
||||
);
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
open={open}
|
||||
onClose={onClose}
|
||||
maxWidth="sm"
|
||||
fullWidth
|
||||
>
|
||||
<DialogTitle
|
||||
sx={{ display: "flex", justifyContent: "space-between", alignItems: "center" }}
|
||||
>
|
||||
{t("pages.statusPages.monitorsList.incidents.modalTitle", { date })} ({timezone})
|
||||
<IconButton
|
||||
onClick={onClose}
|
||||
size="small"
|
||||
aria-label="close"
|
||||
>
|
||||
<XIcon size={20} />
|
||||
</IconButton>
|
||||
</DialogTitle>
|
||||
<DialogContent>
|
||||
{isLoading ? (
|
||||
<Typography mt={SPACING.SM}>
|
||||
{t("pages.statusPages.monitorsList.incidents.loading")}
|
||||
</Typography>
|
||||
) : error ? (
|
||||
<Typography
|
||||
mt={SPACING.SM}
|
||||
color={theme.palette.error.main}
|
||||
>
|
||||
{t("pages.statusPages.monitorsList.incidents.errorLoading")}
|
||||
</Typography>
|
||||
) : data?.incidents && data.incidents.length > 0 ? (
|
||||
<Stack
|
||||
gap={SPACING.LG}
|
||||
mt={SPACING.SM}
|
||||
>
|
||||
{data.incidents.map((incident) => (
|
||||
<Stack
|
||||
key={incident.id}
|
||||
p={SPACING.LG}
|
||||
border={1}
|
||||
borderColor="divider"
|
||||
borderRadius={theme.shape.borderRadius}
|
||||
>
|
||||
<Typography
|
||||
variant="subtitle2"
|
||||
fontWeight="bold"
|
||||
>
|
||||
{incident.message ||
|
||||
(incident.statusCode ? `Error ${incident.statusCode}` : "Incident")}
|
||||
</Typography>
|
||||
<Typography
|
||||
variant="body2"
|
||||
color={theme.palette.text.secondary}
|
||||
>
|
||||
{formatDateWithTz(incident.startTime, "h:mm A", timezone)} -{" "}
|
||||
{incident.endTime
|
||||
? formatDateWithTz(incident.endTime, "h:mm A", timezone)
|
||||
: t("pages.statusPages.monitorsList.incidents.ongoing")}
|
||||
</Typography>
|
||||
</Stack>
|
||||
))}
|
||||
</Stack>
|
||||
) : (
|
||||
<Typography mt={SPACING.SM}>
|
||||
{t("pages.statusPages.monitorsList.incidents.noIncidents")}
|
||||
</Typography>
|
||||
)}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
};
|
||||
@@ -2,6 +2,8 @@ import Tooltip from "@mui/material/Tooltip";
|
||||
import Box from "@mui/material/Box";
|
||||
import type { SxProps, Theme } from "@mui/material/styles";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { mergeSx } from "@/Pages/StatusPage/Status/themes/shared/sx";
|
||||
|
||||
import type {
|
||||
ChartCell,
|
||||
HeatCellKind,
|
||||
@@ -11,17 +13,15 @@ interface Props {
|
||||
cells: ChartCell[];
|
||||
containerSx: SxProps<Theme>;
|
||||
cellSx: (kind: HeatCellKind, severity?: number) => SxProps<Theme>;
|
||||
onCellClick?: (date: string) => void;
|
||||
}
|
||||
|
||||
export const ThemedHeatmap = ({ cells, containerSx, cellSx }: Props) => {
|
||||
export const ThemedHeatmap = ({ cells, containerSx, cellSx, onCellClick }: Props) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
return (
|
||||
<Box
|
||||
sx={[
|
||||
...(Array.isArray(containerSx) ? containerSx : [containerSx]),
|
||||
{ gridTemplateColumns: `repeat(${cells.length}, 1fr)` },
|
||||
]}
|
||||
sx={mergeSx(containerSx, { gridTemplateColumns: `repeat(${cells.length}, 1fr)` })}
|
||||
role="img"
|
||||
aria-label={t("pages.statusPages.monitorsList.chart.heatmapAria")}
|
||||
>
|
||||
@@ -30,7 +30,15 @@ export const ThemedHeatmap = ({ cells, containerSx, cellSx }: Props) => {
|
||||
return (
|
||||
<Box
|
||||
key={cell.key}
|
||||
sx={cellSx("empty")}
|
||||
sx={mergeSx(
|
||||
cellSx("empty"),
|
||||
!!cell.date && !!onCellClick ? { cursor: "pointer" } : undefined
|
||||
)}
|
||||
onClick={() => {
|
||||
if (cell.date && onCellClick) {
|
||||
onCellClick(cell.date);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -42,8 +50,16 @@ export const ThemedHeatmap = ({ cells, containerSx, cellSx }: Props) => {
|
||||
placement="top"
|
||||
>
|
||||
<Box
|
||||
sx={cellSx(cell.heatKind, cell.severity)}
|
||||
sx={mergeSx(
|
||||
cellSx(cell.heatKind, cell.severity),
|
||||
!!cell.date && !!onCellClick ? { cursor: "pointer" } : undefined
|
||||
)}
|
||||
aria-label={cell.ariaLabel}
|
||||
onClick={() => {
|
||||
if (cell.date && onCellClick) {
|
||||
onCellClick(cell.date);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
);
|
||||
|
||||
@@ -5,6 +5,8 @@ import Stack from "@mui/material/Stack";
|
||||
import type { SxProps, Theme } from "@mui/material/styles";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { formatMs } from "@/Utils/TimeUtils";
|
||||
import { mergeSx } from "@/Pages/StatusPage/Status/themes/shared/sx";
|
||||
|
||||
import type {
|
||||
BarKind,
|
||||
ChartCell,
|
||||
@@ -16,6 +18,7 @@ interface Props {
|
||||
barSx: (kind: BarKind, heightPct: number, severity?: number) => SxProps<Theme>;
|
||||
statsSx: SxProps<Theme>;
|
||||
statsGap?: number;
|
||||
onCellClick?: (date: string) => void;
|
||||
}
|
||||
|
||||
export const ThemedHistogram = ({
|
||||
@@ -24,6 +27,7 @@ export const ThemedHistogram = ({
|
||||
barSx,
|
||||
statsSx,
|
||||
statsGap = 1,
|
||||
onCellClick,
|
||||
}: Props) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
@@ -39,17 +43,22 @@ export const ThemedHistogram = ({
|
||||
return (
|
||||
<Stack gap={statsGap}>
|
||||
<Box
|
||||
sx={[
|
||||
...(Array.isArray(containerSx) ? containerSx : [containerSx]),
|
||||
{ gridTemplateColumns: `repeat(${cells.length}, 1fr)` },
|
||||
]}
|
||||
sx={mergeSx(containerSx, { gridTemplateColumns: `repeat(${cells.length}, 1fr)` })}
|
||||
>
|
||||
{cells.map((cell) => {
|
||||
if (cell.barKind === "empty") {
|
||||
return (
|
||||
<Box
|
||||
key={cell.key}
|
||||
sx={barSx("empty", cell.heightPct)}
|
||||
sx={mergeSx(
|
||||
barSx("empty", cell.heightPct),
|
||||
!!cell.date && !!onCellClick ? { cursor: "pointer" } : undefined
|
||||
)}
|
||||
onClick={() => {
|
||||
if (cell.date && onCellClick) {
|
||||
onCellClick(cell.date);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -61,8 +70,16 @@ export const ThemedHistogram = ({
|
||||
placement="top"
|
||||
>
|
||||
<Box
|
||||
sx={barSx(cell.barKind, cell.heightPct, cell.severity)}
|
||||
sx={mergeSx(
|
||||
barSx(cell.barKind, cell.heightPct, cell.severity),
|
||||
!!cell.date && !!onCellClick ? { cursor: "pointer" } : undefined
|
||||
)}
|
||||
aria-label={cell.ariaLabel}
|
||||
onClick={() => {
|
||||
if (cell.date && onCellClick) {
|
||||
onCellClick(cell.date);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { SxProps, Theme } from "@mui/material/styles";
|
||||
|
||||
export const mergeSx = (
|
||||
...styles: Array<SxProps<Theme> | false | null | undefined>
|
||||
): SxProps<Theme> => {
|
||||
const result: SxProps<Theme>[] = [];
|
||||
for (const style of styles) {
|
||||
if (!style) continue;
|
||||
if (Array.isArray(style)) {
|
||||
result.push(...(style.filter(Boolean) as SxProps<Theme>[]));
|
||||
} else {
|
||||
result.push(style);
|
||||
}
|
||||
}
|
||||
return result as SxProps<Theme>;
|
||||
};
|
||||
@@ -0,0 +1,2 @@
|
||||
export const dashboardCardKeys = ["monitorStatus", "monitorsByType"] as const;
|
||||
export type DashboardCardKey = (typeof dashboardCardKeys)[number];
|
||||
@@ -47,3 +47,6 @@ export const getPercentage = (value: number, total: number) => {
|
||||
if (!value || !total) return 0;
|
||||
return (value / total) * 100;
|
||||
};
|
||||
|
||||
// Shown wherever a value is not available for the current data source.
|
||||
export const PLACEHOLDER = "—";
|
||||
@@ -85,6 +85,10 @@ export const lightPalette = {
|
||||
sidebar: {
|
||||
accent: colors.brandGreen,
|
||||
},
|
||||
scrollbar: {
|
||||
track: colors.rule200,
|
||||
thumb: colors.brandGreen,
|
||||
},
|
||||
rowStatus: {
|
||||
running: colors.green200,
|
||||
paused: colors.yellow200,
|
||||
@@ -137,6 +141,10 @@ export const darkPalette = {
|
||||
sidebar: {
|
||||
accent: colors.brandGreenLight,
|
||||
},
|
||||
scrollbar: {
|
||||
track: colors.ruleDark,
|
||||
thumb: colors.brandGreen,
|
||||
},
|
||||
rowStatus: {
|
||||
running: alpha(colors.brandGreen, 0.18),
|
||||
paused: alpha(colors.orange500, 0.18),
|
||||
|
||||
@@ -3,6 +3,7 @@ import { lightPalette, darkPalette, typographyLevels } from "@/Utils/Theme/Palet
|
||||
|
||||
import type { Theme } from "@mui/material/styles";
|
||||
import type {} from "@mui/material/styles";
|
||||
import { SCROLLBAR_SIZE } from "@/Utils/Theme/constants";
|
||||
|
||||
declare module "@mui/material/styles" {
|
||||
interface TypographyVariants {
|
||||
@@ -20,6 +21,7 @@ declare module "@mui/material/styles" {
|
||||
}
|
||||
interface Palette {
|
||||
sidebar: { accent: string };
|
||||
scrollbar: { track: string; thumb: string };
|
||||
rowStatus: { running: string; paused: string };
|
||||
chart: {
|
||||
phases: {
|
||||
@@ -34,6 +36,7 @@ declare module "@mui/material/styles" {
|
||||
}
|
||||
interface PaletteOptions {
|
||||
sidebar?: { accent: string };
|
||||
scrollbar?: { track: string; thumb: string };
|
||||
rowStatus?: { running: string; paused: string };
|
||||
chart?: {
|
||||
phases: {
|
||||
@@ -112,6 +115,27 @@ export const theme = (mode: string, palette: any) =>
|
||||
},
|
||||
|
||||
components: {
|
||||
MuiCssBaseline: {
|
||||
styleOverrides: (theme) => ({
|
||||
"*": {
|
||||
scrollbarWidth: "auto",
|
||||
scrollbarColor: `${theme.palette.scrollbar.thumb} ${theme.palette.scrollbar.track}`,
|
||||
},
|
||||
"*::-webkit-scrollbar": {
|
||||
width: SCROLLBAR_SIZE,
|
||||
height: SCROLLBAR_SIZE,
|
||||
},
|
||||
"*::-webkit-scrollbar-track": {
|
||||
background: theme.palette.scrollbar.track,
|
||||
borderRadius: theme.shape.borderRadius,
|
||||
},
|
||||
"*::-webkit-scrollbar-thumb": {
|
||||
background: theme.palette.scrollbar.thumb,
|
||||
borderRadius: theme.shape.borderRadius,
|
||||
},
|
||||
"*::-webkit-scrollbar-button": { display: "none" },
|
||||
}),
|
||||
},
|
||||
MuiTouchRipple: {
|
||||
styleOverrides: {
|
||||
root: {
|
||||
|
||||
@@ -19,7 +19,7 @@ export const LAYOUT = {
|
||||
} as const;
|
||||
|
||||
export const INPUT_BASE_HEIGHT = 32; // Height for inputs/controls
|
||||
|
||||
export const SCROLLBAR_SIZE = 16;
|
||||
export const HOVER = {
|
||||
DARKEN: 0.06, // This is a coefficient for darkening function
|
||||
ROW: 0.025, // Overlay alpha for hoverable rows and cards
|
||||
|
||||
@@ -110,23 +110,40 @@ const portSchema = baseSchema.extend({
|
||||
.max(65535, "Port must be at most 65535"),
|
||||
});
|
||||
|
||||
// Docker monitor schema. The url forms mirror DOCKER_TLS_URL and
|
||||
// DOCKER_SOCKET_URL in server/src/utils/dockerHost.ts.
|
||||
// Docker monitor schema. The URL forms mirror the Docker source helpers in
|
||||
// server/src/utils/dockerHost.ts.
|
||||
const dockerTlsUrlRegex = /^(tcp|https):\/\/([^/\s:]+)(?::(\d+))?\/?$/;
|
||||
const dockerSocketUrlRegex = /^(?:unix:\/\/\/\S+|\/\S+)$/;
|
||||
|
||||
export const isDockerTlsUrl = (url: string | undefined): boolean =>
|
||||
dockerTlsUrlRegex.test(url?.trim() ?? "");
|
||||
|
||||
export const isCaptureDockerUrl = (url: string | undefined): boolean => {
|
||||
try {
|
||||
const parsed = new URL(url?.trim() ?? "");
|
||||
const pathname = parsed.pathname.replace(/\/+$/, "");
|
||||
return (
|
||||
(parsed.protocol === "http:" || parsed.protocol === "https:") &&
|
||||
pathname.endsWith("/metrics/docker")
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const dockerSchema = baseSchema.extend({
|
||||
type: z.literal("docker"),
|
||||
url: z
|
||||
.string()
|
||||
.min(1, "Docker host URL is required")
|
||||
.refine(
|
||||
(url) => dockerSocketUrlRegex.test(url.trim()) || isDockerTlsUrl(url),
|
||||
"Docker host must be unix:///path, an absolute socket path, or tcp://host[:port]"
|
||||
(url) =>
|
||||
dockerSocketUrlRegex.test(url.trim()) ||
|
||||
isDockerTlsUrl(url) ||
|
||||
isCaptureDockerUrl(url),
|
||||
"Docker host must be a socket path, a TLS daemon URL, or a Capture /metrics/docker endpoint"
|
||||
),
|
||||
secret: z.string(),
|
||||
ignoreTlsErrors: z.boolean(),
|
||||
dockerTlsCa: z.string(),
|
||||
dockerTlsCert: z.string(),
|
||||
@@ -250,6 +267,14 @@ export const monitorSchema = monitorSchemaUnion.superRefine((data, ctx) => {
|
||||
});
|
||||
}
|
||||
|
||||
if (data.type === "docker" && isCaptureDockerUrl(data.url) && !data.secret.trim()) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: ["secret"],
|
||||
message: "Capture API secret is required",
|
||||
});
|
||||
}
|
||||
|
||||
if (data.type === "docker" && isDockerTlsUrl(data.url)) {
|
||||
if (!data.dockerTlsCert.trim()) {
|
||||
ctx.addIssue({
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "صيانة",
|
||||
"paused": "متوقف مؤقتاً",
|
||||
"up": "قيد التشغيل"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "فشل تحميل الحوادث."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"uptime": {
|
||||
"title": "Temps en línia",
|
||||
"responseTime": "Temps de resposta"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "No s'han pogut carregar les incidències."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Údržba",
|
||||
"paused": "Pozastaveno",
|
||||
"up": "V provozu"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Nepodařilo se načíst incidenty."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1378,6 +1378,9 @@
|
||||
"maintenance": "Wartung",
|
||||
"paused": "Pausiert",
|
||||
"up": "Betriebsbereit"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Vorfälle konnten nicht geladen werden."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -720,7 +720,7 @@
|
||||
"http": "Enter the URL or IP to monitor (e.g., https://example.com/ or 192.168.1.100) and add a clear display name that appears on the dashboard. Set an optional proxy server to connect through.",
|
||||
"ping": "Enter the IP address or hostname to ping (e.g., 192.168.1.100 or example.com) and add a clear display name that appears on the dashboard.",
|
||||
"port": "Enter the URL or IP of the server, the port number and a clear display name that appears on the dashboard.",
|
||||
"docker": "Enter the Docker socket to monitor, as unix:///var/run/docker.sock or an absolute path such as /var/run/docker.sock, or a remote daemon as tcp://host:2376 to connect with TLS client certificates. Every container on the host is monitored.",
|
||||
"docker": "Enter a Docker socket such as unix:///var/run/docker.sock, a remote daemon such as tcp://host:2376 with TLS client certificates, or a Capture endpoint ending in /metrics/docker with its authorization secret. Every container on the host is monitored.",
|
||||
"game": "Enter the IP address or hostname and the port number to ping (e.g., 192.168.1.100 or example.com) and choose game type. Some games require enabling server queries in their config file — check your game server's documentation. See the <gamedigLink>supported games list</gamedigLink> for protocol details.",
|
||||
"pagespeed": "Track page load performance, Core Web Vitals, and optimization scores for your website.",
|
||||
"grpc": "Enter the hostname and port of the gRPC server, optionally specify a service name for the health check, and add a display name.",
|
||||
@@ -1712,6 +1712,13 @@
|
||||
"uptime": {
|
||||
"title": "Uptime",
|
||||
"responseTime": "Response time"
|
||||
},
|
||||
"incidents": {
|
||||
"modalTitle": "Incidents for {{date}}",
|
||||
"loading": "Loading...",
|
||||
"noIncidents": "No incidents reported for this day.",
|
||||
"ongoing": "Ongoing",
|
||||
"errorLoading": "Failed to load incidents."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Mantenimiento",
|
||||
"paused": "Pausado",
|
||||
"up": "Operativo"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Error al cargar incidentes."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Huolto",
|
||||
"paused": "Keskeytetty",
|
||||
"up": "Toiminnassa"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Häiriöiden lataaminen epäonnistui."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Maintenance",
|
||||
"paused": "En pause",
|
||||
"up": "Opérationnel"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Échec du chargement des incidents."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1473,6 +1473,9 @@
|
||||
"uptime": {
|
||||
"title": "Uptime",
|
||||
"responseTime": "Tempo di risposta"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Impossibile caricare gli incidenti."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "メンテナンス",
|
||||
"paused": "一時停止中",
|
||||
"up": "稼働中"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "インシデントの読み込みに失敗しました。"
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1584,6 +1584,9 @@
|
||||
"uptime": {
|
||||
"responseTime": "Czas odpowiedzi",
|
||||
"title": "Czas pracy"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Nie udało się załadować incydentów."
|
||||
}
|
||||
},
|
||||
"preview": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Manutenção",
|
||||
"paused": "Pausado",
|
||||
"up": "Operacional"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Falha ao carregar incidentes."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Обслуживание",
|
||||
"paused": "Приостановлено",
|
||||
"up": "Работает"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Не удалось загрузить инциденты."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "การบำรุงรักษา",
|
||||
"paused": "หยุดชั่วคราว",
|
||||
"up": "ใช้งานได้"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "ไม่สามารถโหลดเหตุการณ์ได้"
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Bakım",
|
||||
"paused": "Duraklatıldı",
|
||||
"up": "Çalışıyor"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Olaylar yüklenemedi."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Обслуговування",
|
||||
"paused": "Призупинено",
|
||||
"up": "Працює"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Не вдалося завантажити інциденти."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "Bảo trì",
|
||||
"paused": "Tạm dừng",
|
||||
"up": "Hoạt động"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "Không thể tải sự cố."
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1378,6 +1378,9 @@
|
||||
"maintenance": "维护中",
|
||||
"paused": "已暂停",
|
||||
"up": "运行中"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "无法加载事件。"
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -1377,6 +1377,9 @@
|
||||
"maintenance": "維護中",
|
||||
"paused": "已暫停",
|
||||
"up": "運行中"
|
||||
},
|
||||
"incidents": {
|
||||
"errorLoading": "無法載入事件。"
|
||||
}
|
||||
},
|
||||
"statusBar": {
|
||||
|
||||
@@ -13307,11 +13307,6 @@
|
||||
},
|
||||
"maxItems": 10
|
||||
},
|
||||
"egressPollIntervalSeconds": {
|
||||
"type": "integer",
|
||||
"minimum": 5,
|
||||
"maximum": 600
|
||||
},
|
||||
"egressNotifications": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
||||
Generated
+7
@@ -18,6 +18,7 @@
|
||||
"compression": "1.8.1",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"dayjs": "^1.11.23",
|
||||
"dockerode": "5.0.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.19.2",
|
||||
@@ -5803,6 +5804,12 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/dayjs": {
|
||||
"version": "1.11.23",
|
||||
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz",
|
||||
"integrity": "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
|
||||
+2
-1
@@ -15,7 +15,7 @@
|
||||
"worker": "QUEUE_MODE=worker node ./dist/index.js",
|
||||
"worker:dev": "QUEUE_MODE=worker nodemon --exec tsx src/index.js",
|
||||
"clean": "rm -rf dist",
|
||||
"build": "npm run clean && npm run build:openapi && tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && cp -r src/templates dist/templates",
|
||||
"build": "npm run clean && npm run build:openapi && tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && cp -r src/templates dist/templates && mkdir -p dist/service && cp -r src/service/protos dist/service/",
|
||||
"typeCheck": "tsc --noEmit",
|
||||
"build:openapi": "tsc --noEmit && tsx scripts/buildOpenApi.ts",
|
||||
"lint": "eslint .",
|
||||
@@ -42,6 +42,7 @@
|
||||
"compression": "1.8.1",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"dayjs": "^1.11.23",
|
||||
"dockerode": "5.0.0",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.19.2",
|
||||
|
||||
@@ -10,7 +10,10 @@ type WhoisModule = typeof whoiser;
|
||||
export const fetchMonitorCertificate = async (checker: SSLCheckerType, monitor: Monitor): Promise<SSLDetails> => {
|
||||
const monitorUrl = new URL(monitor.url);
|
||||
const hostname = monitorUrl.hostname;
|
||||
const cert = await checker(hostname);
|
||||
// An explicit port in the URL is the most visible to the user, so it takes precedence over monitor.port.
|
||||
const urlPort = monitorUrl.port ? Number(monitorUrl.port) : undefined;
|
||||
const port = urlPort ?? monitor.port;
|
||||
const cert = await checker(hostname, port !== undefined ? { port } : undefined);
|
||||
if (cert?.validTo === null || cert?.validTo === undefined) {
|
||||
throw new Error("Certificate not found");
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import {
|
||||
getStatusPageQueryValidation,
|
||||
imageValidation,
|
||||
resolveStatusPageQueryValidation,
|
||||
getPublicMonitorIncidentsParamValidation,
|
||||
getPublicMonitorIncidentsQueryValidation,
|
||||
} from "@/api/validation/statusPageValidation.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { requireTeamId, requireUserId } from "@/api/controllers/controllerUtils.js";
|
||||
@@ -17,6 +19,7 @@ export interface IStatusPageController {
|
||||
createStatusPage: RequestHandler;
|
||||
updateStatusPage: RequestHandler;
|
||||
getStatusPageByUrl: RequestHandler;
|
||||
getPublicMonitorIncidents: RequestHandler;
|
||||
resolveStatusPageByDomain: RequestHandler;
|
||||
getStatusPagesByTeamId: RequestHandler;
|
||||
deleteStatusPage: RequestHandler;
|
||||
@@ -85,6 +88,19 @@ class StatusPageController implements IStatusPageController {
|
||||
});
|
||||
});
|
||||
|
||||
getPublicMonitorIncidents = catchAsync(async (req: Request, res: Response) => {
|
||||
const { url, monitorId } = getPublicMonitorIncidentsParamValidation.parse(req.params);
|
||||
const { date } = getPublicMonitorIncidentsQueryValidation.parse(req.query);
|
||||
|
||||
const incidents = await this.statusPageService.getPublicMonitorIncidents(url, monitorId, date, req.user?.teamId);
|
||||
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
msg: "Incidents retrieved successfully",
|
||||
data: { incidents },
|
||||
});
|
||||
});
|
||||
|
||||
resolveStatusPageByDomain = catchAsync(async (req: Request, res: Response) => {
|
||||
const { range } = resolveStatusPageQueryValidation.parse(req.query);
|
||||
const domain = resolveStatusPageDomainFromRequest(req.hostname, req.query.domain as string | undefined);
|
||||
|
||||
@@ -16,7 +16,15 @@ export const isPublicStatusPageApiPath = (method: string, path: string): boolean
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!slug || slug === "team" || slug.includes("/")) {
|
||||
if (!slug || slug === "team") {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (slug.includes("/")) {
|
||||
const parts = slug.split("/");
|
||||
if (parts.length === 3 && parts[1] === "incidents") {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ export const createStatusPageRoutes = (
|
||||
router.put("/:id", imageUpload.single("logo"), verifyJWT, isAllowed(["admin", "superadmin"]), statusPageController.updateStatusPage);
|
||||
router.get("/resolve", statusPageController.resolveStatusPageByDomain);
|
||||
router.get("/:url", verifyStatusPageAccess, statusPageController.getStatusPageByUrl);
|
||||
router.get("/:url/incidents/:monitorId", verifyStatusPageAccess, statusPageController.getPublicMonitorIncidents);
|
||||
router.delete("/:id", verifyJWT, isAllowed(["admin", "superadmin"]), statusPageController.deleteStatusPage);
|
||||
return router;
|
||||
};
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
import { DateRanges, SortOrders } from "@/types/query.js";
|
||||
import { DockerContainerStates, DockerHealthStatuses, DockerLogStreams, DockerPortProtocols } from "@/domain/docker/docker.type.js";
|
||||
import { DOCKER_LOG_PAGE_DEFAULT, DOCKER_LOG_PAGE_MAX } from "@/domain/docker/docker-log.type.js";
|
||||
import { isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { isCaptureDockerUrl, isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { X509Certificate } from "node:crypto";
|
||||
import { keyMatchesCertificate, parseCertificates, parsePrivateKey } from "@/utils/pem.js";
|
||||
|
||||
@@ -125,15 +125,21 @@ const refineProxySelection = (body: { proxyMode?: string; proxyId?: string }, ct
|
||||
|
||||
const refineDockerUrl = (data: { type?: string; url?: string }, ctx: z.RefinementCtx) => {
|
||||
if (data.type !== "docker" || data.url === undefined) return;
|
||||
if (!isDockerSocketUrl(data.url) && !isDockerTlsUrl(data.url)) {
|
||||
if (!isDockerSocketUrl(data.url) && !isDockerTlsUrl(data.url) && !isCaptureDockerUrl(data.url)) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: ["url"],
|
||||
message: "Docker host must be unix:///path, an absolute socket path, or tcp://host[:port]",
|
||||
message: "Docker host must be a socket path, a TLS daemon URL, or a Capture /metrics/docker endpoint",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Edits are checked in the monitor service against the stored secret, since the body may omit either the url or the secret.
|
||||
const refineCaptureSecret = (data: { type?: string; url?: string; secret?: string }, ctx: z.RefinementCtx) => {
|
||||
if (data.type !== "docker" || !isCaptureDockerUrl(data.url)) return;
|
||||
if (!data.secret?.trim()) ctx.addIssue({ code: "custom", path: ["secret"], message: "Capture API secret is required" });
|
||||
};
|
||||
|
||||
type DockerTlsFields = {
|
||||
type?: string;
|
||||
url?: string;
|
||||
@@ -226,6 +232,7 @@ export const createMonitorBodyValidation = z
|
||||
.superRefine(refineRegexPattern)
|
||||
.superRefine(refineProxySelection)
|
||||
.superRefine(refineDockerUrl)
|
||||
.superRefine(refineCaptureSecret)
|
||||
.superRefine(refineDockerTls("create"));
|
||||
|
||||
export const editMonitorBodyValidation = z
|
||||
@@ -353,7 +360,8 @@ const importedMonitorSchema = z
|
||||
.superRefine(refineHeadMatching)
|
||||
.superRefine(refineRegexPattern)
|
||||
.superRefine(refineProxySelection)
|
||||
.superRefine(refineDockerUrl);
|
||||
.superRefine(refineDockerUrl)
|
||||
.superRefine(refineCaptureSecret);
|
||||
|
||||
export const importMonitorsBodyValidation = z.object({
|
||||
monitors: z.array(importedMonitorSchema).min(1, "At least one monitor is required"),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
|
||||
import { MAX_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
import { parseEgressTarget } from "@/utils/egressTarget.js";
|
||||
import { z } from "zod";
|
||||
|
||||
@@ -56,7 +56,6 @@ export const updateAppSettingsBodyValidation = z
|
||||
|
||||
egressCheckEnabled: z.boolean().optional(),
|
||||
egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
|
||||
egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
|
||||
egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(),
|
||||
})
|
||||
.strip()
|
||||
|
||||
@@ -20,6 +20,15 @@ export const getStatusPageParamValidation = z.object({
|
||||
url: z.string().min(1, "URL is required"),
|
||||
});
|
||||
|
||||
export const getPublicMonitorIncidentsParamValidation = z.object({
|
||||
url: z.string().min(1, "URL is required"),
|
||||
monitorId: z.string().regex(/^[0-9a-fA-F]{24}$/, "Must be a valid monitor ID"),
|
||||
});
|
||||
|
||||
export const getPublicMonitorIncidentsQueryValidation = z.object({
|
||||
date: z.string().date("Date must be a valid calendar date in YYYY-MM-DD format"),
|
||||
});
|
||||
|
||||
export const getStatusPageQueryValidation = z.object({
|
||||
type: z.union([z.enum(StatusPageTypes), z.array(z.enum(StatusPageTypes))]).transform((val) => (Array.isArray(val) ? val : [val])),
|
||||
range: z.enum(StatusPageRanges).optional().default("latest"),
|
||||
|
||||
@@ -99,7 +99,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
emailService,
|
||||
});
|
||||
|
||||
const statusPageService = new StatusPageService(statusPagesRepository, settingsService, monitorsRepository, checksRepository);
|
||||
const statusPageService = new StatusPageService(statusPagesRepository, settingsService, monitorsRepository, checksRepository, incidentsRepository);
|
||||
const tagsService = new TagsService(tagsRepository, monitorsRepository);
|
||||
const diagnosticService = new DiagnosticService(db);
|
||||
const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService);
|
||||
|
||||
@@ -17,12 +17,9 @@ import { INetworkService, NetworkService } from "@/service/networkService.js";
|
||||
import { IBufferService, BufferService } from "@/service/bufferService.js";
|
||||
import { IStatusService, StatusService } from "@/service/statusService.js";
|
||||
import { SecretsRotationService } from "@/service/encryption/secretsRotationService.js";
|
||||
import { IQueueWorker } from "@/worker/worker.interface.js";
|
||||
import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
import { IQueueWorker, JobHandlers } from "@/worker/worker.interface.js";
|
||||
import { WorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { CheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { GeoChecksPipeline } from "@/worker/worker.geo-pipeline.js";
|
||||
import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
|
||||
import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
|
||||
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
@@ -42,6 +39,7 @@ import { GrpcProvider } from "@/service/network/GrpcProvider.js";
|
||||
import { WebSocketProvider } from "@/service/network/WebSocketProvider.js";
|
||||
import { DNSProvider } from "@/service/network/DNSProvider.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { WorkerPipeline } from "@/worker/worker.pipeline.js";
|
||||
export interface WorkerServices {
|
||||
worker: IQueueWorker;
|
||||
networkService: INetworkService;
|
||||
@@ -82,7 +80,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
|
||||
const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
|
||||
const hardwareProvider = new HardwareProvider(httpProvider);
|
||||
const dockerProvider = new DockerProvider(logger, Docker, encryptionService);
|
||||
const dockerProvider = new DockerProvider(logger, Docker, encryptionService, httpProvider);
|
||||
const portProvider = new PortProvider(net);
|
||||
const gameProvider = new GameProvider(logger, GameDig);
|
||||
const grpcProvider = new GrpcProvider(grpc, protoLoader);
|
||||
@@ -103,36 +101,33 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
]);
|
||||
|
||||
const proxyResolver = new ProxyResolver(proxiesRepository, settingsService, logger);
|
||||
const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
|
||||
const bufferService = new BufferService(logger, geoChecksService, dockerLogsService, settingsService, (checks): Promise<void> =>
|
||||
pipeline.ingestChecks(checks)
|
||||
);
|
||||
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
|
||||
const monitorStatusPolicy = new MonitorStatusPolicy();
|
||||
const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger);
|
||||
|
||||
// ***********************
|
||||
// Reactors and dispatcher
|
||||
// Handles notifications and incidents
|
||||
// ***********************
|
||||
|
||||
const notificationReactor = new NotificationReactor(notificationsService);
|
||||
const incidentReactor = new IncidentReactor(incidentService);
|
||||
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
|
||||
|
||||
// ***********************
|
||||
// Check producer/evaluator
|
||||
// Handles creating and evaluatiog checks
|
||||
// ***********************
|
||||
const checkProducer = new CheckProducer(
|
||||
const pipeline = new WorkerPipeline({
|
||||
logger,
|
||||
monitorsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
checksRepository,
|
||||
jobsRepository,
|
||||
checkService,
|
||||
networkService,
|
||||
proxyResolver,
|
||||
bufferService,
|
||||
dockerLogsService,
|
||||
logger
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
|
||||
const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
|
||||
egressService,
|
||||
statusService,
|
||||
dispatcher: reactorDispatcher,
|
||||
});
|
||||
|
||||
const geoCheckPipeline = new GeoChecksPipeline(monitorsRepository, maintenanceWindowsRepository, geoChecksService, bufferService, logger);
|
||||
|
||||
// ***********************
|
||||
// Worker
|
||||
@@ -153,19 +148,22 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
egressService
|
||||
);
|
||||
|
||||
const handlers: JobHandlers = {
|
||||
check: pipeline.handleCheck,
|
||||
evaluate: pipeline.handleEvaluate,
|
||||
"geo-check": geoCheckPipeline.handle,
|
||||
"cleanup-orphaned": workerHelper.getCleanupOrphanedJob(),
|
||||
"cleanup-retention": workerHelper.getCleanupRetentionJob(),
|
||||
egress: workerHelper.getEgressRecoveryJob(),
|
||||
};
|
||||
|
||||
const worker = await DBQueueWorker.create({
|
||||
logger,
|
||||
isDbConnected: () => mongoose.connection.readyState === 1,
|
||||
jobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
bufferService,
|
||||
checkProducer,
|
||||
checkEvaluator,
|
||||
geoCheckPipeline,
|
||||
dispatcher: reactorDispatcher,
|
||||
helper: workerHelper,
|
||||
handlers,
|
||||
queueWorkersRepository,
|
||||
queueMode: envSettings.queueMode,
|
||||
queuePrimaryProcesses: envSettings.queuePrimaryProcesses,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Schema, model, type Types } from "mongoose";
|
||||
import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
|
||||
_id: Types.ObjectId;
|
||||
@@ -45,7 +45,6 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
|
||||
globalProxyId: { type: String, default: null },
|
||||
egressCheckEnabled: { type: Boolean, default: false },
|
||||
egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
|
||||
egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
|
||||
egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
|
||||
},
|
||||
{ timestamps: true }
|
||||
|
||||
@@ -3,7 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
|
||||
import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
|
||||
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
import { DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
class MongoSettingsRepository implements ISettingsRepository {
|
||||
private toEntity = (doc: AppSettingsDocument): Settings => {
|
||||
@@ -33,7 +33,6 @@ class MongoSettingsRepository implements ISettingsRepository {
|
||||
globalProxyId: doc.globalProxyId ?? undefined,
|
||||
egressCheckEnabled: doc.egressCheckEnabled ?? false,
|
||||
egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
|
||||
egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
egressNotifications: doc.egressNotifications ?? [],
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
|
||||
@@ -54,7 +54,6 @@ export interface Settings {
|
||||
// Egress self-check (see domain/egress)
|
||||
egressCheckEnabled: boolean;
|
||||
egressCheckTargets: string[];
|
||||
egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
|
||||
egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import CheckModel from "@/domain/checks/check.model.js";
|
||||
import { DockerContainerStatsBucket, DockerStatsBucket } from "@/domain/checks/check.type.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
|
||||
import mongoose from "mongoose";
|
||||
|
||||
type DateRange = { start: Date; end: Date };
|
||||
@@ -9,6 +10,7 @@ export const getDockerTotalChecks = async (monitorId: string, dates: DateRange):
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
});
|
||||
|
||||
export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Promise<{ totalChecks: number }> => {
|
||||
@@ -16,6 +18,7 @@ export const getDockerUpChecks = async (monitorId: string, dates: DateRange): Pr
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
status: true,
|
||||
});
|
||||
return { totalChecks: count };
|
||||
@@ -28,6 +31,7 @@ export const getDockerStats = async (monitorId: string, dates: DateRange, dateSt
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -48,6 +52,7 @@ export const getDockerLatestCheck = async (monitorId: string) =>
|
||||
CheckModel.findOne({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
})
|
||||
.sort({ createdAt: -1 })
|
||||
.select("containers containerSummary createdAt")
|
||||
@@ -65,6 +70,7 @@ export const getDockerContainerStats = async (
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
"containers.name": containerName,
|
||||
},
|
||||
},
|
||||
@@ -88,6 +94,7 @@ export const getDockerContainerLatestCheck = async (monitorId: string, container
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "docker",
|
||||
"containers.name": containerName,
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
})
|
||||
.sort({ createdAt: -1 })
|
||||
.select("containers createdAt")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { CheckModel } from "@/domain/checks/check.model.js";
|
||||
import mongoose from "mongoose";
|
||||
import type { HardwareDiskStats, HardwareNetStats } from "@/domain/checks/check.type.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
|
||||
|
||||
type DateRange = { start: Date; end: Date };
|
||||
type HardwareUpChecks = { totalChecks: number };
|
||||
@@ -70,6 +71,7 @@ export const getHardwareTotalChecks = async (monitorId: string, dates: DateRange
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "hardware",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -78,6 +80,7 @@ export const getHardwareUpChecks = async (monitorId: string, dates: DateRange):
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "hardware",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
status: true,
|
||||
});
|
||||
return { totalChecks: count };
|
||||
@@ -95,6 +98,7 @@ export const getHardwareStats = async (monitorId: string, dates: DateRange, date
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
"metadata.type": "hardware",
|
||||
createdAt: { $gte: dates.start, $lte: dates.end },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{ $sort: { createdAt: 1 } },
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Schema, model, Types } from "mongoose";
|
||||
import { MonitorTypes, type MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { EgressStatuses } from "@/domain/egress/egress.type.js";
|
||||
import type {
|
||||
Check,
|
||||
CheckAudits,
|
||||
@@ -337,6 +338,11 @@ const CheckSchema = new Schema<CheckDocument>(
|
||||
type: containerSummarySchema,
|
||||
default: undefined,
|
||||
},
|
||||
egressStatus: {
|
||||
type: String,
|
||||
enum: EgressStatuses,
|
||||
default: undefined,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
|
||||
// A check recorded while the instance's own egress was down says nothing about the target, so it is left out of
|
||||
// every aggregate: uptime percentages, up/down counts and the response-time series alike. A degraded episode
|
||||
// therefore reads as a gap in monitoring rather than as downtime or as a dip in response time.
|
||||
// Spread this into the $match or filter of any query that reports statistics. Check listings keep degraded checks.
|
||||
export const EXCLUDE_DEGRADED_EGRESS_MATCH = { egressStatus: { $ne: "degraded" satisfies EgressStatus } } as const;
|
||||
@@ -16,6 +16,7 @@ import type {
|
||||
} from "@/domain/checks/check.type.js";
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { CheckModel, type CheckDocument } from "@/domain/checks/check.model.js";
|
||||
import { EXCLUDE_DEGRADED_EGRESS_MATCH } from "@/domain/checks/check.query.js";
|
||||
import mongoose from "mongoose";
|
||||
import { getDateFormat, getDateForRange } from "@/utils/dataUtils.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
@@ -179,6 +180,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
audits: mapAudits(doc.audits),
|
||||
containers: doc.containers,
|
||||
containerSummary: doc.containerSummary,
|
||||
egressStatus: doc.egressStatus,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
@@ -348,6 +350,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const baseMatch = {
|
||||
"metadata.teamId": new mongoose.Types.ObjectId(teamId),
|
||||
createdAt: { $gte: getDateForRange(dateRange) },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
};
|
||||
|
||||
const [totalResult, downResult] = await Promise.all([
|
||||
@@ -393,6 +396,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
$match: {
|
||||
"metadata.monitorId": { $in: objectIds },
|
||||
createdAt: { $gte: windowStart },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -473,6 +477,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const matchStage = {
|
||||
"metadata.monitorId": monitorObjectId,
|
||||
createdAt: { $gte: startDate, $lte: endDate },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
};
|
||||
const [result] = await CheckModel.aggregate([
|
||||
{ $match: matchStage },
|
||||
@@ -638,6 +643,7 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
const matchStage = {
|
||||
"metadata.monitorId": monitorObjectId,
|
||||
createdAt: { $gte: startDate, $lte: endDate },
|
||||
...EXCLUDE_DEGRADED_EGRESS_MATCH,
|
||||
};
|
||||
|
||||
const [result] = await CheckModel.aggregate([
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { DockerContainerInfo, DockerContainerSummary } from "@/domain/docker/docker.type.js";
|
||||
import type { Response } from "got";
|
||||
|
||||
@@ -116,6 +117,10 @@ export interface Check {
|
||||
seo?: number;
|
||||
performance?: number;
|
||||
audits?: CheckAudits;
|
||||
// Set on failing checks while the egress check is enabled; absent otherwise.
|
||||
// "degraded": no reliability target was reachable (or the instance was already degraded), so the failure is not attributable to the target.
|
||||
// "ok": a reliability target was reachable.
|
||||
egressStatus?: EgressStatus;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import type { INetworkService } from "@/service/networkService.js";
|
||||
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import {
|
||||
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
EGRESS_RECOVERY_POLL_SECONDS,
|
||||
DEFAULT_EGRESS_TARGETS,
|
||||
type EgressAssessment,
|
||||
type EgressProbeResult,
|
||||
@@ -26,7 +26,7 @@ const RECOVERY_JOB_TYPE = "egress" as const;
|
||||
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
|
||||
|
||||
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
|
||||
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
export const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
|
||||
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
|
||||
const PROBE_MONITOR_ID = "egress-probe";
|
||||
@@ -136,11 +136,6 @@ export class EgressService implements IEgressService {
|
||||
return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
|
||||
};
|
||||
|
||||
private toIntervalMs = (seconds: number | undefined): number => {
|
||||
const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
|
||||
return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
|
||||
};
|
||||
|
||||
// ****************************
|
||||
// Event-triggered assessment
|
||||
// ****************************
|
||||
@@ -181,7 +176,7 @@ export class EgressService implements IEgressService {
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status === "degraded") {
|
||||
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
|
||||
await this.scheduleRecoveryCheck(false);
|
||||
return "degraded";
|
||||
}
|
||||
|
||||
@@ -195,7 +190,7 @@ export class EgressService implements IEgressService {
|
||||
|
||||
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
|
||||
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
|
||||
await this.scheduleRecoveryCheck(true);
|
||||
const degraded = await this.egressStateRepository.markDegraded(results, now);
|
||||
this.logger.warn({
|
||||
message: degraded
|
||||
@@ -224,8 +219,8 @@ export class EgressService implements IEgressService {
|
||||
|
||||
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
|
||||
// so re-arming while degraded never pushes a pending run back.
|
||||
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
|
||||
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
|
||||
private scheduleRecoveryCheck = async (reschedule: boolean) => {
|
||||
const intervalMs = EGRESS_RECOVERY_POLL_SECONDS * 1000;
|
||||
const seed: JobSeed = {
|
||||
id: RECOVERY_JOB_ID,
|
||||
type: RECOVERY_JOB_TYPE,
|
||||
|
||||
@@ -9,11 +9,13 @@ export type EgressAssessment = EgressStatus | "disabled";
|
||||
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
|
||||
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
|
||||
export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
|
||||
export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
|
||||
export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
|
||||
export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
|
||||
export const MAX_EGRESS_TARGETS = 10;
|
||||
|
||||
// While the instance is marked degraded, every failing check is flagged from stored state without probing, so
|
||||
// this interval is the window in which a recovered instance still discards its checks. It is deliberately fixed
|
||||
// and not configurable: the only effect of raising it is a longer blind window.
|
||||
export const EGRESS_RECOVERY_POLL_SECONDS = 5;
|
||||
|
||||
// Instance-global singleton persisted in MongoDB so that every worker process and the API
|
||||
// process share one view of the state (the queue worker may run separately from the API).
|
||||
export interface EgressState {
|
||||
|
||||
@@ -82,6 +82,7 @@ IncidentSchema.index({ teamId: 1, startTime: -1 });
|
||||
IncidentSchema.index({ status: 1, startTime: -1 });
|
||||
IncidentSchema.index({ resolutionType: 1, status: 1 });
|
||||
IncidentSchema.index({ resolvedBy: 1, status: 1 });
|
||||
IncidentSchema.index({ monitorId: 1, startTime: -1 });
|
||||
IncidentSchema.index({ createdAt: -1 });
|
||||
|
||||
const IncidentModel = model<IncidentDocument>("Incident", IncidentSchema);
|
||||
|
||||
@@ -17,6 +17,7 @@ export interface IIncidentsRepository {
|
||||
monitorId?: string,
|
||||
resolutionType?: string
|
||||
): Promise<Incident[]>;
|
||||
findByMonitorIdAndDate(monitorId: string, teamId: string, dateStart: Date, dateEnd: Date): Promise<Incident[]>;
|
||||
findSummaryByTeamId(teamId: string, limit?: number): Promise<IncidentSummary>;
|
||||
countByTeamId(teamId: string, startDate: Date | undefined, status?: boolean, monitorId?: string, resolutionType?: string): Promise<number>;
|
||||
|
||||
|
||||
@@ -120,6 +120,17 @@ class MongoIncidentsRepository implements IIncidentsRepository {
|
||||
return this.mapDocuments(incidents);
|
||||
};
|
||||
|
||||
findByMonitorIdAndDate = async (monitorId: string, teamId: string, dateStart: Date, dateEnd: Date): Promise<Incident[]> => {
|
||||
const matchStage = {
|
||||
monitorId: new mongoose.Types.ObjectId(monitorId),
|
||||
teamId: new mongoose.Types.ObjectId(teamId),
|
||||
startTime: { $lt: dateEnd },
|
||||
$or: [{ endTime: null }, { endTime: { $gte: dateStart } }],
|
||||
};
|
||||
const incidents = await IncidentModel.find(matchStage).sort({ startTime: -1 });
|
||||
return this.mapDocuments(incidents);
|
||||
};
|
||||
|
||||
updateById = async (incidentId: string, teamId: string, patch: Partial<Incident>) => {
|
||||
const updatedIncident = await IncidentModel.findOneAndUpdate(
|
||||
{ _id: new mongoose.Types.ObjectId(incidentId), teamId: new mongoose.Types.ObjectId(teamId) },
|
||||
|
||||
@@ -8,7 +8,7 @@ import type { IMonitorsRepository } from "@/domain/monitors/monitor.repository.i
|
||||
import type { IUsersRepository } from "@/domain/users/user.repository.interface.js";
|
||||
import type { Incident, IncidentSummary } from "@/domain/incidents/incident.type.js";
|
||||
import type { User } from "@/domain/users/user.type.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import { DateRange } from "@/types/query.js";
|
||||
|
||||
@@ -20,6 +20,11 @@ export interface Incident {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export type PublicIncident = Pick<
|
||||
Incident,
|
||||
"id" | "monitorId" | "status" | "startTime" | "endTime" | "resolutionType" | "message" | "statusCode" | "createdAt"
|
||||
>;
|
||||
|
||||
export interface IncidentSummaryTopMonitor {
|
||||
monitorId: string;
|
||||
monitorName: string | null;
|
||||
|
||||
@@ -34,7 +34,7 @@ import { IJobScheduler } from "@/worker/worker.interface.js";
|
||||
import { DateRange } from "@/types/query.js";
|
||||
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
|
||||
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
|
||||
import { isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { isCaptureDockerUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
|
||||
const SERVICE_NAME = "MonitorService";
|
||||
|
||||
@@ -194,6 +194,9 @@ export class MonitorService implements IMonitorService {
|
||||
|
||||
// Encrypt docker TLS keys
|
||||
this.encryptDockerTls(body, "create");
|
||||
if (body.type === "docker" && isCaptureDockerUrl(body.url)) {
|
||||
body.dockerLogsEnabled = false;
|
||||
}
|
||||
|
||||
const monitor = await this.monitorsRepository.create(body, teamId, userId);
|
||||
if (!monitor) {
|
||||
@@ -561,6 +564,7 @@ export class MonitorService implements IMonitorService {
|
||||
});
|
||||
}
|
||||
}
|
||||
await this.normalizeCaptureDocker(body, monitorId, teamId);
|
||||
const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body, { unsetProxyId });
|
||||
await this.scheduler.updateJob(editedMonitor);
|
||||
return editedMonitor;
|
||||
@@ -749,6 +753,26 @@ export class MonitorService implements IMonitorService {
|
||||
return { imported: createdMonitors!.length, errors };
|
||||
};
|
||||
|
||||
// Capture-backed Docker monitors authenticate with the API secret and never collect logs; socket and TLS hosts have no secret.
|
||||
private normalizeCaptureDocker = async (body: Partial<Monitor>, monitorId: string, teamId: string) => {
|
||||
if (body.type !== "docker") return;
|
||||
|
||||
if (body.url !== undefined && !isCaptureDockerUrl(body.url)) {
|
||||
body.secret = "";
|
||||
return;
|
||||
}
|
||||
if (body.url === undefined && body.secret === undefined) return;
|
||||
|
||||
const stored = await this.monitorsRepository.findById(monitorId, teamId);
|
||||
if (body.url === undefined && !isCaptureDockerUrl(stored.url)) return;
|
||||
|
||||
body.dockerLogsEnabled = false;
|
||||
const secret = body.secret === undefined ? stored.secret : body.secret.trim();
|
||||
if (!secret) {
|
||||
throw new AppError({ message: "Capture API secret is required", status: 422, service: SERVICE_NAME, method: "editMonitor" });
|
||||
}
|
||||
};
|
||||
|
||||
private encryptDockerTls = (body: Partial<Monitor>, mode: "create" | "edit") => {
|
||||
if (body.type !== "docker" && mode === "create") return;
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import http from "node:http";
|
||||
import { HardwareStats } from "@/domain/checks/check.type.js";
|
||||
import { MonitorStats } from "@/domain/monitor-stats/monitor-stats.type.js";
|
||||
import { DockerLogPage } from "@/domain/docker/docker-log.type.js";
|
||||
import { isDockerSocketUrl } from "@/utils/dockerHost.js";
|
||||
|
||||
export const HttpStatusCodes = [
|
||||
...Object.keys(http.STATUS_CODES).map(Number),
|
||||
@@ -50,6 +51,30 @@ export const UptimeDetailsSupportedTypes = ["http", "ping", "port", "game", "grp
|
||||
export type UptimeDetailsSupportedType = (typeof UptimeDetailsSupportedTypes)[number];
|
||||
export const supportsUptimeDetails = (type: MonitorType): type is UptimeDetailsSupportedType => UptimeDetailsSupportedTypes.some((t) => t === type);
|
||||
|
||||
// Types whose check leaves the instance, and whose failure to reach the target can therefore be the instance's
|
||||
// own loss of egress. A hardware check is an ordinary outbound HTTP request to the Capture agent, so it counts
|
||||
// however far away the agent is. `unknown` is excluded because no request is made for it.
|
||||
export const EgressAttributableTypes = [
|
||||
"http",
|
||||
"ping",
|
||||
"pagespeed",
|
||||
"hardware",
|
||||
"docker",
|
||||
"port",
|
||||
"game",
|
||||
"grpc",
|
||||
"websocket",
|
||||
"dns",
|
||||
] as const satisfies readonly MonitorType[];
|
||||
export type EgressAttributableType = (typeof EgressAttributableTypes)[number];
|
||||
|
||||
// A Docker daemon reached over a unix socket (`unix:///path` or a bare absolute path) is local IPC and cannot
|
||||
// fail through egress; one reached over TCP or TLS is as remote as any other target. The URL, not the type, settles it.
|
||||
const isLocalDockerSocket = (monitor: Pick<Monitor, "type" | "url">): boolean => monitor.type === "docker" && isDockerSocketUrl(monitor.url);
|
||||
|
||||
export const isEgressAttributable = (monitor: Pick<Monitor, "type" | "url">): boolean =>
|
||||
EgressAttributableTypes.some((t) => t === monitor.type) && !isLocalDockerSocket(monitor);
|
||||
|
||||
export const MonitorStatuses = ["up", "down", "paused", "initializing", "maintenance", "breached"] as const;
|
||||
export type MonitorStatus = (typeof MonitorStatuses)[number];
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { HardwareStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
import type {
|
||||
NotificationMessage,
|
||||
NotificationType,
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { NotificationMessage } from "@/domain/notifications/notification.ty
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
|
||||
import { INotificationProvider } from "@/domain/notifications/providers/INotificationProvider.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
import type { MonitorActionDecision } from "@/worker/worker.interface.js";
|
||||
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import type { INotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import { type IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
|
||||
import type { Incident, PublicIncident } from "@/domain/incidents/incident.type.js";
|
||||
import { type IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js";
|
||||
import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import type { DailyCheckBucket } from "@/domain/checks/check.type.js";
|
||||
import dayjs from "dayjs";
|
||||
import utc from "dayjs/plugin/utc.js";
|
||||
import timezone from "dayjs/plugin/timezone.js";
|
||||
import {
|
||||
DEFAULT_STATUS_PAGE_THEME,
|
||||
DEFAULT_STATUS_PAGE_THEME_MODE,
|
||||
@@ -12,8 +19,9 @@ import {
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { normalizeStatusPageDomain } from "@/utils/statusPageDomain.js";
|
||||
import { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import type { DailyCheckBucket } from "@/domain/checks/check.type.js";
|
||||
|
||||
dayjs.extend(utc);
|
||||
dayjs.extend(timezone);
|
||||
|
||||
export interface IStatusPageService {
|
||||
createStatusPage(userId: string, teamId: string, image: Express.Multer.File | undefined, data: Partial<StatusPage>): Promise<StatusPage>;
|
||||
@@ -21,6 +29,7 @@ export interface IStatusPageService {
|
||||
getStatusPageByCustomDomain(customDomain: string): Promise<StatusPage>;
|
||||
getStatusPagesByTeamId(teamId: string): Promise<StatusPage[]>;
|
||||
getPublicStatusPagePayload(statusPage: StatusPage, requesterTeamId: string | undefined, range: StatusPageRange): Promise<PublicStatusPagePayload>;
|
||||
getPublicMonitorIncidents(url: string, monitorId: string, date: string, requesterTeamId?: string): Promise<PublicIncident[]>;
|
||||
updateStatusPage(id: string, teamId: string, image: Express.Multer.File | undefined, data: Partial<StatusPage>): Promise<StatusPage>;
|
||||
|
||||
deleteStatusPage(statusPageId: string, teamId: string): Promise<StatusPage>;
|
||||
@@ -31,7 +40,8 @@ export class StatusPageService implements IStatusPageService {
|
||||
private statusPagesRepository: IStatusPagesRepository,
|
||||
private settingsService: ISettingsService,
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private checksRepository: IChecksRepository
|
||||
private checksRepository: IChecksRepository,
|
||||
private incidentsRepository: IIncidentsRepository
|
||||
) {}
|
||||
|
||||
private assertCustomDomainAllowed = (customDomain: string | null | undefined) => {
|
||||
@@ -95,6 +105,18 @@ export class StatusPageService implements IStatusPageService {
|
||||
return base;
|
||||
};
|
||||
|
||||
private toPublicIncident = (incident: Incident): PublicIncident => ({
|
||||
id: incident.id,
|
||||
monitorId: incident.monitorId,
|
||||
status: incident.status,
|
||||
startTime: incident.startTime,
|
||||
endTime: incident.endTime,
|
||||
resolutionType: incident.resolutionType,
|
||||
message: incident.message ?? null,
|
||||
statusCode: incident.statusCode ?? null,
|
||||
createdAt: incident.createdAt,
|
||||
});
|
||||
|
||||
createStatusPage = async (
|
||||
userId: string,
|
||||
teamId: string,
|
||||
@@ -167,6 +189,29 @@ export class StatusPageService implements IStatusPageService {
|
||||
};
|
||||
};
|
||||
|
||||
getPublicMonitorIncidents = async (url: string, monitorId: string, date: string, requesterTeamId?: string): Promise<PublicIncident[]> => {
|
||||
const statusPage = await this.getStatusPageByUrl(url);
|
||||
|
||||
// Ensure the status page is public or the requester is the owner
|
||||
if (!statusPage.isPublished) {
|
||||
if (!requesterTeamId || statusPage.teamId !== requesterTeamId) {
|
||||
throw new AppError({ message: "Forbidden", status: 403 });
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the requested monitor actually belongs to this status page
|
||||
if (!statusPage.monitors.includes(monitorId)) {
|
||||
throw new AppError({ message: "Monitor not found on this status page", status: 404 });
|
||||
}
|
||||
|
||||
const tz = statusPage.timezone ?? "Etc/UTC";
|
||||
const dateStart = dayjs.tz(date, tz).startOf("day").toDate();
|
||||
const dateEnd = dayjs.tz(date, tz).endOf("day").toDate();
|
||||
|
||||
const incidents = await this.incidentsRepository.findByMonitorIdAndDate(monitorId, statusPage.teamId, dateStart, dateEnd);
|
||||
return incidents.map(this.toPublicIncident);
|
||||
};
|
||||
|
||||
updateStatusPage = async (id: string, teamId: string, image: Express.Multer.File | undefined, data: Partial<StatusPage>): Promise<StatusPage> => {
|
||||
const normalizedData = this.normalizeCustomDomainInput(this.normalizeInput(data));
|
||||
const updated = await this.statusPagesRepository.updateById(id, teamId, image, normalizedData);
|
||||
|
||||
@@ -3,18 +3,15 @@ import type { GeoCheck } from "@/domain/geo-checks/geo-check.type.js";
|
||||
import type { IGeoChecksService } from "../domain/geo-checks/geo-check.service.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { ICheckService } from "../domain/checks/check.service.js";
|
||||
import { DockerLog } from "@/domain/docker/docker-log.type.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
const SERVICE_NAME = "BufferService";
|
||||
|
||||
export type CheckIngest = (checks: Check[]) => Promise<void>;
|
||||
export interface IBufferService {
|
||||
addToBuffer(check: Check): void;
|
||||
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
|
||||
addDockerLogToBuffer(dockerLog: DockerLog): void;
|
||||
ingestChecks(checks: Check[]): Promise<void>;
|
||||
scheduleNextFlush(): void;
|
||||
flushBuffer(): Promise<void>;
|
||||
flushGeoBuffer(): Promise<void>;
|
||||
@@ -31,28 +28,20 @@ export class BufferService implements IBufferService {
|
||||
private geoBuffer: GeoCheck[];
|
||||
private dockerLogBuffer: DockerLog[];
|
||||
private bufferTimer: NodeJS.Timeout | null = null;
|
||||
private checksService: ICheckService;
|
||||
private geoChecksService: IGeoChecksService;
|
||||
private dockerLogsService: IDockerLogsService;
|
||||
private jobsRepository: IJobsRepository;
|
||||
|
||||
// Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated.
|
||||
private unarmed: Map<string, PendingCheck[]> = new Map();
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
checkService: ICheckService,
|
||||
geoChecksService: IGeoChecksService,
|
||||
dockerLogsService: IDockerLogsService,
|
||||
settingsService: ISettingsService,
|
||||
jobsRepository: IJobsRepository
|
||||
private ingest: CheckIngest
|
||||
) {
|
||||
this.BUFFER_TIMEOUT = settingsService.getSettings().nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute
|
||||
this.logger = logger;
|
||||
this.checksService = checkService;
|
||||
this.geoChecksService = geoChecksService;
|
||||
this.dockerLogsService = dockerLogsService;
|
||||
this.jobsRepository = jobsRepository;
|
||||
this.SERVICE_NAME = SERVICE_NAME;
|
||||
this.buffer = [];
|
||||
this.geoBuffer = [];
|
||||
@@ -104,38 +93,6 @@ export class BufferService implements IBufferService {
|
||||
}
|
||||
}
|
||||
|
||||
ingestChecks = async (checks: Check[]) => {
|
||||
if (checks.length > 0) await this.checksService.createChecks(checks);
|
||||
|
||||
// Group by monitor, starting from any entries a previous flush failed to arm
|
||||
const byMonitor = this.unarmed;
|
||||
this.unarmed = new Map();
|
||||
for (const check of checks) {
|
||||
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
|
||||
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
|
||||
byMonitor.set(check.metadata.monitorId, pending);
|
||||
}
|
||||
if (byMonitor.size === 0) return;
|
||||
|
||||
const now = Date.now();
|
||||
const ops: Promise<boolean>[] = [];
|
||||
for (const [monitorId, pendingChecks] of byMonitor) {
|
||||
ops.push(
|
||||
this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
|
||||
// The checks are already stored, so a failed arm is retried on the next flush rather than dropped
|
||||
this.unarmed.set(monitorId, pendingChecks);
|
||||
this.logger.error({
|
||||
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "ingestChecks",
|
||||
});
|
||||
return false;
|
||||
})
|
||||
);
|
||||
}
|
||||
await Promise.all(ops);
|
||||
};
|
||||
|
||||
scheduleNextFlush() {
|
||||
if (this.bufferTimer) {
|
||||
clearTimeout(this.bufferTimer);
|
||||
@@ -160,19 +117,14 @@ export class BufferService implements IBufferService {
|
||||
}
|
||||
|
||||
async flushBuffer() {
|
||||
if (this.buffer.length === 0 && this.unarmed.size === 0) {
|
||||
return;
|
||||
}
|
||||
// Take the batch first so a write that drains it can't be appended to mid-flush
|
||||
const batch = this.buffer;
|
||||
this.buffer = [];
|
||||
try {
|
||||
this.logger.debug({
|
||||
message: `Flushing ${batch.length} checks to database`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushBuffer",
|
||||
});
|
||||
await this.ingestChecks(batch);
|
||||
if (batch.length > 0) {
|
||||
this.logger.debug({ message: `Flushing ${batch.length} checks to database`, service: this.SERVICE_NAME, method: "flushBuffer" });
|
||||
}
|
||||
await this.ingest(batch);
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : "Unknown error",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { DNSStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import dns from "dns";
|
||||
import type { Resolver } from "dns/promises";
|
||||
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
@@ -8,6 +9,16 @@ import { NETWORK_ERROR } from "@/types/network.js";
|
||||
|
||||
const SERVICE_NAME = "DNSProvider";
|
||||
|
||||
// c-ares codes where the DNS server replied: the name does not exist, holds no such record, or the server
|
||||
// declined or answered badly. Anything else (timeout, connection refused, a local resolver fault) means we
|
||||
// never got an answer, so the failure could be our own loss of egress rather than the target's.
|
||||
const ANSWERED_DNS_CODES: ReadonlySet<string> = new Set([dns.NODATA, dns.FORMERR, dns.SERVFAIL, dns.NOTFOUND, dns.NOTIMP, dns.REFUSED, dns.BADRESP]);
|
||||
|
||||
const dnsServerAnswered = (error: unknown): boolean => {
|
||||
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
||||
return typeof code === "string" && ANSWERED_DNS_CODES.has(code);
|
||||
};
|
||||
|
||||
export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
|
||||
readonly type = "dns";
|
||||
constructor(private createResolver: () => Resolver) {}
|
||||
@@ -63,6 +74,7 @@ export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: NETWORK_ERROR,
|
||||
peerResponded: dnsServerAnswered(error),
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
responseTime,
|
||||
payload: {
|
||||
@@ -81,6 +93,7 @@ export class DNSProvider implements IStatusProvider<DNSStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "Success",
|
||||
responseTime,
|
||||
payload: {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import https from "node:https";
|
||||
|
||||
import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import { CheckContext, DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { HttpProvider } from "@/service/network/HttpProvider.js";
|
||||
import { CaptureDockerContainer, CaptureDockerStatusPayload, CheckContext, DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
|
||||
import {
|
||||
DOCKER_LOG_TAIL_LINES,
|
||||
@@ -23,10 +24,10 @@ import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import Dockerode from "dockerode";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import { IEncryptionService } from "@/service/encryption/encryptionService.js";
|
||||
import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { DOCKER_TLS_URL, isCaptureDockerUrl, isDockerTlsUrl } from "@/utils/dockerHost.js";
|
||||
import { splitCertificateBundle } from "@/utils/pem.js";
|
||||
|
||||
type DockerodeType = typeof Dockerode;
|
||||
@@ -40,6 +41,7 @@ const DOCKER_LOG_MAX_LINE_BYTES = 4096;
|
||||
const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]";
|
||||
const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /;
|
||||
const DOCKER_TLS_DEFAULT_PORT = 2376;
|
||||
const CAPTURE_FATAL_ERROR_METRICS = new Set(["docker.client", "docker.container.list"]);
|
||||
|
||||
export interface DockerError extends Error {
|
||||
statusCode?: number;
|
||||
@@ -53,7 +55,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
constructor(
|
||||
private logger: ILogger,
|
||||
private DockerLib: DockerodeType,
|
||||
private encryptionService: IEncryptionService
|
||||
private encryptionService: IEncryptionService,
|
||||
private httpProvider: HttpProvider
|
||||
) {}
|
||||
|
||||
supports(type: MonitorType): boolean {
|
||||
@@ -137,6 +140,93 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return status && (DockerHealthStatuses as readonly string[]).includes(status) ? (status as DockerHealthStatus) : "none";
|
||||
}
|
||||
|
||||
private toContainerSummary(containers: DockerContainerInfo[]): DockerContainerSummary {
|
||||
return {
|
||||
total: containers.length,
|
||||
running: containers.filter((container) => container.state === "running").length,
|
||||
stopped: containers.filter((container) => container.state === "exited" || container.state === "dead").length,
|
||||
unhealthy: containers.filter((container) => container.health === "unhealthy").length,
|
||||
};
|
||||
}
|
||||
|
||||
private toCaptureContainer(container: CaptureDockerContainer): DockerContainerInfo {
|
||||
const startedAt = container.started_at && container.started_at > 0 ? new Date(container.started_at * 1000).toISOString() : undefined;
|
||||
// Capture reports zeroed stats for stopped containers; only running ones carry real metrics, matching toContainerInfo.
|
||||
const stats =
|
||||
container.status === "running" && container.stats
|
||||
? {
|
||||
cpuPct: container.stats.cpu_percent / 100,
|
||||
memoryUsedBytes: container.stats.memory_usage,
|
||||
memoryLimitBytes: container.stats.memory_limit,
|
||||
memoryPct: container.stats.memory_percent / 100,
|
||||
}
|
||||
: {};
|
||||
return {
|
||||
id: container.container_id,
|
||||
name: container.container_name || container.container_id.slice(0, 12),
|
||||
image: container.base_image,
|
||||
state: this.toContainerState(container.status),
|
||||
status: container.status,
|
||||
// Capture derives health from state when no Docker healthcheck exists.
|
||||
// Keep Checkmate's `none` value in that case so summaries retain their existing semantics.
|
||||
health: container.health?.source === "container_health_check" ? (container.health.healthy ? "healthy" : "unhealthy") : "none",
|
||||
startedAt,
|
||||
ports: (container.exposed_ports ?? []).flatMap((port) => {
|
||||
const privatePort = Number(port.port);
|
||||
return Number.isInteger(privatePort) ? [{ privatePort, protocol: this.toPortProtocol(port.protocol) }] : [];
|
||||
}),
|
||||
...stats,
|
||||
};
|
||||
}
|
||||
|
||||
private isCapturePayload = (payload: unknown): payload is CaptureDockerStatusPayload => {
|
||||
if (!payload || typeof payload !== "object") return false;
|
||||
const { data, errors } = payload as Partial<CaptureDockerStatusPayload>;
|
||||
if (!Array.isArray(data) && data !== null) return false;
|
||||
if (errors === undefined || errors === null) return true;
|
||||
return Array.isArray(errors) && errors.every((error) => Array.isArray(error?.metric));
|
||||
};
|
||||
|
||||
private handleCapture = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
|
||||
const captureUrl = new URL(monitor.url.trim());
|
||||
captureUrl.searchParams.set("all", "true");
|
||||
const response = await this.httpProvider.handle<CaptureDockerStatusPayload>({ ...monitor, url: captureUrl.toString() }, ctx);
|
||||
if (!response.status) return this.failed(monitor, response.code, response.message, response.responseTime ?? 0);
|
||||
|
||||
const payload = response.payload;
|
||||
if (!this.isCapturePayload(payload)) {
|
||||
this.logger.warn({
|
||||
message: "Capture returned an invalid Docker metrics payload",
|
||||
service: SERVICE_NAME,
|
||||
method: "handleCapture",
|
||||
});
|
||||
return this.failed(monitor, NETWORK_ERROR, "Capture returned an invalid Docker metrics payload", response.responseTime ?? 0);
|
||||
}
|
||||
|
||||
const errors = payload.errors ?? [];
|
||||
const fatalError = errors.find((error) => error.metric.some((metric) => CAPTURE_FATAL_ERROR_METRICS.has(metric)));
|
||||
if (fatalError) return this.failed(monitor, NETWORK_ERROR, fatalError.err, response.responseTime ?? 0);
|
||||
if (errors.length > 0) {
|
||||
this.logger.warn({
|
||||
message: `Capture returned ${errors.length} partial Docker collection error(s)`,
|
||||
service: SERVICE_NAME,
|
||||
method: "handleCapture",
|
||||
});
|
||||
}
|
||||
|
||||
const containers = (payload.data ?? []).map((container) => this.toCaptureContainer(container));
|
||||
return {
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: response.code,
|
||||
message: "Docker host is reachable through Capture",
|
||||
responseTime: response.responseTime,
|
||||
payload: { containers, summary: this.toContainerSummary(containers) },
|
||||
};
|
||||
};
|
||||
|
||||
private computeCpuPct(stats: Dockerode.ContainerStats): number {
|
||||
const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage;
|
||||
const systemDelta = stats.cpu_stats.system_cpu_usage - (stats.precpu_stats.system_cpu_usage ?? 0);
|
||||
@@ -315,12 +405,19 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
return rawLogs.filter((log) => log !== null);
|
||||
};
|
||||
|
||||
private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse<DockerStatusPayload> => ({
|
||||
private failed = (
|
||||
monitor: Monitor,
|
||||
code: number,
|
||||
message: string,
|
||||
responseTime: number,
|
||||
peerResponded = false
|
||||
): MonitorStatusResponse<DockerStatusPayload> => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code,
|
||||
peerResponded,
|
||||
message,
|
||||
responseTime,
|
||||
payload: null,
|
||||
@@ -328,6 +425,8 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
|
||||
handle = async (monitor: Monitor, ctx?: CheckContext): Promise<MonitorStatusResponse<DockerStatusPayload>> => {
|
||||
try {
|
||||
if (isCaptureDockerUrl(monitor.url)) return await this.handleCapture(monitor, ctx);
|
||||
|
||||
const credentials = this.resolveTlsCredentials(monitor, ctx);
|
||||
if (credentials && !credentials.ok) return this.failed(monitor, NETWORK_ERROR, credentials.message, 0);
|
||||
|
||||
@@ -337,31 +436,31 @@ export class DockerProvider implements IStatusProvider<DockerStatusPayload> {
|
||||
if (error) {
|
||||
let message = "Docker host is unreachable";
|
||||
let code = NETWORK_ERROR;
|
||||
// A status code from the daemon is the daemon answering; otherwise only a refusal or a reset
|
||||
// proves we reached the host, and anything else could be the instance's own loss of egress.
|
||||
let peerResponded = peerAnsweredFromError(error);
|
||||
if (this.isDockerError(error)) {
|
||||
code = error.statusCode ?? NETWORK_ERROR;
|
||||
message = error.json?.message ?? error.reason ?? error.message;
|
||||
peerResponded = peerResponded || error.statusCode !== undefined;
|
||||
} else if (error instanceof Error) {
|
||||
message = error.message;
|
||||
}
|
||||
return this.failed(monitor, code, message, responseTime);
|
||||
return this.failed(monitor, code, message, responseTime, peerResponded);
|
||||
}
|
||||
|
||||
const summaries = await docker.listContainers({ all: true });
|
||||
const containers = await this.mapWithConcurrency(summaries, STATS_CONCURRENCY, (s) => this.toContainerInfo(docker, s));
|
||||
const logs = monitor.dockerLogsEnabled ? await this.collectLogs(docker, summaries) : undefined;
|
||||
|
||||
const summary: DockerContainerSummary = {
|
||||
total: containers.length,
|
||||
running: containers.filter((c) => c.state === "running").length,
|
||||
stopped: containers.filter((c) => c.state === "exited" || c.state === "dead").length,
|
||||
unhealthy: containers.filter((c) => c.health === "unhealthy").length,
|
||||
};
|
||||
const summary = this.toContainerSummary(containers);
|
||||
return {
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "Docker host is reachable",
|
||||
responseTime,
|
||||
payload: { containers, summary, logs },
|
||||
|
||||
@@ -43,6 +43,9 @@ export class GameProvider implements IStatusProvider<GameStatusPayload> {
|
||||
});
|
||||
|
||||
if (!state) {
|
||||
// gamedig reports a failed query the same way whichever leg failed, so `peerResponded` is left
|
||||
// unset and a failure is put to the egress probe. A game server that stops answering during a
|
||||
// spell is therefore recorded as degraded rather than down.
|
||||
return {
|
||||
monitorId: id,
|
||||
teamId: teamId,
|
||||
|
||||
@@ -25,6 +25,17 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
|
||||
return type === "grpc";
|
||||
}
|
||||
|
||||
// gRPC codes that mean the call never reached a server. Everything else in 0-16 is the server's own answer,
|
||||
// including NOT_SERVING, which shares NETWORK_ERROR with a connection failure and so cannot be read from `code`.
|
||||
// A call that fails with no gRPC code at all never got as far as the protocol.
|
||||
private peerAnswered(code: number | undefined): boolean {
|
||||
const UNREACHABLE_GRPC_CODES: ReadonlySet<number> = new Set([
|
||||
4, // DEADLINE_EXCEEDED
|
||||
14, // UNAVAILABLE
|
||||
]);
|
||||
return typeof code === "number" && !UNREACHABLE_GRPC_CODES.has(code);
|
||||
}
|
||||
|
||||
private getGrpcStatusName(code: number): string {
|
||||
const statusNames: Record<number, string> = {
|
||||
0: "OK",
|
||||
@@ -64,7 +75,7 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
|
||||
const target = `${host}:${port}`;
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const protoPath = path.join(path.dirname(currentFilePath), "protos", "health.proto");
|
||||
const protoPath = path.join(path.dirname(currentFilePath), "..", "protos", "health.proto");
|
||||
const packageDefinition = this.protoLoader.loadSync(protoPath, {
|
||||
keepCase: true,
|
||||
longs: String,
|
||||
@@ -148,6 +159,7 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
|
||||
type: type,
|
||||
status: false,
|
||||
code: grpcError.grpcCode ?? 5000,
|
||||
peerResponded: this.peerAnswered(grpcError.grpcCode),
|
||||
message: grpcError.message ?? "gRPC health check failed",
|
||||
responseTime,
|
||||
payload: payload ?? null,
|
||||
@@ -163,6 +175,8 @@ export class GrpcProvider implements IStatusProvider<GrpcStatusPayload> {
|
||||
type: type,
|
||||
status: isServing,
|
||||
code: isServing ? 200 : 5000,
|
||||
// The server replied, whatever it said, so this failure is never the instance's own egress.
|
||||
peerResponded: true,
|
||||
message: isServing ? `gRPC service healthy (${grpcPayload.servingStatus})` : `gRPC service unhealthy (${grpcPayload.servingStatus})`,
|
||||
responseTime,
|
||||
payload: grpcPayload,
|
||||
|
||||
@@ -6,7 +6,7 @@ import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { Agent as HttpsAgent } from "https";
|
||||
import { Agent as HttpAgent } from "http";
|
||||
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { isStatusUp } from "@/service/network/utils.js";
|
||||
import { isStatusUp, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import CacheableLookup from "cacheable-lookup";
|
||||
import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
|
||||
@@ -107,6 +107,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: statusUp,
|
||||
code: statusCode,
|
||||
peerResponded: true,
|
||||
message,
|
||||
responseTime,
|
||||
timings,
|
||||
@@ -123,6 +124,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: statusCode,
|
||||
peerResponded: true,
|
||||
message: "Response is not JSON",
|
||||
responseTime,
|
||||
timings,
|
||||
@@ -149,6 +151,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: statusUp && matchResult.ok,
|
||||
code: statusCode,
|
||||
peerResponded: true,
|
||||
message: matchResult.ok ? message : matchResult.message,
|
||||
responseTime,
|
||||
timings,
|
||||
@@ -172,6 +175,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: statusCode ?? NETWORK_ERROR,
|
||||
peerResponded: statusCode !== undefined || (!ctx?.proxyUrl && peerAnsweredFromError(error)),
|
||||
message: message,
|
||||
responseTime,
|
||||
timings: error.timings,
|
||||
@@ -196,6 +200,7 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: NETWORK_ERROR,
|
||||
peerResponded: !ctx?.proxyUrl && peerAnsweredFromError(error),
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
responseTime: 0,
|
||||
payload: null as T,
|
||||
|
||||
@@ -58,6 +58,9 @@ export class PingProvider implements IStatusProvider<PingStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: response.alive ?? false,
|
||||
code: response.alive ? 200 : 5000,
|
||||
// An echo reply is the host answering. Its absence is silence, which is indistinguishable
|
||||
// from the instance having lost its egress, so the egress probe decides that case.
|
||||
peerResponded: response.alive ?? false,
|
||||
message: response.alive ? "Success" : "Ping failed",
|
||||
responseTime,
|
||||
payload: response,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import { PortStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import * as net from "net";
|
||||
type NetType = typeof net;
|
||||
@@ -59,6 +59,9 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: NETWORK_ERROR,
|
||||
// A refused or reset connection came from the host itself; a timeout or an unreachable
|
||||
// network did not, and could equally be the instance's own loss of egress.
|
||||
peerResponded: peerAnsweredFromError(error),
|
||||
message: errorMessage,
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
@@ -72,6 +75,7 @@ export class PortProvider implements IStatusProvider<PortStatusPayload> {
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "Port check successful",
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { IStatusProvider } from "@/service/network/IStatusProvider.js";
|
||||
import { WebSocketStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { Monitor, MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { timeRequest, peerAnsweredFromError } from "@/service/network/utils.js";
|
||||
import { NETWORK_ERROR } from "@/types/network.js";
|
||||
import type WebSocket from "ws";
|
||||
|
||||
@@ -63,6 +63,9 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: NETWORK_ERROR,
|
||||
// `ws` rejects a non-101 handshake with a plain Error carrying only a message, so only the
|
||||
// errno cases — a refusal or a reset — can be told apart from the instance losing egress.
|
||||
peerResponded: peerAnsweredFromError(error),
|
||||
message: errorMessage,
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
@@ -76,6 +79,7 @@ export class WebSocketProvider implements IStatusProvider<WebSocketStatusPayload
|
||||
type: monitor.type,
|
||||
status: true,
|
||||
code: 200,
|
||||
peerResponded: true,
|
||||
message: "WebSocket check successful",
|
||||
responseTime: responseTime,
|
||||
timings: undefined,
|
||||
|
||||
@@ -16,3 +16,27 @@ export const isStatusUp = (statusCode: number | undefined, customUpCodes: HttpSt
|
||||
if (statusCode === undefined) return false;
|
||||
return (statusCode >= 200 && statusCode < 300) || customUpCodes.includes(statusCode);
|
||||
};
|
||||
|
||||
// Errno codes that can only arise once the target's own stack has answered: it refused the connection, reset
|
||||
// an established one, or got far enough into a TLS handshake to present a certificate. Every other failure —
|
||||
// a timeout, an unreachable network, a name that would not resolve — looks the same whether the target is
|
||||
// down or the instance has lost its egress, so those are left for the egress probe to decide.
|
||||
const PEER_ANSWERED_ERROR_CODES: ReadonlySet<string> = new Set([
|
||||
"ECONNREFUSED",
|
||||
"ECONNRESET",
|
||||
"EPIPE",
|
||||
"EPROTO",
|
||||
"CERT_HAS_EXPIRED",
|
||||
"CERT_NOT_YET_VALID",
|
||||
"DEPTH_ZERO_SELF_SIGNED_CERT",
|
||||
"SELF_SIGNED_CERT_IN_CHAIN",
|
||||
"UNABLE_TO_GET_ISSUER_CERT_LOCALLY",
|
||||
"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
|
||||
"ERR_TLS_CERT_ALTNAME_INVALID",
|
||||
"ERR_SSL_WRONG_VERSION_NUMBER",
|
||||
]);
|
||||
|
||||
export const peerAnsweredFromError = (error: unknown): boolean => {
|
||||
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
||||
return typeof code === "string" && PEER_ANSWERED_ERROR_CODES.has(code);
|
||||
};
|
||||
@@ -33,6 +33,12 @@ export interface MonitorStatusResponse<
|
||||
status: boolean;
|
||||
code: number;
|
||||
message: string;
|
||||
// Whether the peer answered, for failures where `code` alone cannot say. `code` carries an HTTP status for
|
||||
// providers that speak HTTP, but DNS and gRPC report their own codes, and gRPC reuses NETWORK_ERROR for a
|
||||
// server that replied NOT_SERVING. Providers that can tell the two apart set this; the rest leave it unset
|
||||
// and the HTTP-status heuristic stands. Consumed by the egress check, which must only blame the instance's
|
||||
// own connectivity when nothing answered.
|
||||
peerResponded?: boolean;
|
||||
responseTime?: number;
|
||||
payload?: T | string | null;
|
||||
timings?: GotTimings;
|
||||
@@ -99,6 +105,34 @@ export interface HardwareStatusPayload {
|
||||
|
||||
// Docker host monitoring
|
||||
|
||||
export interface CaptureDockerContainer {
|
||||
container_id: string;
|
||||
container_name: string;
|
||||
status: string;
|
||||
health?: {
|
||||
healthy: boolean;
|
||||
source: string;
|
||||
} | null;
|
||||
running: boolean;
|
||||
base_image: string;
|
||||
exposed_ports?: Array<{
|
||||
port: string;
|
||||
protocol: string;
|
||||
}>;
|
||||
started_at?: number;
|
||||
stats?: {
|
||||
cpu_percent: number;
|
||||
memory_usage: number;
|
||||
memory_limit: number;
|
||||
memory_percent: number;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface CaptureDockerStatusPayload {
|
||||
data: CaptureDockerContainer[] | null;
|
||||
errors: CheckErrorInfo[] | null;
|
||||
}
|
||||
|
||||
export interface DockerStatusPayload {
|
||||
containers: DockerContainerInfo[];
|
||||
summary: DockerContainerSummary;
|
||||
|
||||
@@ -3,3 +3,13 @@ export const DOCKER_SOCKET_URL = /^(?:unix:\/\/\/\S+|\/\S+)$/;
|
||||
|
||||
export const isDockerTlsUrl = (url: string | undefined): boolean => DOCKER_TLS_URL.test(url?.trim() ?? "");
|
||||
export const isDockerSocketUrl = (url: string | undefined): boolean => DOCKER_SOCKET_URL.test(url?.trim() ?? "");
|
||||
|
||||
export const isCaptureDockerUrl = (url: string | undefined): boolean => {
|
||||
try {
|
||||
const parsed = new URL(url?.trim() ?? "");
|
||||
const pathname = parsed.pathname.replace(/\/+$/, "");
|
||||
return (parsed.protocol === "http:" || parsed.protocol === "https:") && pathname.endsWith("/metrics/docker");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
@@ -1,34 +0,0 @@
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
|
||||
export interface ICheckEvaluator {
|
||||
evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
|
||||
}
|
||||
|
||||
export class CheckEvaluator implements ICheckEvaluator {
|
||||
constructor(
|
||||
private statusService: IStatusService,
|
||||
private monitorStatusPolicy: IMonitorStatusPolicy
|
||||
) {}
|
||||
evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
|
||||
// ****************************
|
||||
// Step 3: Evaluate and return result to reactors
|
||||
// ****************************
|
||||
const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor);
|
||||
|
||||
// Step 5. Get decisions and create an evaluation obj
|
||||
const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
|
||||
const evaluation: MonitorEvaluation = {
|
||||
monitor: statusChangeResult.monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: statusChangeResult,
|
||||
decision,
|
||||
};
|
||||
return evaluation;
|
||||
};
|
||||
}
|
||||
@@ -1,104 +0,0 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IBufferService } from "@/service/bufferService.js";
|
||||
import { INetworkService } from "@/service/networkService.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
|
||||
export interface ICheckProducer {
|
||||
produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "CheckProducer";
|
||||
|
||||
export class CheckProducer implements ICheckProducer {
|
||||
constructor(
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
|
||||
private checkService: ICheckService,
|
||||
private networkService: INetworkService,
|
||||
private proxyResolver: IProxyResolver,
|
||||
private bufferService: IBufferService,
|
||||
private dockerLogsService: IDockerLogsService,
|
||||
private logger: ILogger
|
||||
) {}
|
||||
|
||||
private async isInMaintenanceWindow(monitorId: string, teamId: string) {
|
||||
const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
|
||||
const now = new Date();
|
||||
return windows.some((w) => isWindowActive(w, now));
|
||||
}
|
||||
|
||||
private async resolveDockerTlsKey(monitor: Monitor): Promise<string | undefined> {
|
||||
if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined;
|
||||
const dockerTlsKey = await this.monitorsRepository.findDockerTlsKeyById(monitor.id);
|
||||
return dockerTlsKey ?? undefined;
|
||||
}
|
||||
|
||||
produce = async (monitor: Monitor) => {
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
|
||||
}
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
// ****************************
|
||||
|
||||
// Step 1a: Maintenance window gate - skip if in maintenance
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
});
|
||||
if (monitor.status !== "maintenance") {
|
||||
// Clear the status window to avoid incidents being created on next check
|
||||
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 1b: Acquire status
|
||||
const proxyUrl = await this.proxyResolver.resolve(monitor);
|
||||
const dockerTlsKey = await this.resolveDockerTlsKey(monitor);
|
||||
|
||||
const status = await this.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey });
|
||||
if (!status) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
// ****************************
|
||||
|
||||
// Step 2a: Create & record a check, return null if fail
|
||||
const check = this.checkService.toCheck(status);
|
||||
if (!check) {
|
||||
this.logger.warn({
|
||||
message: `No check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
details: { code: status.code, message: status.message },
|
||||
});
|
||||
return null;
|
||||
}
|
||||
// Step 2b: Add to buffer
|
||||
this.bufferService.addToBuffer(check);
|
||||
|
||||
// Step 2c: Handle docker logs
|
||||
if (status.type === "docker") {
|
||||
const dockerLogs = await this.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
|
||||
for (const dockerLog of dockerLogs) {
|
||||
this.bufferService.addDockerLogToBuffer(dockerLog);
|
||||
}
|
||||
}
|
||||
return { status, check };
|
||||
};
|
||||
}
|
||||
@@ -1,15 +1,8 @@
|
||||
import { IQueueWorker } from "@/worker/worker.interface.js";
|
||||
import { IQueueWorker, JobHandlers } from "@/worker/worker.interface.js";
|
||||
import { type Job, type JobType, LOCK_MS } from "@/domain/jobs/job.type.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { ICheckProducer } from "@/worker/worker.check-producer.js";
|
||||
import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
|
||||
import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { IWorkerHelper } from "@/worker/worker.helper.js";
|
||||
import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
|
||||
import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js";
|
||||
import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
@@ -36,14 +29,8 @@ export interface DBQueueWorkerDependencies {
|
||||
isDbConnected: () => boolean;
|
||||
jobsRepository: IJobsRepository; // forwarded
|
||||
monitorsRepository: IMonitorsRepository; // forwarded
|
||||
checksRepository: IChecksRepository;
|
||||
checkService: ICheckService;
|
||||
handlers: JobHandlers;
|
||||
bufferService: IBufferService;
|
||||
checkProducer: ICheckProducer;
|
||||
checkEvaluator: ICheckEvaluator;
|
||||
geoCheckPipeline: ICheckPipeline;
|
||||
dispatcher: IReactorDispatcher;
|
||||
helper: IWorkerHelper;
|
||||
queueWorkersRepository: IQueueWorkersRepository; // forwarded
|
||||
queueMode: QueueMode; // forwarded
|
||||
queuePrimaryProcesses: boolean; // consumed immediately
|
||||
@@ -65,15 +52,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
egress: 0,
|
||||
};
|
||||
|
||||
private checksRepository: IChecksRepository;
|
||||
private checkProducer: ICheckProducer;
|
||||
private checkService: ICheckService;
|
||||
private checkEvaluator: ICheckEvaluator;
|
||||
private dispatcher: IReactorDispatcher;
|
||||
private geoCheckPipeline: ICheckPipeline;
|
||||
private helper: IWorkerHelper;
|
||||
private bufferService: IBufferService;
|
||||
private isDbConnected: () => boolean;
|
||||
private handlers: JobHandlers;
|
||||
|
||||
constructor(dependencies: DBQueueWorkerDependencies) {
|
||||
super(
|
||||
@@ -85,13 +66,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
dependencies.workerId
|
||||
);
|
||||
this.processesJobs = dependencies.queuePrimaryProcesses === true || dependencies.queueMode === "worker";
|
||||
this.checksRepository = dependencies.checksRepository;
|
||||
this.checkProducer = dependencies.checkProducer;
|
||||
this.checkService = dependencies.checkService;
|
||||
this.checkEvaluator = dependencies.checkEvaluator;
|
||||
this.dispatcher = dependencies.dispatcher;
|
||||
this.geoCheckPipeline = dependencies.geoCheckPipeline;
|
||||
this.helper = dependencies.helper;
|
||||
this.handlers = dependencies.handlers;
|
||||
this.bufferService = dependencies.bufferService;
|
||||
this.isDbConnected = dependencies.isDbConnected;
|
||||
}
|
||||
@@ -104,64 +79,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
|
||||
private getInFlightCount = () => Object.values(this.inFlight).reduce((sum, n) => sum + n, 0);
|
||||
|
||||
// ********************
|
||||
// Stage 1: This can eventually be offloaded to a separate service
|
||||
// ********************
|
||||
|
||||
private runCheck = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
await this.checkProducer.produce(monitor);
|
||||
};
|
||||
|
||||
// ********************
|
||||
// Stage 2: Evaluator loop stays here
|
||||
// ********************
|
||||
|
||||
private runEvaluate = async (job: Job) => {
|
||||
if (!job.refId || job.pendingChecks.length === 0) return;
|
||||
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
|
||||
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||
if (!monitor) {
|
||||
await this.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
|
||||
return;
|
||||
}
|
||||
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
|
||||
|
||||
let current = monitor;
|
||||
for (const check of checks) {
|
||||
const status = this.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
|
||||
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
if (!leaseHeld) {
|
||||
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
|
||||
this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" });
|
||||
return;
|
||||
}
|
||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||
}
|
||||
|
||||
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
|
||||
const found = new Set(checks.map((check) => check.id));
|
||||
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
|
||||
if (missing.length > 0) {
|
||||
this.logger.warn({
|
||||
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runEvaluate",
|
||||
});
|
||||
await this.jobsRepository.pullEvaluated(job.id, missing);
|
||||
}
|
||||
};
|
||||
|
||||
private runGeoCheck = async (job: Job) => {
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId!);
|
||||
if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
|
||||
};
|
||||
|
||||
// Extend the lock while a job runs so a slow-but-alive job is never reclaimed.
|
||||
private renewLock = async (job: Job): Promise<boolean> => {
|
||||
try {
|
||||
@@ -183,27 +100,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
if (!renewLock) clearInterval(renewTimer);
|
||||
}, LOCK_RENEW_MS);
|
||||
try {
|
||||
switch (job.type) {
|
||||
case "check":
|
||||
await this.runCheck(job);
|
||||
break;
|
||||
case "evaluate":
|
||||
await this.runEvaluate(job);
|
||||
break;
|
||||
case "geo-check":
|
||||
await this.runGeoCheck(job);
|
||||
break;
|
||||
case "cleanup-orphaned":
|
||||
await this.helper.getCleanupOrphanedJob()(); // Get job and execute
|
||||
break;
|
||||
case "cleanup-retention":
|
||||
await this.helper.getCleanupRetentionJob()(); // Get job and execute
|
||||
break;
|
||||
case "egress":
|
||||
await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
|
||||
break;
|
||||
}
|
||||
|
||||
await this.handlers[job.type](job);
|
||||
if (job.intervalMs === null) {
|
||||
// One-shot job
|
||||
await this.jobsRepository.recordOneShot(job.id, Date.now());
|
||||
|
||||
+20
-10
@@ -1,5 +1,5 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { JobHandler, MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
@@ -7,15 +7,19 @@ import { IBufferService } from "@/service/bufferService.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { supportsGeoCheck } from "@/domain/monitors/monitor.type.js";
|
||||
import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
|
||||
import { Job } from "@/domain/jobs/job.type.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
|
||||
const SERVICE_NAME = "CheckPipeline";
|
||||
const SERVICE_NAME = "GeoChecksPipeline";
|
||||
|
||||
export interface ICheckPipeline {
|
||||
export interface IGeoChecksPipeline {
|
||||
handle: JobHandler;
|
||||
run(monitor: Monitor): Promise<MonitorEvaluation | null>; // null = skipped
|
||||
}
|
||||
|
||||
export class GeoChecksPipeline implements ICheckPipeline {
|
||||
export class GeoChecksPipeline implements IGeoChecksPipeline {
|
||||
constructor(
|
||||
private monitorsRepository: IMonitorsRepository,
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
|
||||
private geoChecksService: IGeoChecksService,
|
||||
private bufferService: IBufferService,
|
||||
@@ -28,6 +32,12 @@ export class GeoChecksPipeline implements ICheckPipeline {
|
||||
return maintenanceWindows.some((window) => isWindowActive(window, now));
|
||||
}
|
||||
|
||||
handle: JobHandler = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId);
|
||||
if (monitor) await this.run(monitor);
|
||||
};
|
||||
|
||||
run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
@@ -36,7 +46,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
|
||||
// Step 1a: Guards - skip if unsupported or not enabled
|
||||
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getHeartbeatGeoJob" });
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "run" });
|
||||
}
|
||||
|
||||
if (!monitor.geoCheckEnabled) {
|
||||
@@ -46,7 +56,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
|
||||
this.logger.debug({
|
||||
message: `Monitor ${monitor.id} type does not support geo checks, skipping`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runGeoChecksPipeline",
|
||||
method: "run",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
@@ -55,7 +65,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
|
||||
this.logger.warn({
|
||||
message: `No geo check locations configured for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runGeoChecksPipeline",
|
||||
method: "run",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
@@ -66,7 +76,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
|
||||
this.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window, skipping geo check`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runGeoChecksPipeline",
|
||||
method: "run",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
@@ -81,7 +91,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
|
||||
this.logger.warn({
|
||||
message: `No geo check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runGeoChecksPipeline",
|
||||
method: "run",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
@@ -92,7 +102,7 @@ export class GeoChecksPipeline implements ICheckPipeline {
|
||||
this.logger.debug({
|
||||
message: `Geo check job executed for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runGeoChecksPipeline",
|
||||
method: "run",
|
||||
});
|
||||
return null;
|
||||
};
|
||||
@@ -20,20 +20,6 @@ export interface IWorkerHelper {
|
||||
getEgressRecoveryJob(): (job: Job) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface MonitorActionDecision {
|
||||
shouldCreateIncident: boolean;
|
||||
shouldResolveIncident: boolean;
|
||||
shouldSendNotification: boolean;
|
||||
incidentReason: "status_down" | "threshold_breach" | null;
|
||||
notificationReason: "status_change" | "threshold_breach" | null;
|
||||
thresholdBreaches?: {
|
||||
cpu?: boolean;
|
||||
memory?: boolean;
|
||||
disk?: boolean;
|
||||
temp?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export class WorkerHelper implements IWorkerHelper {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
|
||||
@@ -1,17 +1,32 @@
|
||||
import { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { JobType } from "@/domain/jobs/job.type.js";
|
||||
import { Job, JobType } from "@/domain/jobs/job.type.js";
|
||||
import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
import type { QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
|
||||
export type JobHandler = (job: Job) => Promise<void>;
|
||||
export type JobHandlers = Record<JobType, JobHandler>;
|
||||
export interface MonitorActionDecision {
|
||||
shouldCreateIncident: boolean;
|
||||
shouldResolveIncident: boolean;
|
||||
shouldSendNotification: boolean;
|
||||
incidentReason: "status_down" | "threshold_breach" | null;
|
||||
notificationReason: "status_change" | "threshold_breach" | null;
|
||||
thresholdBreaches?: {
|
||||
cpu?: boolean;
|
||||
memory?: boolean;
|
||||
disk?: boolean;
|
||||
temp?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export type MonitorEvaluation = {
|
||||
monitor: Monitor;
|
||||
status: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>; // raw result from networkService.requestStatus
|
||||
check: Check;
|
||||
statusChange: StatusChangeResult; // from statusService.updateMonitorStatus
|
||||
decision: MonitorActionDecision; // from MonitorStatusPolicy
|
||||
decision: MonitorActionDecision;
|
||||
};
|
||||
|
||||
export type WorkerJobFailure = {
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
import { StatusChangeResult } from "@/types/network.js";
|
||||
import { MonitorActionDecision } from "@/worker/worker.helper.js";
|
||||
|
||||
export interface IMonitorStatusPolicy {
|
||||
evaluate(statusChange: StatusChangeResult): MonitorActionDecision;
|
||||
}
|
||||
|
||||
export class MonitorStatusPolicy implements IMonitorStatusPolicy {
|
||||
evaluate = (statusChangeResult: StatusChangeResult): MonitorActionDecision => {
|
||||
const { monitor, statusChanged, prevStatus } = statusChangeResult;
|
||||
|
||||
// Initialize result
|
||||
const decision: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
|
||||
if (!statusChanged) {
|
||||
return decision;
|
||||
}
|
||||
|
||||
if (monitor.status === "down") {
|
||||
// Monitor went down (unreachable)
|
||||
decision.shouldCreateIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "status_down";
|
||||
decision.notificationReason = "status_change";
|
||||
} else if (monitor.status === "breached") {
|
||||
// Hardware monitor exceeded thresholds
|
||||
decision.shouldCreateIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "threshold_breach";
|
||||
decision.notificationReason = "threshold_breach";
|
||||
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
|
||||
// Monitor recovered from down or breached state
|
||||
decision.shouldResolveIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.notificationReason = "status_change";
|
||||
}
|
||||
|
||||
return decision;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
|
||||
import { ICheckService } from "@/domain/checks/check.service.js";
|
||||
import { Check } from "@/domain/checks/check.type.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
import { IEgressService, isHttpStatusCode } from "@/domain/egress/egress.service.js";
|
||||
import type { EgressStatus } from "@/domain/egress/egress.type.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { isEgressAttributable, Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { IBufferService } from "@/service/bufferService.js";
|
||||
import { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import { INetworkService } from "@/service/networkService.js";
|
||||
import { IStatusService } from "@/service/statusService.js";
|
||||
import { DockerStatusPayload, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { JobHandler, MonitorActionDecision, MonitorEvaluation } from "@/worker/worker.interface.js";
|
||||
|
||||
const SERVICE_NAME = "WorkerPipeline";
|
||||
|
||||
// The three entry points: the queue calls the two handlers, the buffer calls ingestChecks.
|
||||
// produce and evaluateCheck stay public on the class so tests can drive one stage without the queue or the buffer.
|
||||
export interface IWorkerPipeline {
|
||||
handleCheck: JobHandler;
|
||||
ingestChecks(checks: Check[]): Promise<void>;
|
||||
handleEvaluate: JobHandler;
|
||||
}
|
||||
|
||||
export interface WorkerPipelineDependencies {
|
||||
logger: ILogger;
|
||||
monitorsRepository: IMonitorsRepository;
|
||||
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
checksRepository: IChecksRepository;
|
||||
jobsRepository: IJobsRepository;
|
||||
checkService: ICheckService;
|
||||
networkService: INetworkService;
|
||||
proxyResolver: IProxyResolver;
|
||||
bufferService: IBufferService;
|
||||
dockerLogsService: IDockerLogsService;
|
||||
egressService: IEgressService;
|
||||
statusService: IStatusService;
|
||||
dispatcher: IReactorDispatcher;
|
||||
}
|
||||
|
||||
export class WorkerPipeline implements IWorkerPipeline {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private unarmed: Map<string, PendingCheck[]> = new Map();
|
||||
|
||||
constructor(private deps: WorkerPipelineDependencies) {}
|
||||
|
||||
private isInMaintenanceWindow = async (monitorId: string, teamId: string) => {
|
||||
const windows = await this.deps.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
|
||||
const now = new Date();
|
||||
return windows.some((window) => isWindowActive(window, now));
|
||||
};
|
||||
|
||||
// ****************************************************************
|
||||
// Stage 1: Check job. Probe the host and buffer the result.
|
||||
// ****************************************************************
|
||||
|
||||
handleCheck: JobHandler = async (job) => {
|
||||
if (!job.refId) return;
|
||||
const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId);
|
||||
if (!monitor) return;
|
||||
await this.produce(monitor);
|
||||
};
|
||||
|
||||
produce = async (monitor: Monitor) => {
|
||||
if (!monitor.id) {
|
||||
throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
|
||||
}
|
||||
// ****************************
|
||||
// Step 1: Acquire
|
||||
// ****************************
|
||||
|
||||
// Step 1a: Maintenance window gate - skip if in maintenance
|
||||
const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
|
||||
if (maintenanceWindowActive) {
|
||||
this.deps.logger.debug({
|
||||
message: `Monitor ${monitor.id} is in maintenance window`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
});
|
||||
if (monitor.status !== "maintenance") {
|
||||
// Clear the status window to avoid incidents being created on next check
|
||||
await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] });
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Leaving the window is explicit rather than a side effect of evaluating the resulting check, which the
|
||||
// degraded-egress short-circuit skips. "initializing" is resolved to up or down by the first check that is
|
||||
// evaluated, so the monitor cannot sit in "maintenance" for the length of an egress outage.
|
||||
if (monitor.status === "maintenance") {
|
||||
monitor = await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "initializing" });
|
||||
}
|
||||
|
||||
// Step 1b: Acquire status
|
||||
const proxyUrl = await this.deps.proxyResolver.resolve(monitor);
|
||||
const dockerTlsKey = await this.resolveDockerTlsKey(monitor);
|
||||
|
||||
const status = await this.deps.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey });
|
||||
if (!status) {
|
||||
throw new Error("No network response");
|
||||
}
|
||||
|
||||
// ****************************
|
||||
// Step 2: Record
|
||||
// ****************************
|
||||
|
||||
// Step 2a: Create & record a check, return null if fail.
|
||||
// Built before the egress assessment below so that the probe's duration stays out of `createdAt`.
|
||||
const check = this.deps.checkService.toCheck(status);
|
||||
if (!check) {
|
||||
this.deps.logger.warn({
|
||||
message: `No check could be built for monitor ${monitor.id}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "produce",
|
||||
details: { code: status.code, message: status.message },
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
// Step 2b: On a transport failure, ask whether the instance itself can reach anything before blaming the
|
||||
// target. Null means the egress check is disabled (or failed internally) and the check is treated as usual.
|
||||
const egressStatus = this.isTransportFailure(monitor, status) ? await this.assessEgress(monitor.id) : null;
|
||||
if (egressStatus !== null) {
|
||||
check.egressStatus = egressStatus;
|
||||
}
|
||||
|
||||
// Step 2c: Add to buffer
|
||||
this.deps.bufferService.addToBuffer(check);
|
||||
|
||||
// Step 2d: As with the maintenance gate above, the window is cleared at the point the monitor stops
|
||||
// being evaluated, so the results either side of a spell never end up adjacent and cross the threshold
|
||||
// on stale data. The length guard keeps this to one write per spell rather than one per degraded check,
|
||||
// and it runs after the check is buffered so that a failed write cannot cost us the check.
|
||||
if (egressStatus === "degraded" && monitor.statusWindow?.length) {
|
||||
await this.deps.monitorsRepository.updateById(monitor.id, monitor.teamId, { statusWindow: [] });
|
||||
}
|
||||
|
||||
// Step 2e: Handle docker logs
|
||||
if (status.type === "docker") {
|
||||
const dockerLogs = await this.deps.dockerLogsService.buildDockerLogs(status as MonitorStatusResponse<DockerStatusPayload>);
|
||||
for (const dockerLog of dockerLogs) {
|
||||
this.deps.bufferService.addDockerLogToBuffer(dockerLog);
|
||||
}
|
||||
}
|
||||
return { status, check };
|
||||
};
|
||||
|
||||
private async resolveDockerTlsKey(monitor: Monitor): Promise<string | undefined> {
|
||||
if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined;
|
||||
const dockerTlsKey = await this.deps.monitorsRepository.findDockerTlsKeyById(monitor.id);
|
||||
return dockerTlsKey ?? undefined;
|
||||
}
|
||||
|
||||
// Only a failure to reach the target at all can be the instance's own fault. A provider that can tell the
|
||||
// two apart says so outright; for the rest, any HTTP status code proves the target answered (a 4xx, a 5xx,
|
||||
// or a 200 with a content mismatch).
|
||||
private peerAnswered(status: MonitorStatusResponse): boolean {
|
||||
return status.peerResponded ?? isHttpStatusCode(status.code);
|
||||
}
|
||||
|
||||
private isTransportFailure(monitor: Monitor, status: MonitorStatusResponse): boolean {
|
||||
return status.status === false && isEgressAttributable(monitor) && !this.peerAnswered(status);
|
||||
}
|
||||
|
||||
// The egress check must never stop a check being recorded, so a rejection is logged and treated as unknown.
|
||||
private async assessEgress(monitorId: string): Promise<EgressStatus | null> {
|
||||
try {
|
||||
return await this.deps.egressService.assessAfterFailure();
|
||||
} catch (error: unknown) {
|
||||
this.deps.logger.warn({
|
||||
message: `Egress assessment failed for monitor ${monitorId}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "assessEgress",
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ****************************************************************
|
||||
// Stage 2: Ingestion
|
||||
// Called by the buffer, checks are inserted and evaluation jobs created
|
||||
// ****************************************************************
|
||||
|
||||
ingestChecks = async (checks: Check[]) => {
|
||||
if (checks.length > 0) await this.deps.checkService.createChecks(checks);
|
||||
|
||||
// Group by monitor, starting from any entries a previous flush failed to arm
|
||||
const byMonitor = this.unarmed;
|
||||
this.unarmed = new Map();
|
||||
for (const check of checks) {
|
||||
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
|
||||
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
|
||||
byMonitor.set(check.metadata.monitorId, pending);
|
||||
}
|
||||
if (byMonitor.size === 0) return;
|
||||
|
||||
const now = Date.now();
|
||||
const ops: Promise<boolean>[] = [];
|
||||
for (const [monitorId, pendingChecks] of byMonitor) {
|
||||
ops.push(
|
||||
this.deps.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
|
||||
// The checks are already stored, so a failed arm is retried on the next ingest rather than dropped
|
||||
this.unarmed.set(monitorId, pendingChecks);
|
||||
this.deps.logger.error({
|
||||
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next ingest: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "ingestChecks",
|
||||
});
|
||||
return false;
|
||||
})
|
||||
);
|
||||
}
|
||||
await Promise.all(ops);
|
||||
};
|
||||
|
||||
// ****************************************************************
|
||||
// Stage 3: Evaluation
|
||||
// Apply each pending check to the monitor -> decide -> react.
|
||||
// ****************************************************************
|
||||
|
||||
handleEvaluate: JobHandler = async (job) => {
|
||||
if (!job.refId || job.pendingChecks.length === 0) return;
|
||||
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
|
||||
|
||||
const monitor = await this.deps.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||
if (!monitor) {
|
||||
await this.deps.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
|
||||
return;
|
||||
}
|
||||
const checks = await this.deps.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
|
||||
|
||||
let current = monitor;
|
||||
for (const check of checks) {
|
||||
const status = this.deps.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.evaluateCheck(status, check, current);
|
||||
await this.deps.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
const leaseHeld = await this.deps.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
if (!leaseHeld) {
|
||||
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
|
||||
this.deps.logger.warn({
|
||||
message: `Lost lease on ${job.id} after check ${check.id}, stopping`,
|
||||
service: SERVICE_NAME,
|
||||
method: "handleEvaluate",
|
||||
});
|
||||
return;
|
||||
}
|
||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||
}
|
||||
|
||||
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
|
||||
const found = new Set(checks.map((check) => check.id));
|
||||
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
|
||||
if (missing.length > 0) {
|
||||
this.deps.logger.warn({
|
||||
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "handleEvaluate",
|
||||
});
|
||||
await this.deps.jobsRepository.pullEvaluated(job.id, missing);
|
||||
}
|
||||
};
|
||||
|
||||
// A check taken while the instance had no outbound connectivity says nothing about the target.
|
||||
// It is left out of the status window, running stats and monitor status entirely, so that no incident
|
||||
// opens for it and no spurious "resolved"/"up" fires once egress returns.
|
||||
private skipDegradedEgressCheck = (status: MonitorStatusResponse, check: Check, monitor: Monitor): MonitorEvaluation => {
|
||||
this.deps.logger.debug({
|
||||
message: `Skipping evaluation of check ${check.id} for monitor ${monitor.id}: instance egress was degraded`,
|
||||
service: SERVICE_NAME,
|
||||
method: "evaluateCheck",
|
||||
});
|
||||
const decision: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
return {
|
||||
monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: status.code, timestamp: Date.now() },
|
||||
decision,
|
||||
};
|
||||
};
|
||||
|
||||
evaluateCheck = async (status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation> => {
|
||||
if (check.egressStatus === "degraded") {
|
||||
return this.skipDegradedEgressCheck(status, check, monitor);
|
||||
}
|
||||
|
||||
const statusChange = await this.deps.statusService.updateMonitorStatus(status, check, monitor);
|
||||
|
||||
const decision = this.decide(statusChange);
|
||||
return { monitor: statusChange.monitor, status, check, statusChange, decision };
|
||||
};
|
||||
|
||||
private decide = (statusChange: StatusChangeResult): MonitorActionDecision => {
|
||||
const { monitor, statusChanged, prevStatus } = statusChange;
|
||||
const decision: MonitorActionDecision = {
|
||||
shouldCreateIncident: false,
|
||||
shouldResolveIncident: false,
|
||||
shouldSendNotification: false,
|
||||
incidentReason: null,
|
||||
notificationReason: null,
|
||||
};
|
||||
if (!statusChanged) return decision;
|
||||
|
||||
if (monitor.status === "down") {
|
||||
decision.shouldCreateIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "status_down";
|
||||
decision.notificationReason = "status_change";
|
||||
} else if (monitor.status === "breached") {
|
||||
decision.shouldCreateIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.incidentReason = "threshold_breach";
|
||||
decision.notificationReason = "threshold_breach";
|
||||
} else if (monitor.status === "up" && (prevStatus === "down" || prevStatus === "breached")) {
|
||||
decision.shouldResolveIncident = true;
|
||||
decision.shouldSendNotification = true;
|
||||
decision.notificationReason = "status_change";
|
||||
}
|
||||
return decision;
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,5 @@
|
||||
import { jest } from "@jest/globals";
|
||||
import { MonitorStatusPolicy } from "../../src/worker/worker.monitor-status-policy.ts";
|
||||
import { CheckProducer } from "../../src/worker/worker.check-producer.ts";
|
||||
import { CheckEvaluator } from "../../src/worker/worker.check-evaluator.ts";
|
||||
import { WorkerPipeline } from "../../src/worker/worker.pipeline.ts";
|
||||
import { NotificationReactor } from "../../src/worker/reactors/reactor.notification.ts";
|
||||
import { IncidentReactor } from "../../src/worker/reactors/reactor.incident.ts";
|
||||
import { ReactorDispatcher } from "../../src/worker/reactors/reactor.dispatcher.ts";
|
||||
@@ -14,6 +12,7 @@ import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||
import type { Check } from "../../src/domain/checks/check.type.ts";
|
||||
import type { MaintenanceWindow } from "../../src/domain/maintenance-windows/maintenance-window.type.ts";
|
||||
import type { EgressStatus } from "../../src/domain/egress/egress.type.ts";
|
||||
|
||||
let checkCounter = 0;
|
||||
|
||||
@@ -79,9 +78,11 @@ export interface HeartbeatTestHarness {
|
||||
bufferStub: { addToBuffer: jest.Mock; addGeoCheckToBuffer: jest.Mock; scheduleNextFlush: jest.Mock };
|
||||
maintenanceWindowsRepo: { findByMonitorId: jest.Mock };
|
||||
messageBuilder: { extractThresholdBreaches: jest.Mock };
|
||||
egressService: { assessAfterFailure: jest.Mock };
|
||||
heartbeatJob: (monitor: Monitor) => Promise<void>;
|
||||
setNextResponse: (status: boolean, code: number) => void;
|
||||
setNextResponseFull: (response: MonitorStatusResponse) => void;
|
||||
setEgressStatus: (status: EgressStatus | null) => void;
|
||||
}
|
||||
|
||||
export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
@@ -129,31 +130,42 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
const maintenanceWindowsRepo = { findByMonitorId: jest.fn().mockResolvedValue([]) };
|
||||
const proxyResolver = { resolve: jest.fn().mockResolvedValue(undefined) };
|
||||
const dockerLogsService = { buildDockerLogs: jest.fn().mockResolvedValue([]) };
|
||||
|
||||
const checkProducer = new CheckProducer(
|
||||
monitorsRepo as any,
|
||||
maintenanceWindowsRepo as any,
|
||||
checkService as any,
|
||||
networkService as any,
|
||||
proxyResolver as any,
|
||||
bufferStub as any,
|
||||
dockerLogsService as any,
|
||||
logger
|
||||
);
|
||||
const checkEvaluator = new CheckEvaluator(statusService as any, new MonitorStatusPolicy());
|
||||
// Null mirrors the egress check being disabled, so the existing heartbeat suites run unchanged.
|
||||
let nextEgressStatus: EgressStatus | null = null;
|
||||
const egressService = { assessAfterFailure: jest.fn().mockImplementation(() => Promise.resolve(nextEgressStatus)) };
|
||||
const setEgressStatus = (status: EgressStatus | null) => {
|
||||
nextEgressStatus = status;
|
||||
};
|
||||
|
||||
const notificationReactor = new NotificationReactor(notificationsService as any);
|
||||
const incidentReactor = new IncidentReactor(incidentService as any);
|
||||
const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
|
||||
|
||||
// The harness drives stages 1 and 3 directly, so the stage 2 repositories are never reached.
|
||||
const pipeline = new WorkerPipeline({
|
||||
logger,
|
||||
monitorsRepository: monitorsRepo as any,
|
||||
maintenanceWindowsRepository: maintenanceWindowsRepo as any,
|
||||
checksRepository: { findUnevaluatedByMonitorId: jest.fn() } as any,
|
||||
jobsRepository: { upsertEvaluate: jest.fn(), pullEvaluated: jest.fn() } as any,
|
||||
checkService: checkService as any,
|
||||
networkService: networkService as any,
|
||||
proxyResolver: proxyResolver as any,
|
||||
bufferService: bufferStub as any,
|
||||
dockerLogsService: dockerLogsService as any,
|
||||
egressService: egressService as any,
|
||||
statusService,
|
||||
dispatcher: reactorDispatcher,
|
||||
});
|
||||
|
||||
// Mirror the production check→evaluate flow: read the monitor fresh each cycle (so the
|
||||
// accumulated statusWindow is visible), produce a check, then evaluate against that monitor.
|
||||
const heartbeatJob = async (seedMonitor: Monitor) => {
|
||||
const monitor = await monitorsRepo.findByIdLean(seedMonitor.id);
|
||||
if (!monitor) return;
|
||||
const result = await checkProducer.produce(monitor);
|
||||
const result = await pipeline.produce(monitor);
|
||||
if (!result) return; // skipped (e.g. maintenance window)
|
||||
const evaluation = await checkEvaluator.evaluate(result.status, result.check, monitor);
|
||||
const evaluation = await pipeline.evaluateCheck(result.status, result.check, monitor);
|
||||
await reactorDispatcher.dispatch(evaluation);
|
||||
};
|
||||
|
||||
@@ -167,8 +179,10 @@ export function createHeartbeatTestHarness(): HeartbeatTestHarness {
|
||||
bufferStub,
|
||||
maintenanceWindowsRepo,
|
||||
messageBuilder,
|
||||
egressService,
|
||||
heartbeatJob,
|
||||
setNextResponse,
|
||||
setNextResponseFull,
|
||||
setEgressStatus,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
|
||||
import { CheckModel } from "../../src/domain/checks/check.model.ts";
|
||||
import type {
|
||||
Check,
|
||||
DockerChecksResult,
|
||||
HardwareChecksResult,
|
||||
PageSpeedChecksResult,
|
||||
UptimeChecksResult,
|
||||
} from "../../src/domain/checks/check.type.ts";
|
||||
import { NETWORK_ERROR } from "../../src/types/network.ts";
|
||||
import type { ILogger } from "../../src/utils/logger.ts";
|
||||
import { createMockLogger } from "../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The guarantee under test — a check flagged egressStatus "degraded" is left out of
|
||||
// every aggregate, so a degraded episode reads as a gap in monitoring, while the check
|
||||
// still appears in listings — lives in $match stages spread into several aggregation
|
||||
// pipelines, so it can only be exercised against a live mongod.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await CheckModel.createCollection(); // timeseries collections must exist before insert
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await CheckModel.deleteMany({});
|
||||
});
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const MONITOR_ID = new mongoose.Types.ObjectId();
|
||||
const TEAM_ID = new mongoose.Types.ObjectId();
|
||||
|
||||
// One shared createdAt keeps every seeded check in the same bucket of the "day" range.
|
||||
const BUCKET_TIME = new Date(Date.now() - 10 * 60 * 1000);
|
||||
|
||||
const seedCheck = (overrides: Record<string, unknown> = {}) =>
|
||||
CheckModel.create({
|
||||
metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "http" },
|
||||
status: true,
|
||||
responseTime: 100,
|
||||
createdAt: BUCKET_TIME,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const seedDegradedFailure = (overrides: Record<string, unknown> = {}) =>
|
||||
seedCheck({ status: false, responseTime: 300, egressStatus: "degraded", ...overrides });
|
||||
|
||||
const utcDate = (date: Date) => date.toISOString().slice(0, 10);
|
||||
|
||||
const makeCheck = (overrides: Partial<Check> = {}): Check =>
|
||||
({
|
||||
metadata: { monitorId: MONITOR_ID.toString(), teamId: TEAM_ID.toString(), type: "http" },
|
||||
status: false,
|
||||
responseTime: 300,
|
||||
statusCode: 500,
|
||||
message: "Error",
|
||||
createdAt: BUCKET_TIME.toISOString(),
|
||||
updatedAt: BUCKET_TIME.toISOString(),
|
||||
...overrides,
|
||||
}) as Check;
|
||||
|
||||
describe("MongoChecksRepository degraded-egress exclusion", () => {
|
||||
let repo: MongoChecksRepository;
|
||||
|
||||
beforeEach(() => {
|
||||
repo = new MongoChecksRepository(createMockLogger() as unknown as ILogger);
|
||||
});
|
||||
|
||||
it("round-trips egressStatus through createChecks and the unevaluated-checks read, leaving it absent when unset", async () => {
|
||||
const created = await repo.createChecks([makeCheck({ egressStatus: "degraded" }), makeCheck({ egressStatus: "ok" }), makeCheck()]);
|
||||
const pending = created.map((check) => ({ checkId: check.id as string, createdAt: new Date(check.createdAt as string).getTime() }));
|
||||
|
||||
const checks = await repo.findUnevaluatedByMonitorId(MONITOR_ID.toString(), pending);
|
||||
|
||||
// The evaluator runs from this read, so the flag must survive persistence for the short-circuit to fire.
|
||||
expect(checks.map((check) => check.egressStatus)).toEqual(["degraded", "ok", undefined]);
|
||||
expect(checks[2].egressStatus).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects a value outside the EgressStatuses tuple", async () => {
|
||||
await expect(seedCheck({ status: false, egressStatus: "unknown" })).rejects.toThrow(/egressStatus/);
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the uptime percentage and from the response-time series alike", async () => {
|
||||
await seedCheck(); // up
|
||||
await seedCheck({ status: false }); // real failure
|
||||
await seedDegradedFailure();
|
||||
await seedDegradedFailure();
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
|
||||
|
||||
// 1 up of 2 attributable checks. Counting the degraded pair would give 25%.
|
||||
expect(result.uptimePercentage).toBe(0.5);
|
||||
// Every series sees the same two checks, so the episode is a gap rather than a dip.
|
||||
expect(result.avgResponseTime).toBe(100);
|
||||
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 2, avgResponseTime: 100 });
|
||||
expect(result.groupedDownChecks[0]).toMatchObject({ totalChecks: 1 });
|
||||
expect(result.groupedUpChecks[0]).toMatchObject({ totalChecks: 1 });
|
||||
});
|
||||
|
||||
it("leaves a bucket out of the response-time series entirely when every check in it was degraded", async () => {
|
||||
await seedDegradedFailure();
|
||||
await seedDegradedFailure();
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
|
||||
|
||||
expect(result.groupedChecks).toEqual([]);
|
||||
expect(result.groupedDownChecks).toEqual([]);
|
||||
expect(result.uptimePercentage).toBe(0);
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the pagespeed series", async () => {
|
||||
await seedCheck({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.9 });
|
||||
await seedDegradedFailure({ metadata: { monitorId: MONITOR_ID, teamId: TEAM_ID, type: "pagespeed" }, performance: 0.1 });
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "pagespeed" })) as PageSpeedChecksResult;
|
||||
|
||||
expect(result.groupedChecks[0]).toMatchObject({ totalChecks: 1, performance: 0.9 });
|
||||
});
|
||||
|
||||
it("does not flag a failure that carries egressStatus ok", async () => {
|
||||
await seedCheck();
|
||||
await seedCheck({ status: false, egressStatus: "ok" });
|
||||
|
||||
const result = (await repo.findByDateRangeAndMonitorId(MONITOR_ID.toString(), "day", { type: "http" })) as UptimeChecksResult;
|
||||
|
||||
expect(result.uptimePercentage).toBe(0.5);
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the team summary totals", async () => {
|
||||
await seedCheck();
|
||||
await seedCheck({ status: false });
|
||||
await seedDegradedFailure();
|
||||
|
||||
const summary = await repo.findSummaryByTeamId(TEAM_ID.toString(), "day");
|
||||
|
||||
expect(summary).toEqual({ totalChecks: 2, downChecks: 1 });
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the daily status buckets, dropping a day that saw nothing else", async () => {
|
||||
const otherMonitor = new mongoose.Types.ObjectId();
|
||||
await seedCheck();
|
||||
await seedCheck({ status: false });
|
||||
await seedDegradedFailure();
|
||||
await seedDegradedFailure({ metadata: { monitorId: otherMonitor, teamId: TEAM_ID, type: "http" } });
|
||||
|
||||
const buckets = await repo.getDailyStatusBuckets([MONITOR_ID.toString(), otherMonitor.toString()], 7, "UTC");
|
||||
|
||||
// The other monitor saw only degraded checks that day, so it gets no row rather than a 0/0 bucket.
|
||||
expect(buckets).toEqual([
|
||||
{ monitorId: MONITOR_ID.toString(), date: utcDate(BUCKET_TIME), totalChecks: 2, upChecks: 1, downChecks: 1, avgResponseTime: 100 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps degraded checks in the paginated listing so they can be shown as such", async () => {
|
||||
await seedCheck({ status: false });
|
||||
await seedDegradedFailure();
|
||||
|
||||
const { checksCount, checks } = await repo.findByMonitorId(MONITOR_ID.toString(), "desc", "day", 0, 10, undefined, "down");
|
||||
|
||||
expect(checksCount).toBe(2);
|
||||
expect(checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
|
||||
});
|
||||
|
||||
it("keeps degraded checks in the team listing and under the resolve filter", async () => {
|
||||
await seedCheck({ status: false, statusCode: NETWORK_ERROR });
|
||||
await seedDegradedFailure({ statusCode: NETWORK_ERROR });
|
||||
|
||||
// The Checks page reads this listing when no single monitor is selected, and a degraded check has the
|
||||
// same shape the resolve filter looks for, so neither may quietly drop it.
|
||||
const team = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "down");
|
||||
const resolve = await repo.findByTeamId("desc", "day", 0, 10, TEAM_ID.toString(), "resolve");
|
||||
|
||||
expect(team.checksCount).toBe(2);
|
||||
expect(resolve.checksCount).toBe(2);
|
||||
expect(resolve.checks.map((check) => check.egressStatus).sort()).toEqual([undefined, "degraded"].sort());
|
||||
});
|
||||
|
||||
it("excludes degraded checks from the docker and hardware aggregates", async () => {
|
||||
const dockerMonitor = new mongoose.Types.ObjectId();
|
||||
const hardwareMonitor = new mongoose.Types.ObjectId();
|
||||
const dockerMeta = { monitorId: dockerMonitor, teamId: TEAM_ID, type: "docker" };
|
||||
const hardwareMeta = { monitorId: hardwareMonitor, teamId: TEAM_ID, type: "hardware" };
|
||||
|
||||
// A Capture agent is reached by an outbound HTTP request and a docker daemon can be reached over TCP,
|
||||
// so both are egress attributable and their checks can carry the flag.
|
||||
await seedCheck({ metadata: dockerMeta });
|
||||
await seedDegradedFailure({ metadata: dockerMeta });
|
||||
await seedCheck({ metadata: hardwareMeta });
|
||||
await seedDegradedFailure({ metadata: hardwareMeta });
|
||||
|
||||
const docker = (await repo.findByDateRangeAndMonitorId(dockerMonitor.toString(), "day", { type: "docker" })) as DockerChecksResult;
|
||||
const hardware = (await repo.findByDateRangeAndMonitorId(hardwareMonitor.toString(), "day", { type: "hardware" })) as HardwareChecksResult;
|
||||
|
||||
expect(docker.aggregateData.totalChecks).toBe(1);
|
||||
expect(docker.aggregate[0]).toMatchObject({ upCount: 1, totalCount: 1 });
|
||||
expect(hardware.aggregateData.totalChecks).toBe(1);
|
||||
expect(hardware.upChecks.totalChecks).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import JobModel from "../../src/domain/jobs/job.model.ts";
|
||||
import { CheckService } from "../../src/domain/checks/check.service.ts";
|
||||
import { BufferService } from "../../src/service/bufferService.ts";
|
||||
import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts";
|
||||
import { WorkerPipeline } from "../../src/worker/worker.pipeline.ts";
|
||||
import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts";
|
||||
import { createMockLogger } from "../helpers/createMockLogger.ts";
|
||||
import type { Job } from "../../src/domain/jobs/job.type.ts";
|
||||
@@ -29,8 +30,9 @@ import type { ILogger } from "../../src/utils/logger.ts";
|
||||
//
|
||||
// The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each
|
||||
// is applied, so a late batch is always evaluated. This drives the real time-series query, the
|
||||
// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an
|
||||
// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's.
|
||||
// real jobs collection, the real buffer flush, the real pipeline ingest and evaluate stages and the
|
||||
// real DBQueueWorker job path against an in-process mongod, so the handoff semantics under test
|
||||
// are MongoDB's, not a mock's.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
@@ -105,16 +107,39 @@ describe("Evaluate cursor with more than one processing worker", () => {
|
||||
let evaluator: DBQueueWorker;
|
||||
let evaluatorJobsRepository: MongoJobsRepository;
|
||||
|
||||
// One BufferService per worker process, each flushing into the same collection and jobs table.
|
||||
const createWorkerBuffer = (workerId: string) =>
|
||||
new BufferService(
|
||||
// One pipeline and BufferService per worker process, each flushing into the same collection and
|
||||
// jobs table. Only stages 2 and 3 are exercised: the status service is stubbed to record which
|
||||
// checks reached evaluation, and stage 1's network dependencies are never called.
|
||||
const createWorkerPipeline = (workerId: string) => {
|
||||
const jobsRepository = new MongoJobsRepository(workerId);
|
||||
const bufferService = new BufferService(
|
||||
logger,
|
||||
checkService,
|
||||
stubGeoChecksService as any,
|
||||
stubDockerLogsService as any,
|
||||
stubSettingsService as any,
|
||||
new MongoJobsRepository(workerId)
|
||||
(checks): Promise<void> => pipeline.ingestChecks(checks)
|
||||
);
|
||||
const pipeline = new WorkerPipeline({
|
||||
logger,
|
||||
monitorsRepository: monitorsRepository as any,
|
||||
maintenanceWindowsRepository: { findByMonitorId: jest.fn<any>().mockResolvedValue([]) } as any,
|
||||
checksRepository,
|
||||
jobsRepository,
|
||||
checkService,
|
||||
networkService: { requestStatus: jest.fn<any>() } as any,
|
||||
proxyResolver: { resolve: jest.fn<any>() } as any,
|
||||
bufferService,
|
||||
dockerLogsService: { buildDockerLogs: jest.fn<any>() } as any,
|
||||
statusService: {
|
||||
updateMonitorStatus: async (_status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
|
||||
evaluatedCheckIds.push(check.id);
|
||||
return { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 };
|
||||
},
|
||||
} as any,
|
||||
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) } as any,
|
||||
});
|
||||
return { pipeline, bufferService, jobsRepository };
|
||||
};
|
||||
|
||||
// Claim every due evaluate job and run it through the worker's real per-job path.
|
||||
const runDueEvaluateJobs = async () => {
|
||||
@@ -130,35 +155,28 @@ describe("Evaluate cursor with more than one processing worker", () => {
|
||||
monitorsRepository.seed(makeMonitor());
|
||||
checksRepository = new MongoChecksRepository(logger);
|
||||
checkService = new CheckService(monitorsRepository, logger, checksRepository);
|
||||
bufferA = createWorkerBuffer("worker-a");
|
||||
bufferB = createWorkerBuffer("worker-b");
|
||||
|
||||
evaluatedCheckIds = [];
|
||||
evaluatorJobsRepository = new MongoJobsRepository("worker-b");
|
||||
const workerA = createWorkerPipeline("worker-a");
|
||||
const workerB = createWorkerPipeline("worker-b");
|
||||
bufferA = workerA.bufferService;
|
||||
bufferB = workerB.bufferService;
|
||||
|
||||
const unused = jest.fn<any>().mockResolvedValue(undefined);
|
||||
evaluatorJobsRepository = workerB.jobsRepository;
|
||||
evaluator = new DBQueueWorker({
|
||||
logger,
|
||||
isDbConnected: () => true,
|
||||
jobsRepository: evaluatorJobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
bufferService: bufferB,
|
||||
checkProducer: { produce: jest.fn<any>() },
|
||||
checkEvaluator: {
|
||||
evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
|
||||
evaluatedCheckIds.push(check.id);
|
||||
return {
|
||||
monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 },
|
||||
decision: {},
|
||||
};
|
||||
},
|
||||
} as any,
|
||||
geoCheckPipeline: { run: jest.fn<any>() },
|
||||
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
|
||||
helper: {} as any,
|
||||
handlers: {
|
||||
check: unused,
|
||||
evaluate: workerB.pipeline.handleEvaluate,
|
||||
"geo-check": unused,
|
||||
"cleanup-orphaned": unused,
|
||||
"cleanup-retention": unused,
|
||||
egress: unused,
|
||||
},
|
||||
queueWorkersRepository: {} as any,
|
||||
queueMode: "worker",
|
||||
queuePrimaryProcesses: true,
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import { describe, expect, it, beforeEach } from "@jest/globals";
|
||||
import { createHeartbeatTestHarness, makeMonitor, type HeartbeatTestHarness } from "../helpers/heartbeatTestHarness.ts";
|
||||
import { NETWORK_ERROR } from "../../src/types/network.ts";
|
||||
|
||||
// Egress is only consulted for transport failures, so these heartbeats fail with the network-error
|
||||
// sentinel (timeout / connection refused) rather than an HTTP status code.
|
||||
|
||||
describe("Heartbeat job: degraded egress", () => {
|
||||
let h: HeartbeatTestHarness;
|
||||
|
||||
beforeEach(() => {
|
||||
h = createHeartbeatTestHarness();
|
||||
});
|
||||
|
||||
it("records failing checks as degraded without changing monitor status, opening an incident or notifying", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// Enough failures to cross the 60% threshold if they were counted
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Every check was buffered, flagged, so the repository can exclude it from uptime
|
||||
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
|
||||
expect(h.bufferStub.addToBuffer).toHaveBeenCalledTimes(3);
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).toMatchObject({ status: false, egressStatus: "degraded" });
|
||||
}
|
||||
|
||||
// Nothing downstream moved
|
||||
const storedMonitor = await h.monitorsRepo.findById("mon-1", "team-1");
|
||||
expect(storedMonitor.status).toBe("up");
|
||||
// The window is cleared once as the spell starts, as entering a maintenance window does.
|
||||
expect(storedMonitor.statusWindow).toEqual([]);
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not fire a spurious recovery once egress returns", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
// Egress is back and the target is fine
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(true, 200);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
expect(h.notificationsService.handleNotifications).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detects down at the normal threshold when the egress check is disabled", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).toHaveBeenCalledTimes(3);
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detects down at the normal threshold when egress is confirmed ok", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("ok");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).toMatchObject({ status: false, egressStatus: "ok" });
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluates HTTP error responses normally while egress is degraded", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// The target answered, so the failure is its own even though the instance is degraded
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, 503);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.incidentsRepo.getAll()[0].statusCode).toBe(503);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluates a failing docker check normally while egress is degraded", async () => {
|
||||
const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" });
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect(h.egressService.assessAfterFailure).not.toHaveBeenCalled();
|
||||
for (const [check] of h.bufferStub.addToBuffer.mock.calls) {
|
||||
expect(check).not.toHaveProperty("egressStatus");
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("leaves an open incident open when egress drops during it", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// The target goes down first and is detected normally
|
||||
h.setEgressStatus(null);
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
|
||||
// Egress then drops. The monitor must stay down rather than be resolved by the silence.
|
||||
h.setEgressStatus("degraded");
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
expect(h.notificationsService.handleNotifications).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("starts a fresh status window after a degraded spell", async () => {
|
||||
const monitor = makeMonitor();
|
||||
h.monitorsRepo.seed(monitor);
|
||||
|
||||
// 2 real failures: window [t, t, t, f, f] = 40% < 60%
|
||||
h.setNextResponse(false, NETWORK_ERROR);
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
|
||||
h.setEgressStatus("degraded");
|
||||
await h.heartbeatJob(monitor);
|
||||
await h.heartbeatJob(monitor);
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).statusWindow).toEqual([]);
|
||||
|
||||
// A third real failure would have taken the old window to 60%. Against a window that starts again
|
||||
// at the end of the spell it is one result, and the results either side are never adjacent.
|
||||
h.setEgressStatus(null);
|
||||
await h.heartbeatJob(monitor);
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("up");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(0);
|
||||
|
||||
// The monitor still goes down, once the rebuilt window holds enough real failures to cross.
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await h.heartbeatJob(monitor);
|
||||
}
|
||||
|
||||
expect((await h.monitorsRepo.findById("mon-1", "team-1")).status).toBe("down");
|
||||
expect(h.incidentsRepo.getAll()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
Loaded 100 of 126 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user