Compare commits
41
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a277a89bef | ||
|
|
9d0dc3f330 | ||
|
|
da8b230cec | ||
|
|
f4864b5455 | ||
|
|
f299098548 | ||
|
|
87eab6f996 | ||
|
|
31400a2502 | ||
|
|
fa0c0abcac | ||
|
|
9102335d3e | ||
|
|
011b93ecc1 | ||
|
|
20ce11497e | ||
|
|
50d0721822 | ||
|
|
c66872c7f4 | ||
|
|
b1be38368c | ||
|
|
278f652ea9 | ||
|
|
d3e12722b7 | ||
|
|
8c9ff41074 | ||
|
|
71a1194373 | ||
|
|
580a025353 | ||
|
|
33994cb3df | ||
|
|
44edf320a0 | ||
|
|
fea2ef4dbe | ||
|
|
360b4b8035 | ||
|
|
3857a71e62 | ||
|
|
b68367b04f | ||
|
|
08f104e283 | ||
|
|
46eb0b4063 | ||
|
|
84539f76cf | ||
|
|
308672d558 | ||
|
|
0f474d82d6 | ||
|
|
650cfe00ce | ||
|
|
69eb9957c6 | ||
|
|
ec01a5dfe4 | ||
|
|
b2ace1bec5 | ||
|
|
5da472da14 | ||
|
|
4bb366f5ab | ||
|
|
cc4b8b38f5 | ||
|
|
86b59da909 | ||
|
|
71dd998390 | ||
|
|
896d73f12e | ||
|
|
c9b99eabb4 |
No files matched your search
@@ -92,7 +92,7 @@ JWT_SECRET="$(openssl rand -hex 32)" docker compose up -d
|
||||
Then open http://localhost:52345. If the app is reached at another origin (domain or LAN IP), set `CLIENT_HOST` accordingly. To build the image yourself, run `docker build -f docker/Dockerfile -t checkmate .` from a checkout. For TLS, put any reverse proxy (Caddy, Traefik, nginx) in front of port 52345.
|
||||
|
||||
There are also 1-click installation options like [Repocloud](https://repocloud.io/details/Checkmate),
|
||||
[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). Note that the Helm chart has not yet been migrated to the all-in-one image: it still deploys the legacy `checkmate-client`, `checkmate-backend` and `checkmate-mongo` images, pinned at v3.8.1.
|
||||
[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [Easypanel](https://easypanel.io/templates/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). Note that the Helm chart has not yet been migrated to the all-in-one image: it still deploys the legacy `checkmate-client`, `checkmate-backend` and `checkmate-mongo` images, pinned at v3.8.1.
|
||||
|
||||
|
||||
### Configuration
|
||||
|
||||
@@ -8,8 +8,11 @@ import { typographyLevels } from "@/Utils/Theme/Palette";
|
||||
import useMediaQuery from "@mui/material/useMediaQuery";
|
||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||
import { formatDuration } from "@/Utils/TimeUtils";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
export const MonitorStatus = ({ monitor }: { monitor: Monitor }) => {
|
||||
const theme = useTheme();
|
||||
const { t, i18n } = useTranslation();
|
||||
const isSmall = useMediaQuery(theme.breakpoints.down("md"));
|
||||
|
||||
if (!monitor) {
|
||||
@@ -52,7 +55,12 @@ export const MonitorStatus = ({ monitor }: { monitor: Monitor }) => {
|
||||
<>
|
||||
<Dot />
|
||||
<Typography>
|
||||
Checking every {formatDuration(monitor?.interval, true)}
|
||||
{t("pages.common.monitors.statBoxes.checkingEvery", {
|
||||
interval: formatDuration(monitor.interval, {
|
||||
long: true,
|
||||
locale: i18n.resolvedLanguage,
|
||||
}),
|
||||
})}
|
||||
</Typography>
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -5,9 +5,10 @@ import {
|
||||
type FieldValues,
|
||||
} from "react-hook-form";
|
||||
import { Autocomplete } from "@/Components/inputs";
|
||||
import type { AutocompleteProps } from "@mui/material/Autocomplete";
|
||||
|
||||
export interface AutocompleteOption {
|
||||
id: string;
|
||||
id: string | number;
|
||||
name: string;
|
||||
}
|
||||
|
||||
@@ -16,6 +17,9 @@ interface FormAutocompleteProps<T extends FieldValues, O extends AutocompleteOpt
|
||||
options: O[];
|
||||
fieldLabel?: string;
|
||||
placeholder?: string;
|
||||
filterOptions?: AutocompleteProps<O, false, false, false>["filterOptions"];
|
||||
getOptionDisabled?: AutocompleteProps<O, false, false, false>["getOptionDisabled"];
|
||||
disableClearable?: boolean;
|
||||
}
|
||||
|
||||
export const FormAutocompleteField = <
|
||||
@@ -26,6 +30,9 @@ export const FormAutocompleteField = <
|
||||
options,
|
||||
fieldLabel,
|
||||
placeholder,
|
||||
filterOptions,
|
||||
getOptionDisabled,
|
||||
disableClearable,
|
||||
}: FormAutocompleteProps<T, O>) => {
|
||||
const { control } = useFormContext<T>();
|
||||
return (
|
||||
@@ -39,10 +46,14 @@ export const FormAutocompleteField = <
|
||||
isOptionEqualToValue={(option, value) => option.id === value.id}
|
||||
value={options.find((o) => o.id === field.value) ?? null}
|
||||
onChange={(_: unknown, newValue: O | null) => {
|
||||
if (newValue === null && disableClearable) return;
|
||||
field.onChange(newValue?.id ?? "");
|
||||
}}
|
||||
fieldLabel={fieldLabel}
|
||||
placeholder={placeholder}
|
||||
filterOptions={filterOptions}
|
||||
getOptionDisabled={getOptionDisabled}
|
||||
disableClearable={disableClearable}
|
||||
error={!!fieldState.error}
|
||||
helperText={fieldState.error?.message ?? ""}
|
||||
/>
|
||||
|
||||
@@ -21,7 +21,7 @@ export const MonitorStatBoxes = ({
|
||||
domainExpiry,
|
||||
}: MonitorStatBoxesProps) => {
|
||||
const theme = useTheme();
|
||||
const { t } = useTranslation();
|
||||
const { t, i18n } = useTranslation();
|
||||
if (!monitorStats || !monitor) {
|
||||
return null;
|
||||
}
|
||||
@@ -29,13 +29,16 @@ export const MonitorStatBoxes = ({
|
||||
const timeOfLastFailure = monitorStats?.timeOfLastFailure || 0;
|
||||
const timeSinceLastFailure = timeOfLastFailure > 0 ? Date.now() - timeOfLastFailure : 0;
|
||||
|
||||
// Determine time since last check
|
||||
const timeOfLastCheck = monitorStats?.lastCheckTimestamp || 0;
|
||||
const timeSinceLastCheck = Date.now() - timeOfLastCheck || 0;
|
||||
|
||||
const streakTime = formatDuration(timeSinceLastFailure);
|
||||
|
||||
const lastCheckTime = formatDuration(timeSinceLastCheck);
|
||||
const lastCheckTime = timeOfLastCheck
|
||||
? formatDuration(Date.now() - timeOfLastCheck, {
|
||||
long: true,
|
||||
relative: true,
|
||||
locale: i18n.resolvedLanguage,
|
||||
})
|
||||
: t("common.labels.na");
|
||||
const isActive =
|
||||
monitor?.status === "up" ||
|
||||
monitor?.status === "paused" ||
|
||||
|
||||
@@ -20,12 +20,18 @@ import {
|
||||
Tag,
|
||||
Waypoints,
|
||||
Container,
|
||||
// LayoutDashboard,
|
||||
} from "lucide-react";
|
||||
export const getMenu = (t: Function) => {
|
||||
return [
|
||||
{
|
||||
group: t("components.sidebar.menu.groups.monitoring"),
|
||||
items: [
|
||||
// {
|
||||
// name: t("components.sidebar.menu.dashboard"),
|
||||
// path: "dashboard",
|
||||
// icon: <Icon icon={LayoutDashboard} />,
|
||||
// },
|
||||
{
|
||||
name: t("components.sidebar.menu.uptime"),
|
||||
path: "uptime",
|
||||
|
||||
@@ -2,6 +2,10 @@ import { createSlice, type PayloadAction } from "@reduxjs/toolkit";
|
||||
|
||||
type ThemeMode = "light" | "dark";
|
||||
type ChartType = "histogram" | "line";
|
||||
|
||||
export const dashboardCardKeys = ["monitorStatus", "monitorsByType"] as const;
|
||||
export type DashboardCardKey = (typeof dashboardCardKeys)[number];
|
||||
|
||||
type TableName =
|
||||
| "monitors"
|
||||
| "team"
|
||||
@@ -35,6 +39,7 @@ interface UIState {
|
||||
language: string;
|
||||
starPromptOpen: boolean;
|
||||
chartType: ChartType;
|
||||
dashboardVisibleCards: DashboardCardKey[];
|
||||
}
|
||||
|
||||
const initialMode: ThemeMode = window?.matchMedia?.("(prefers-color-scheme: dark)")
|
||||
@@ -74,6 +79,7 @@ const initialState: UIState = {
|
||||
language: "en",
|
||||
starPromptOpen: true,
|
||||
chartType: "histogram",
|
||||
dashboardVisibleCards: [...dashboardCardKeys],
|
||||
};
|
||||
|
||||
const uiSlice = createSlice({
|
||||
@@ -117,6 +123,9 @@ const uiSlice = createSlice({
|
||||
setChartType: (state, action: PayloadAction<ChartType>) => {
|
||||
state.chartType = action.payload;
|
||||
},
|
||||
setDashboardVisibleCards: (state, action: PayloadAction<DashboardCardKey[]>) => {
|
||||
state.dashboardVisibleCards = action.payload;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -133,4 +142,5 @@ export const {
|
||||
setLanguage,
|
||||
setStarPromptOpen,
|
||||
setChartType,
|
||||
setDashboardVisibleCards,
|
||||
} = uiSlice.actions;
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
PageSpeedStrategies,
|
||||
DnsRecordTypes,
|
||||
MonitorIntervalOptions,
|
||||
MIN_MONITOR_INTERVAL_MS,
|
||||
DefaultMonitorMatchMethod,
|
||||
MonitorMatchMethods,
|
||||
GeoCheckIntervalOptions,
|
||||
@@ -58,9 +59,14 @@ import { FormTextField } from "@/Components/inputs/forms/FormTextField";
|
||||
import { FormRadioGroup } from "@/Components/inputs/forms/FormRadioGroupField";
|
||||
import { FormMultiSelectField } from "@/Components/inputs/forms/FormMultiSelectField";
|
||||
import { FormSelectField } from "@/Components/inputs/forms/FormSelectField";
|
||||
import {
|
||||
FormAutocompleteField,
|
||||
type AutocompleteOption,
|
||||
} from "@/Components/inputs/forms/FormAutoCompleteField";
|
||||
import { FormSliderField } from "@/Components/inputs/forms/FormSliderField";
|
||||
import { FormSwitchField } from "@/Components/inputs/forms/FormSwitchField";
|
||||
import type { ProxyResponse } from "@/Types/Proxy";
|
||||
import { formatDuration } from "@/Utils/TimeUtils";
|
||||
|
||||
const httpMethodOptions = HttpMethods.map((method) => ({
|
||||
value: method,
|
||||
@@ -75,6 +81,40 @@ const maskedInputSx = {
|
||||
"&:not(.Mui-focused) textarea": { WebkitTextSecurity: "disc" },
|
||||
};
|
||||
|
||||
interface IntervalOption extends AutocompleteOption {
|
||||
id: number;
|
||||
}
|
||||
|
||||
const INTERVAL_UNIT_MULTIPLIERS = [1000, 60000, 3600000, 86400000] as const;
|
||||
|
||||
const toIntervalOption = (
|
||||
intervalMs: number,
|
||||
locale: string | undefined
|
||||
): IntervalOption => ({
|
||||
id: intervalMs,
|
||||
name: formatDuration(intervalMs, { long: true, locale }),
|
||||
});
|
||||
|
||||
const filterIntervalOptions = (
|
||||
options: IntervalOption[],
|
||||
inputValue: string,
|
||||
locale: string | undefined
|
||||
): IntervalOption[] => {
|
||||
const input = inputValue.trim();
|
||||
if (input === "" || options.some((option) => option.name === input)) {
|
||||
return options;
|
||||
}
|
||||
|
||||
const amount = Number(input);
|
||||
if (!Number.isFinite(amount) || amount <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return INTERVAL_UNIT_MULTIPLIERS.map((multiplier) => amount * multiplier)
|
||||
.filter(Number.isSafeInteger)
|
||||
.map((ms) => toIntervalOption(ms, locale));
|
||||
};
|
||||
|
||||
interface GeneralSettingsConfig {
|
||||
urlLabel: string;
|
||||
urlPlaceholder: string;
|
||||
@@ -252,7 +292,7 @@ const getGeneralSettingsConfig = (
|
||||
|
||||
const CreateMonitorPage = () => {
|
||||
const theme = useTheme();
|
||||
const { t } = useTranslation();
|
||||
const { t, i18n } = useTranslation();
|
||||
const { monitorId } = useParams();
|
||||
const location = useLocation();
|
||||
const navigate = useNavigate();
|
||||
@@ -314,6 +354,7 @@ const CreateMonitorPage = () => {
|
||||
const watchedProxyMode = watch("proxyMode") as ProxyMode | undefined;
|
||||
const watchedUseAdvancedMatching = watch("useAdvancedMatching") as boolean;
|
||||
const watchGeoCheckEnabled = watch("geoCheckEnabled") as boolean;
|
||||
const watchedInterval = watch("interval");
|
||||
|
||||
// Steps without an advanced section drop it, so the last step is the form's.
|
||||
const totalSteps = monitorStepCount(watchedType);
|
||||
@@ -471,16 +512,18 @@ const CreateMonitorPage = () => {
|
||||
[t]
|
||||
);
|
||||
|
||||
const intervalOptions = useMemo(
|
||||
() =>
|
||||
MonitorIntervalOptions.map((option) => ({
|
||||
value: option.value,
|
||||
label: t(
|
||||
`pages.createMonitor.form.frequency.option.frequency.value.${option.labelKey}`
|
||||
),
|
||||
})),
|
||||
[t]
|
||||
);
|
||||
const intervalOptions = useMemo(() => {
|
||||
const options = MonitorIntervalOptions.map(({ value }) =>
|
||||
toIntervalOption(value, i18n.resolvedLanguage)
|
||||
);
|
||||
if (
|
||||
typeof watchedInterval === "number" &&
|
||||
!options.some((option) => option.id === watchedInterval)
|
||||
) {
|
||||
options.push(toIntervalOption(watchedInterval, i18n.resolvedLanguage));
|
||||
}
|
||||
return options;
|
||||
}, [watchedInterval, i18n.resolvedLanguage]);
|
||||
|
||||
const matchMethodOptions = useMemo(
|
||||
() =>
|
||||
@@ -764,12 +807,17 @@ const CreateMonitorPage = () => {
|
||||
title={t("pages.createMonitor.form.frequency.title")}
|
||||
subtitle={t("pages.createMonitor.form.frequency.description")}
|
||||
rightContent={
|
||||
<FormSelectField
|
||||
<FormAutocompleteField
|
||||
name="interval"
|
||||
fieldLabel={t(
|
||||
"pages.createMonitor.form.frequency.option.frequency.label"
|
||||
)}
|
||||
options={intervalOptions}
|
||||
filterOptions={(options, { inputValue }) =>
|
||||
filterIntervalOptions(options, inputValue, i18n.resolvedLanguage)
|
||||
}
|
||||
getOptionDisabled={(option) => option.id < MIN_MONITOR_INTERVAL_MS}
|
||||
disableClearable
|
||||
/>
|
||||
}
|
||||
/>
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import Box from "@mui/material/Box";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||
import { typographyLevels } from "@/Utils/Theme/Palette";
|
||||
|
||||
interface DashboardCardProps {
|
||||
title: string;
|
||||
topRight?: React.ReactNode;
|
||||
children: React.ReactNode;
|
||||
}
|
||||
|
||||
export const DashboardCard = ({ title, topRight, children }: DashboardCardProps) => {
|
||||
const theme = useTheme();
|
||||
|
||||
return (
|
||||
<Box
|
||||
bgcolor={theme.palette.background.paper}
|
||||
border={1}
|
||||
borderColor={theme.palette.divider}
|
||||
borderRadius={`${theme.shape.borderRadius}px`}
|
||||
padding={theme.spacing(LAYOUT.MD)}
|
||||
display="flex"
|
||||
flexDirection="column"
|
||||
gap={theme.spacing(LAYOUT.SM)}
|
||||
height="100%"
|
||||
boxSizing="border-box"
|
||||
>
|
||||
<Stack
|
||||
direction="row"
|
||||
alignItems="center"
|
||||
justifyContent="space-between"
|
||||
>
|
||||
<Typography
|
||||
fontSize={typographyLevels.m}
|
||||
fontWeight={500}
|
||||
color={theme.palette.text.primary}
|
||||
>
|
||||
{title}
|
||||
</Typography>
|
||||
{topRight && (
|
||||
<Typography
|
||||
fontSize={typographyLevels.s}
|
||||
color={theme.palette.text.secondary}
|
||||
>
|
||||
{topRight}
|
||||
</Typography>
|
||||
)}
|
||||
</Stack>
|
||||
{children}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
import { useState } from "react";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import FormControlLabel from "@mui/material/FormControlLabel";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Checkbox, Dialog } from "@/Components/inputs";
|
||||
import { type DashboardCardKey, dashboardCardKeys } from "@/Features/UI/uiSlice";
|
||||
|
||||
export type { DashboardCardKey };
|
||||
|
||||
interface EditCardsModalProps {
|
||||
open: boolean;
|
||||
visibleCards: Set<DashboardCardKey>;
|
||||
onClose: () => void;
|
||||
onSave: (visibleCards: Set<DashboardCardKey>) => void;
|
||||
}
|
||||
|
||||
export const EditCardsModal = ({
|
||||
open,
|
||||
visibleCards,
|
||||
onClose,
|
||||
onSave,
|
||||
}: EditCardsModalProps) => {
|
||||
const { t } = useTranslation();
|
||||
const [draft, setDraft] = useState<Set<DashboardCardKey>>(new Set(visibleCards));
|
||||
|
||||
const handleClose = () => {
|
||||
setDraft(new Set(visibleCards));
|
||||
onClose();
|
||||
};
|
||||
|
||||
const handleToggle = (key: DashboardCardKey) => {
|
||||
setDraft((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(key)) {
|
||||
next.delete(key);
|
||||
} else {
|
||||
next.add(key);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
open={open}
|
||||
title={t("pages.dashboard.editCardsModal.title")}
|
||||
content={t("pages.dashboard.editCardsModal.description")}
|
||||
onCancel={handleClose}
|
||||
onConfirm={() => onSave(draft)}
|
||||
confirmText={t("pages.dashboard.editCardsModal.done")}
|
||||
maxWidth="xs"
|
||||
fullWidth
|
||||
>
|
||||
<Stack>
|
||||
{dashboardCardKeys.map((key) => (
|
||||
<FormControlLabel
|
||||
key={key}
|
||||
label={t(`pages.dashboard.cards.${key}`)}
|
||||
control={
|
||||
<Checkbox
|
||||
checked={draft.has(key)}
|
||||
disabled={draft.has(key) && draft.size === 1}
|
||||
onChange={() => handleToggle(key)}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</Stack>
|
||||
</Dialog>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,28 @@
|
||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||
import Box from "@mui/material/Box";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
|
||||
interface ProgressBarProps {
|
||||
fillRatio: number;
|
||||
}
|
||||
|
||||
export const ProgressBar = ({ fillRatio }: ProgressBarProps) => {
|
||||
const theme = useTheme();
|
||||
const clamped = Math.min(1, Math.max(0, fillRatio));
|
||||
|
||||
return (
|
||||
<Box
|
||||
height={theme.spacing(LAYOUT.XS)}
|
||||
borderRadius={theme.shape.borderRadius}
|
||||
bgcolor={theme.palette.action.disabledBackground}
|
||||
overflow="hidden"
|
||||
>
|
||||
<Box
|
||||
height="100%"
|
||||
width={`${clamped * 100}%`}
|
||||
borderRadius={theme.shape.borderRadius}
|
||||
bgcolor={theme.palette.primary.main}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
import { typographyLevels } from "@/Utils/Theme/Palette";
|
||||
import { SPACING } from "@/Utils/Theme/constants";
|
||||
import { ProgressBar } from "@/Pages/Dashboard/components/ProgressBar";
|
||||
|
||||
interface BarRowProps {
|
||||
label: string;
|
||||
value: string;
|
||||
fillRatio: number;
|
||||
valueColor?: string;
|
||||
}
|
||||
|
||||
export const BarRow = ({ label, value, fillRatio, valueColor }: BarRowProps) => {
|
||||
const theme = useTheme();
|
||||
|
||||
return (
|
||||
<Stack gap={theme.spacing(SPACING.LG)}>
|
||||
<Stack
|
||||
direction="row"
|
||||
justifyContent="space-between"
|
||||
alignItems="center"
|
||||
>
|
||||
<Typography
|
||||
fontSize={typographyLevels.m}
|
||||
color={theme.palette.text.primary}
|
||||
>
|
||||
{label}
|
||||
</Typography>
|
||||
<Typography
|
||||
fontSize={typographyLevels.m}
|
||||
color={valueColor ?? theme.palette.primary.main}
|
||||
fontWeight={500}
|
||||
>
|
||||
{value}
|
||||
</Typography>
|
||||
</Stack>
|
||||
<ProgressBar fillRatio={fillRatio} />
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,58 @@
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
|
||||
import { ProgressBar } from "@/Pages/Dashboard/components/ProgressBar";
|
||||
import { typographyLevels } from "@/Utils/Theme/Palette";
|
||||
import { SPACING } from "@/Utils/Theme/constants";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import type { MonitorsSummary } from "@/Types/Monitor";
|
||||
|
||||
interface MonitorStatusCardProps {
|
||||
summary: MonitorsSummary | null;
|
||||
}
|
||||
|
||||
export const MonitorStatusCard = ({ summary }: MonitorStatusCardProps) => {
|
||||
const theme = useTheme();
|
||||
const { t } = useTranslation();
|
||||
|
||||
const up = summary?.upMonitors ?? 0;
|
||||
const total = summary?.totalMonitors ?? 0;
|
||||
const ratio = total > 0 ? up / total : 0;
|
||||
|
||||
return (
|
||||
<DashboardCard title={t("pages.dashboard.cards.monitorStatus")}>
|
||||
{total === 0 ? (
|
||||
<Typography
|
||||
fontSize={typographyLevels.m}
|
||||
color={theme.palette.text.disabled}
|
||||
>
|
||||
{t("pages.dashboard.noData")}
|
||||
</Typography>
|
||||
) : (
|
||||
<Stack gap={theme.spacing(SPACING.SM)}>
|
||||
<Stack
|
||||
direction="row"
|
||||
alignItems="baseline"
|
||||
gap={theme.spacing(SPACING.XS)}
|
||||
>
|
||||
<Typography
|
||||
fontSize={typographyLevels.xxl}
|
||||
fontWeight={500}
|
||||
color={theme.palette.text.primary}
|
||||
>
|
||||
{up}
|
||||
</Typography>
|
||||
<Typography
|
||||
fontSize={typographyLevels.m}
|
||||
color={theme.palette.text.secondary}
|
||||
>
|
||||
{t("pages.dashboard.monitorStatus.ofUp", { total })}
|
||||
</Typography>
|
||||
</Stack>
|
||||
<ProgressBar fillRatio={ratio} />
|
||||
</Stack>
|
||||
)}
|
||||
</DashboardCard>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,46 @@
|
||||
import Stack from "@mui/material/Stack";
|
||||
import Typography from "@mui/material/Typography";
|
||||
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
|
||||
import { BarRow } from "@/Pages/Dashboard/components/cards/BarRow";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
import type { MonitorTypeCount } from "@/Types/Monitor";
|
||||
import { getMonitorTypeLabel } from "@/Types/Monitor";
|
||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||
import { typographyLevels } from "@/Utils/Theme/Palette";
|
||||
|
||||
interface MonitorsByTypeCardProps {
|
||||
monitorsByType: MonitorTypeCount[];
|
||||
}
|
||||
|
||||
export const MonitorsByTypeCard = ({ monitorsByType }: MonitorsByTypeCardProps) => {
|
||||
const { t } = useTranslation();
|
||||
const theme = useTheme();
|
||||
|
||||
const maxCount = monitorsByType[0]?.count ?? 1;
|
||||
|
||||
return (
|
||||
<DashboardCard title={t("pages.dashboard.cards.monitorsByType")}>
|
||||
{monitorsByType.length === 0 ? (
|
||||
<Typography
|
||||
fontSize={typographyLevels.m}
|
||||
color={theme.palette.text.disabled}
|
||||
>
|
||||
{t("pages.dashboard.noData")}
|
||||
</Typography>
|
||||
) : (
|
||||
<Stack gap={theme.spacing(LAYOUT.SM)}>
|
||||
{monitorsByType.map(({ type, count }) => (
|
||||
<BarRow
|
||||
key={type}
|
||||
label={getMonitorTypeLabel(type, t)}
|
||||
value={String(count)}
|
||||
fillRatio={count / maxCount}
|
||||
valueColor={theme.palette.text.primary}
|
||||
/>
|
||||
))}
|
||||
</Stack>
|
||||
)}
|
||||
</DashboardCard>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
import { useMemo, useState, type ReactNode } from "react";
|
||||
import Box from "@mui/material/Box";
|
||||
import Stack from "@mui/material/Stack";
|
||||
import { useTheme } from "@mui/material/styles";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useSelector, useDispatch } from "react-redux";
|
||||
|
||||
import { BasePage } from "@/Components/design-elements/BasePage";
|
||||
import { Button } from "@/Components/inputs";
|
||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||
import { useMonitorListController } from "@/Hooks/useMonitorListController";
|
||||
import { MonitorTypes, type MonitorTypeCount } from "@/Types/Monitor";
|
||||
import {
|
||||
setDashboardVisibleCards,
|
||||
dashboardCardKeys,
|
||||
type DashboardCardKey,
|
||||
} from "@/Features/UI/uiSlice";
|
||||
import type { RootState, AppDispatch } from "@/store";
|
||||
|
||||
import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard";
|
||||
import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard";
|
||||
import { EditCardsModal } from "@/Pages/Dashboard/components/EditCardsModal";
|
||||
|
||||
const Dashboard = () => {
|
||||
const theme = useTheme();
|
||||
const { t } = useTranslation();
|
||||
const dispatch = useDispatch<AppDispatch>();
|
||||
const [editOpen, setEditOpen] = useState(false);
|
||||
const visibleCardsList = useSelector(
|
||||
(state: RootState) => state.ui.dashboardVisibleCards
|
||||
);
|
||||
const visibleCards = useMemo(() => new Set(visibleCardsList), [visibleCardsList]);
|
||||
|
||||
const { monitors, summary } = useMonitorListController({
|
||||
types: [...MonitorTypes],
|
||||
checksLimit: 1,
|
||||
refreshInterval: 30000,
|
||||
rowsPerPageTable: "monitors",
|
||||
rowsPerPageDefault: 100,
|
||||
});
|
||||
|
||||
const monitorsByType = useMemo<MonitorTypeCount[]>(() => {
|
||||
const countMap = new Map<MonitorTypeCount["type"], number>();
|
||||
(monitors ?? []).forEach((m) =>
|
||||
countMap.set(m.type, (countMap.get(m.type) ?? 0) + 1)
|
||||
);
|
||||
return [...countMap.entries()]
|
||||
.map(([type, count]) => ({ type, count }))
|
||||
.sort((a, b) => b.count - a.count);
|
||||
}, [monitors]);
|
||||
|
||||
const cardComponents: Record<DashboardCardKey, ReactNode> = {
|
||||
monitorStatus: <MonitorStatusCard summary={summary} />,
|
||||
monitorsByType: <MonitorsByTypeCard monitorsByType={monitorsByType} />,
|
||||
};
|
||||
|
||||
return (
|
||||
<BasePage headerKey="dashboard">
|
||||
<Stack gap={theme.spacing(LAYOUT.SM)}>
|
||||
<Stack
|
||||
direction="row"
|
||||
justifyContent="flex-end"
|
||||
>
|
||||
<Button
|
||||
variant="outlined"
|
||||
size="small"
|
||||
onClick={() => setEditOpen(true)}
|
||||
>
|
||||
{t("pages.dashboard.editCards")}
|
||||
</Button>
|
||||
</Stack>
|
||||
|
||||
<Box
|
||||
display="grid"
|
||||
gridTemplateColumns={{ xs: "1fr", md: "1fr 1fr" }}
|
||||
gap={theme.spacing(LAYOUT.MD)}
|
||||
>
|
||||
{dashboardCardKeys
|
||||
.filter((key) => visibleCards.has(key))
|
||||
.map((key) => (
|
||||
<div key={key}>{cardComponents[key]}</div>
|
||||
))}
|
||||
</Box>
|
||||
</Stack>
|
||||
|
||||
<EditCardsModal
|
||||
open={editOpen}
|
||||
visibleCards={visibleCards}
|
||||
onClose={() => setEditOpen(false)}
|
||||
onSave={(next) => {
|
||||
dispatch(setDashboardVisibleCards([...next]));
|
||||
setEditOpen(false);
|
||||
}}
|
||||
/>
|
||||
</BasePage>
|
||||
);
|
||||
};
|
||||
|
||||
export default Dashboard;
|
||||
@@ -8,7 +8,7 @@ import { useTranslation } from "react-i18next";
|
||||
|
||||
const LogsPage = () => {
|
||||
const { t } = useTranslation();
|
||||
const [activeTab, setActiveTab] = useState<number>(1);
|
||||
const [activeTab, setActiveTab] = useState<number>(0);
|
||||
return (
|
||||
<BasePage headerKey="logs">
|
||||
<Tabs
|
||||
|
||||
@@ -8,6 +8,9 @@ import AuthRegister from "@/Pages/Auth/Register";
|
||||
import AuthForgotPassword from "@/Pages/Auth/Recovery";
|
||||
import AuthSetNewPassword from "@/Pages/Auth/SetNewPassword";
|
||||
|
||||
// Dashboard
|
||||
import Dashboard from "@/Pages/Dashboard";
|
||||
|
||||
// Uptime
|
||||
import Uptime from "@/Pages/Uptime/Monitors";
|
||||
import UptimeDetails from "@/Pages/Uptime/Details";
|
||||
@@ -92,6 +95,10 @@ const Routes = () => {
|
||||
path="/"
|
||||
element={<Navigate to="/uptime" />}
|
||||
/>
|
||||
<Route
|
||||
path="dashboard"
|
||||
element={<Dashboard />}
|
||||
/>
|
||||
<Route
|
||||
path="/uptime"
|
||||
element={<Uptime />}
|
||||
|
||||
@@ -60,6 +60,15 @@ export const monitorTypeLabelKey: Record<SelectableMonitorType, string> &
|
||||
pagespeed: "optionPagespeed",
|
||||
};
|
||||
|
||||
export const getMonitorTypeLabel = (
|
||||
type: MonitorType,
|
||||
t: (key: string) => string
|
||||
): string => {
|
||||
const monitorLabel = monitorTypeLabelKey[type];
|
||||
return monitorLabel ? t(`pages.common.monitors.monitorTypes.${monitorLabel}`) : type;
|
||||
};
|
||||
export const MIN_MONITOR_INTERVAL_MS = 10000;
|
||||
|
||||
export const MonitorIntervalOptions = [
|
||||
{ value: 15000, labelKey: "fifteenSeconds" },
|
||||
{ value: 30000, labelKey: "thirtySeconds" },
|
||||
@@ -183,6 +192,11 @@ export interface MonitorsSummary {
|
||||
breachedMonitors: number;
|
||||
}
|
||||
|
||||
export interface MonitorTypeCount {
|
||||
type: MonitorType;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface MonitorsWithChecksResponse {
|
||||
count: number;
|
||||
monitors: MonitorWithChecks[];
|
||||
|
||||
@@ -104,9 +104,52 @@ export const formatMs = (ms: number, hasSpace: boolean = true): string => {
|
||||
return formatted;
|
||||
};
|
||||
|
||||
export const formatDuration = (ms: number, verbose: boolean = false, hasSpace = true) => {
|
||||
if (verbose) {
|
||||
return prettyMilliseconds(ms, { verbose: true });
|
||||
type DurationUnit = "day" | "hour" | "minute" | "second";
|
||||
|
||||
interface FormatDurationOptions {
|
||||
long?: boolean;
|
||||
relative?: boolean;
|
||||
hasSpace?: boolean;
|
||||
locale?: string;
|
||||
}
|
||||
|
||||
const DURATION_UNITS: Array<{ unit: DurationUnit; milliseconds: number }> = [
|
||||
{ unit: "day", milliseconds: MS_PER_DAY },
|
||||
{ unit: "hour", milliseconds: MS_PER_HOUR },
|
||||
{ unit: "minute", milliseconds: MS_PER_MINUTE },
|
||||
{ unit: "second", milliseconds: MS_PER_SECOND },
|
||||
];
|
||||
|
||||
const FALLBACK_DURATION_UNIT = DURATION_UNITS[DURATION_UNITS.length - 1];
|
||||
|
||||
const getDurationUnit = (ms: number) =>
|
||||
DURATION_UNITS.find(({ milliseconds }) => Math.abs(ms) >= milliseconds) ??
|
||||
FALLBACK_DURATION_UNIT;
|
||||
|
||||
const getExactDurationUnit = (ms: number) =>
|
||||
DURATION_UNITS.find(({ milliseconds }) => ms !== 0 && ms % milliseconds === 0) ??
|
||||
FALLBACK_DURATION_UNIT;
|
||||
|
||||
export const formatDuration = (
|
||||
ms: number,
|
||||
{ long = false, relative = false, hasSpace = true, locale }: FormatDurationOptions = {}
|
||||
): string => {
|
||||
if (relative) {
|
||||
const { unit, milliseconds } = getDurationUnit(ms);
|
||||
const value = Math.sign(ms) * Math.floor(Math.abs(ms) / milliseconds);
|
||||
return new Intl.RelativeTimeFormat(locale, {
|
||||
numeric: "always",
|
||||
style: long ? "long" : "narrow",
|
||||
}).format(-value, unit);
|
||||
}
|
||||
|
||||
if (long) {
|
||||
const { unit, milliseconds } = getExactDurationUnit(ms);
|
||||
return new Intl.NumberFormat(locale, {
|
||||
style: "unit",
|
||||
unit,
|
||||
unitDisplay: "long",
|
||||
}).format(ms / milliseconds);
|
||||
}
|
||||
|
||||
const formatted = prettyMilliseconds(ms, {
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
HttpMethods,
|
||||
PageSpeedStrategies,
|
||||
ProxyModes,
|
||||
MIN_MONITOR_INTERVAL_MS,
|
||||
type MonitorType,
|
||||
} from "@/Types/Monitor";
|
||||
import { ALL_HTTP_STATUS_CODES } from "@/Utils/statusCode";
|
||||
@@ -27,7 +28,8 @@ const baseSchema = z.object({
|
||||
description: z.string().optional(),
|
||||
interval: z
|
||||
.number()
|
||||
.min(15000, "Interval must be at least 15 seconds")
|
||||
.int("Interval must be a whole number of milliseconds")
|
||||
.min(MIN_MONITOR_INTERVAL_MS, "Interval must be at least 10 seconds")
|
||||
.register(monitorStepRegistry, { step: 1 }),
|
||||
notifications: z.array(z.string()).register(monitorStepRegistry, { step: 1 }),
|
||||
tags: z.array(z.string()).register(monitorStepRegistry, { step: 1 }),
|
||||
|
||||
@@ -448,6 +448,7 @@
|
||||
},
|
||||
"statBoxes": {
|
||||
"activeFor": "Aktiv seit",
|
||||
"checkingEvery": "Check alle {{interval}}",
|
||||
"certificateExpiry": "Zertifikatablauf",
|
||||
"lastCheck": "Letzter Check",
|
||||
"lastResponseTime": "Letzte Antwortzeit",
|
||||
|
||||
@@ -226,7 +226,8 @@
|
||||
"alerting": "Alerting",
|
||||
"reporting": "Reporting",
|
||||
"configuration": "Configuration"
|
||||
}
|
||||
},
|
||||
"dashboard": "Dashboard"
|
||||
},
|
||||
"bottomMenu": {
|
||||
"support": "Support",
|
||||
@@ -547,6 +548,7 @@
|
||||
},
|
||||
"statBoxes": {
|
||||
"activeFor": "Active for",
|
||||
"checkingEvery": "Checking every {{interval}}",
|
||||
"certificateExpiry": "Certificate expiry",
|
||||
"domainExpiry": "Domain expiry",
|
||||
"lastCheck": "Last check",
|
||||
@@ -1907,6 +1909,26 @@
|
||||
"title": "Uptime monitors",
|
||||
"description": "Watch HTTP endpoints, pings, containers, and ports — and get alerted the moment something goes down."
|
||||
}
|
||||
},
|
||||
"dashboard": {
|
||||
"noData": "No data available at this moment.",
|
||||
"monitorStatus": {
|
||||
"ofUp": "of {{total}} up"
|
||||
},
|
||||
"cards": {
|
||||
"monitorStatus": "Monitor status",
|
||||
"monitorsByType": "Monitors by type"
|
||||
},
|
||||
"header": {
|
||||
"title": "Dashboard",
|
||||
"description": "See every monitor type in one place and spot what needs attention."
|
||||
},
|
||||
"editCards": "Edit cards",
|
||||
"editCardsModal": {
|
||||
"title": "Edit cards",
|
||||
"description": "Select the cards to display on your dashboard.",
|
||||
"done": "Done"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13294,6 +13294,30 @@
|
||||
"globalProxyId": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"egressCheckEnabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"egressCheckTargets": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 253
|
||||
},
|
||||
"maxItems": 10
|
||||
},
|
||||
"egressPollIntervalSeconds": {
|
||||
"type": "integer",
|
||||
"minimum": 5,
|
||||
"maximum": 600
|
||||
},
|
||||
"egressNotifications": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-fA-F]{24}$"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { AppError } from "@/utils/AppError.js";
|
||||
import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import { IEmailService } from "@/service/emailService.js";
|
||||
import { IProxiesService } from "@/domain/proxies/proxy.service.js";
|
||||
import { IEgressStateService } from "@/domain/egress/egress-state.service.js";
|
||||
import { Settings } from "@/domain/app-settings/app-settings.type.js";
|
||||
|
||||
export interface ISettingsController {
|
||||
@@ -18,10 +19,17 @@ class SettingsController implements ISettingsController {
|
||||
private settingsService: ISettingsService;
|
||||
private emailService: IEmailService;
|
||||
private proxiesService: IProxiesService;
|
||||
constructor(settingsService: ISettingsService, emailService: IEmailService, proxiesService: IProxiesService) {
|
||||
private egressStateService: IEgressStateService;
|
||||
constructor(
|
||||
settingsService: ISettingsService,
|
||||
emailService: IEmailService,
|
||||
proxiesService: IProxiesService,
|
||||
egressStateService: IEgressStateService
|
||||
) {
|
||||
this.settingsService = settingsService;
|
||||
this.emailService = emailService;
|
||||
this.proxiesService = proxiesService;
|
||||
this.egressStateService = egressStateService;
|
||||
}
|
||||
|
||||
buildAppSettings = async (dbSettings: Settings) => {
|
||||
@@ -70,7 +78,14 @@ class SettingsController implements ISettingsController {
|
||||
}
|
||||
}
|
||||
|
||||
const previousSettings = await this.settingsService.getDBSettings();
|
||||
const updatedSettings = await this.settingsService.updateDbSettings(validatedBody);
|
||||
|
||||
// Switching the egress check on or off starts from a clean state: no degraded episode, no pending recovery job
|
||||
if (validatedBody.egressCheckEnabled !== undefined && validatedBody.egressCheckEnabled !== previousSettings.egressCheckEnabled) {
|
||||
await this.egressStateService.reset();
|
||||
}
|
||||
|
||||
const returnSettings = await this.buildAppSettings(updatedSettings);
|
||||
return res.status(200).json({
|
||||
success: true,
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
|
||||
import { parseEgressTarget } from "@/utils/egressTarget.js";
|
||||
import { z } from "zod";
|
||||
|
||||
// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL.
|
||||
// Parsed with the same function the probe uses, so nothing the validator accepts can fail to probe.
|
||||
const egressTargetValidation = z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1)
|
||||
.max(253)
|
||||
.refine((target) => parseEgressTarget(target) !== null, "Enter a hostname or IP address, host:port, or http(s) URL");
|
||||
|
||||
//****************************************
|
||||
// Settings Validations
|
||||
//****************************************
|
||||
@@ -42,6 +53,11 @@ export const updateAppSettingsBodyValidation = z
|
||||
.optional(),
|
||||
globalProxyEnabled: z.boolean().optional(),
|
||||
globalProxyId: z.string().nullable().optional(),
|
||||
|
||||
egressCheckEnabled: z.boolean().optional(),
|
||||
egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
|
||||
egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
|
||||
egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(),
|
||||
})
|
||||
.strip()
|
||||
.superRefine((body, ctx) => {
|
||||
|
||||
@@ -36,7 +36,12 @@ export const initializeControllers = (apiServices: ApiServices): InitializedCont
|
||||
return {
|
||||
authController: new AuthController(apiServices.userService),
|
||||
monitorController: new MonitorController(apiServices.monitorService, apiServices.notificationsService),
|
||||
settingsController: new SettingsController(apiServices.settingsService, apiServices.emailService, apiServices.proxiesService),
|
||||
settingsController: new SettingsController(
|
||||
apiServices.settingsService,
|
||||
apiServices.emailService,
|
||||
apiServices.proxiesService,
|
||||
apiServices.egressStateService
|
||||
),
|
||||
checkController: new CheckController(apiServices.checkService),
|
||||
geoCheckController: new GeoCheckController(apiServices.geoChecksService),
|
||||
inviteController: new InviteController(apiServices.inviteService),
|
||||
|
||||
@@ -7,6 +7,7 @@ import { ITagsService, TagsService } from "@/domain/tags/tag.service.js";
|
||||
import { IUserService, UserService } from "@/domain/users/user.service.js";
|
||||
import { IJobScheduler } from "@/worker/worker.interface.js";
|
||||
import { ProxiesService, IProxiesService } from "@/domain/proxies/proxy.service.js";
|
||||
import { EgressStateService, IEgressStateService } from "@/domain/egress/egress-state.service.js";
|
||||
import { SharedServices } from "@/config/services.shared.js";
|
||||
|
||||
// Third-party
|
||||
@@ -24,6 +25,7 @@ export interface ApiServices extends SharedServices {
|
||||
tagsService: ITagsService;
|
||||
diagnosticService: IDiagnosticService;
|
||||
proxiesService: IProxiesService;
|
||||
egressStateService: IEgressStateService;
|
||||
}
|
||||
|
||||
export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => {
|
||||
@@ -48,6 +50,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
maintenanceWindowsRepository,
|
||||
jobsRepository,
|
||||
proxiesRepository,
|
||||
egressStateRepository,
|
||||
} = shared;
|
||||
|
||||
const userService = new UserService({
|
||||
@@ -100,6 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
const tagsService = new TagsService(tagsRepository, monitorsRepository);
|
||||
const diagnosticService = new DiagnosticService(db);
|
||||
const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService);
|
||||
const egressStateService = new EgressStateService(egressStateRepository, jobsRepository);
|
||||
return {
|
||||
...shared,
|
||||
worker: jobScheduler,
|
||||
@@ -111,5 +115,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
|
||||
tagsService,
|
||||
diagnosticService,
|
||||
proxiesService,
|
||||
egressStateService,
|
||||
};
|
||||
};
|
||||
@@ -56,6 +56,7 @@ import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
|
||||
import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.js";
|
||||
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
|
||||
// Mongo repository implementations
|
||||
import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js";
|
||||
@@ -76,6 +77,7 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js";
|
||||
import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
|
||||
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
|
||||
import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js";
|
||||
import MongoEgressStateRepository from "@/domain/egress/egress-state.repository.mongo.js";
|
||||
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
|
||||
|
||||
// Shared infrastructure + business services that both the API and the worker process construct.
|
||||
@@ -114,6 +116,7 @@ export interface SharedServices {
|
||||
teamsRepository: ITeamsRepository;
|
||||
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
proxiesRepository: IProxiesRepository;
|
||||
egressStateRepository: IEgressStateRepository;
|
||||
}
|
||||
|
||||
export const buildShared = async ({
|
||||
@@ -157,6 +160,7 @@ export const buildShared = async ({
|
||||
const teamsRepository = new MongoTeamsRepository();
|
||||
const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
|
||||
const proxiesRepository = new MongoProxiesRepository();
|
||||
const egressStateRepository = new MongoEgressStateRepository();
|
||||
|
||||
// Inject settings repository into settings service (now that DB is connected)
|
||||
(settingsService as SettingsService).setRepository(settingsRepository);
|
||||
@@ -246,6 +250,7 @@ export const buildShared = async ({
|
||||
teamsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
proxiesRepository,
|
||||
egressStateRepository,
|
||||
};
|
||||
return sharedServices;
|
||||
};
|
||||
@@ -28,6 +28,7 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
|
||||
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
|
||||
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
|
||||
import { ProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import { IEgressService, EgressService } from "@/domain/egress/egress.service.js";
|
||||
// Network providers
|
||||
import { PingProvider } from "@/service/network/PingProvider.js";
|
||||
import { HttpProvider } from "@/service/network/HttpProvider.js";
|
||||
@@ -46,6 +47,7 @@ export interface WorkerServices {
|
||||
networkService: INetworkService;
|
||||
bufferService: IBufferService;
|
||||
statusService: IStatusService;
|
||||
egressService: IEgressService;
|
||||
}
|
||||
|
||||
export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise<WorkerServices> => {
|
||||
@@ -70,12 +72,13 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
teamsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
proxiesRepository,
|
||||
egressStateRepository,
|
||||
} = shared;
|
||||
|
||||
// ***********************
|
||||
// Network providers
|
||||
// ***********************
|
||||
const pingProvider = new PingProvider(ping);
|
||||
const pingProvider = new PingProvider(ping, net);
|
||||
const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
|
||||
const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
|
||||
const hardwareProvider = new HardwareProvider(httpProvider);
|
||||
@@ -103,6 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
|
||||
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
|
||||
const monitorStatusPolicy = new MonitorStatusPolicy();
|
||||
const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger);
|
||||
|
||||
// ***********************
|
||||
// Reactors and dispatcher
|
||||
@@ -145,7 +149,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
checksRepository,
|
||||
incidentsRepository,
|
||||
geoChecksRepository,
|
||||
dockerLogsRepository
|
||||
dockerLogsRepository,
|
||||
egressService
|
||||
);
|
||||
|
||||
const worker = await DBQueueWorker.create({
|
||||
@@ -185,5 +190,5 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
|
||||
});
|
||||
}
|
||||
|
||||
return { worker, networkService, bufferService, statusService };
|
||||
return { worker, networkService, bufferService, statusService, egressService };
|
||||
};
|
||||
@@ -6,6 +6,7 @@ export interface ISettingsRepository {
|
||||
findSingleton(): Promise<Settings | null>;
|
||||
// update
|
||||
update(settings: SettingsUpdate): Promise<Settings>;
|
||||
removeEgressNotification(notificationId: string): Promise<void>; // $pull, so it cannot race a concurrent settings update
|
||||
// delete
|
||||
deleteLegacy: () => Promise<boolean>;
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Schema, model, type Types } from "mongoose";
|
||||
import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
|
||||
_id: Types.ObjectId;
|
||||
@@ -42,6 +43,10 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
|
||||
globalThresholds: { type: thresholdsSchema },
|
||||
globalProxyEnabled: { type: Boolean, default: false },
|
||||
globalProxyId: { type: String, default: null },
|
||||
egressCheckEnabled: { type: Boolean, default: false },
|
||||
egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
|
||||
egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
|
||||
egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
@@ -3,6 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
|
||||
import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
|
||||
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
|
||||
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
|
||||
|
||||
class MongoSettingsRepository implements ISettingsRepository {
|
||||
private toEntity = (doc: AppSettingsDocument): Settings => {
|
||||
@@ -30,6 +31,10 @@ class MongoSettingsRepository implements ISettingsRepository {
|
||||
globalThresholds: doc.globalThresholds ?? undefined,
|
||||
globalProxyEnabled: doc.globalProxyEnabled ?? false,
|
||||
globalProxyId: doc.globalProxyId ?? undefined,
|
||||
egressCheckEnabled: doc.egressCheckEnabled ?? false,
|
||||
egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
|
||||
egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
egressNotifications: doc.egressNotifications ?? [],
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
@@ -79,6 +84,10 @@ class MongoSettingsRepository implements ISettingsRepository {
|
||||
return this.toEntity(updatedSettings);
|
||||
};
|
||||
|
||||
removeEgressNotification = async (notificationId: string) => {
|
||||
await AppSettingsModel.updateMany({ egressNotifications: notificationId }, { $pull: { egressNotifications: notificationId } });
|
||||
};
|
||||
|
||||
deleteLegacy = async () => {
|
||||
const res = await AppSettingsModel.deleteMany({ version: { $exists: false } });
|
||||
return res.deletedCount > 0;
|
||||
|
||||
@@ -23,15 +23,27 @@ export interface ISettingsService {
|
||||
getSettings(): EnvConfig;
|
||||
areStatusPageThemesEnabled(): boolean;
|
||||
getDBSettings(): Promise<Settings>;
|
||||
getCachedDBSettings(): Promise<Settings>;
|
||||
updateDbSettings(newSettings: SettingsUpdate): Promise<Settings>;
|
||||
removeEgressNotification(notificationId: string): Promise<void>;
|
||||
}
|
||||
|
||||
const DB_SETTINGS_CACHE_TTL_MS = 60_000;
|
||||
|
||||
type CachedSettings = {
|
||||
value: Promise<Settings>;
|
||||
expiresAt: number;
|
||||
};
|
||||
export class SettingsService implements ISettingsService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
private settings: EnvConfig;
|
||||
private settingsRepository: ISettingsRepository | null = null;
|
||||
private dbSettingsCache: CachedSettings | null = null;
|
||||
|
||||
constructor(env: ValidatedEnv) {
|
||||
constructor(
|
||||
env: ValidatedEnv,
|
||||
private dbSettingsTTLMs: number = DB_SETTINGS_CACHE_TTL_MS
|
||||
) {
|
||||
this.settings = {
|
||||
jwtSecret: env.JWT_SECRET,
|
||||
jwtTTL: env.TOKEN_TTL as StringValue,
|
||||
@@ -75,7 +87,14 @@ export class SettingsService implements ISettingsService {
|
||||
}
|
||||
|
||||
updateDbSettings = async (newSettings: SettingsUpdate) => {
|
||||
return await this.getRepository().update(newSettings);
|
||||
const updated = await this.getRepository().update(newSettings);
|
||||
this.dbSettingsCache = null;
|
||||
return updated;
|
||||
};
|
||||
|
||||
removeEgressNotification = async (notificationId: string) => {
|
||||
await this.getRepository().removeEgressNotification(notificationId);
|
||||
this.dbSettingsCache = null;
|
||||
};
|
||||
|
||||
getDBSettings = async () => {
|
||||
@@ -95,4 +114,17 @@ export class SettingsService implements ISettingsService {
|
||||
|
||||
return settings;
|
||||
};
|
||||
|
||||
getCachedDBSettings = async () => {
|
||||
const hit = this.dbSettingsCache;
|
||||
if (hit && hit.expiresAt > Date.now()) {
|
||||
return hit.value;
|
||||
}
|
||||
const entry: CachedSettings = { value: this.getDBSettings(), expiresAt: Date.now() + this.dbSettingsTTLMs };
|
||||
this.dbSettingsCache = entry;
|
||||
entry.value.catch(() => {
|
||||
if (this.dbSettingsCache === entry) this.dbSettingsCache = null;
|
||||
});
|
||||
return entry.value;
|
||||
};
|
||||
}
|
||||
@@ -51,6 +51,11 @@ export interface Settings {
|
||||
globalThresholds?: SettingsThresholds;
|
||||
globalProxyEnabled: boolean;
|
||||
globalProxyId?: string | null;
|
||||
// Egress self-check (see domain/egress)
|
||||
egressCheckEnabled: boolean;
|
||||
egressCheckTargets: string[];
|
||||
egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
|
||||
egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import type {
|
||||
PageSpeedChecksResult,
|
||||
UptimeChecksResult,
|
||||
} from "@/domain/checks/check.type.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { CheckFilter, DateRange } from "@/types/query.js";
|
||||
|
||||
@@ -43,7 +44,7 @@ export interface IChecksRepository {
|
||||
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult | DockerChecksResult>;
|
||||
findDockerContainerChecks(monitorId: string, containerName: string, dateRange: DateRange): Promise<Omit<DockerContainerStats, "restartsInRange">>;
|
||||
findSummaryByTeamId(teamId: string, dateRange: DateRange): Promise<ChecksSummary>;
|
||||
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
|
||||
findUnevaluatedByMonitorId(monitorId: string, pending: PendingCheck[]): Promise<Check[]>;
|
||||
getDailyStatusBuckets(monitorIds: string[], days: number, timezone: string): Promise<DailyCheckBucket[]>;
|
||||
|
||||
// update
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
getDockerTotalChecks,
|
||||
getDockerUpChecks,
|
||||
} from "@/domain/checks/check.docker.aggregation.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
|
||||
const SERVICE_NAME = "ChecksRepository";
|
||||
|
||||
@@ -360,10 +361,23 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
};
|
||||
};
|
||||
|
||||
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
|
||||
findUnevaluatedByMonitorId = async (monitorId: string, pending: PendingCheck[]) => {
|
||||
if (pending.length === 0) return [];
|
||||
|
||||
const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
|
||||
|
||||
// Bound the query to the pending time range so the monitorId + createdAt index is used
|
||||
let from = Number.POSITIVE_INFINITY;
|
||||
let to = Number.NEGATIVE_INFINITY;
|
||||
for (const entry of pending) {
|
||||
if (entry.createdAt < from) from = entry.createdAt;
|
||||
if (entry.createdAt > to) to = entry.createdAt;
|
||||
}
|
||||
|
||||
const docs = await CheckModel.find({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
createdAt: { $gt: new Date(since) },
|
||||
createdAt: { $gte: new Date(from), $lte: new Date(to) },
|
||||
_id: { $in: ids },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.lean<CheckDocument[]>();
|
||||
|
||||
@@ -12,7 +12,6 @@ const SERVICE_NAME = "checkService";
|
||||
|
||||
export interface ICheckService {
|
||||
createChecks(checks: Check[]): Promise<Check[]>;
|
||||
toLastEvaluatedAt(check: Check): number;
|
||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||
getChecksByMonitor(params: {
|
||||
@@ -55,10 +54,6 @@ export class CheckService implements ICheckService {
|
||||
return this.checksRepository.createChecks(checks);
|
||||
};
|
||||
|
||||
toLastEvaluatedAt = (check: Check) => {
|
||||
return new Date(check.createdAt).getTime();
|
||||
};
|
||||
|
||||
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { Schema, model, type Types } from "mongoose";
|
||||
import { EgressStatuses, type EgressProbeResult, type EgressState } from "@/domain/egress/egress.type.js";
|
||||
|
||||
interface EgressStateDocument extends Omit<EgressState, "id" | "degradedSince" | "lastRecoveredAt" | "lastProbeAt" | "createdAt" | "updatedAt"> {
|
||||
_id: Types.ObjectId;
|
||||
singleton: boolean;
|
||||
degradedSince: Date | null;
|
||||
lastRecoveredAt: Date | null;
|
||||
lastProbeAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
const probeResultSchema = new Schema<EgressProbeResult>(
|
||||
{
|
||||
target: { type: String, required: true },
|
||||
reachable: { type: Boolean, required: true },
|
||||
responseTime: { type: Number, default: 0 },
|
||||
message: { type: String },
|
||||
},
|
||||
{ _id: false }
|
||||
);
|
||||
|
||||
const EgressStateSchema = new Schema<EgressStateDocument>(
|
||||
{
|
||||
singleton: { type: Boolean, required: true, unique: true, default: true },
|
||||
status: { type: String, enum: EgressStatuses, required: true, default: "ok" },
|
||||
degradedSince: { type: Date, default: null },
|
||||
lastRecoveredAt: { type: Date, default: null },
|
||||
lastProbeAt: { type: Date, default: null },
|
||||
lastProbeResults: { type: [probeResultSchema], default: () => [] },
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
const EgressStateModel = model<EgressStateDocument>("EgressState", EgressStateSchema);
|
||||
|
||||
export type { EgressStateDocument };
|
||||
export { EgressStateModel };
|
||||
export default EgressStateModel;
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
|
||||
|
||||
export interface IEgressStateRepository {
|
||||
// read
|
||||
findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call, a plain read after that
|
||||
// update
|
||||
recordProbe(results: EgressProbeResult[], now: Date): Promise<EgressState>;
|
||||
reset(): Promise<EgressState>; // back to "ok" with no episode; used when the feature is toggled
|
||||
// Atomic transitions. Each returns the updated state when this caller performed the transition,
|
||||
// or null when the state was already on the target side (another worker got there first).
|
||||
markDegraded(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
|
||||
markRecovered(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
|
||||
import { EgressStateModel, type EgressStateDocument } from "@/domain/egress/egress-state.model.js";
|
||||
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
|
||||
|
||||
const SERVICE_NAME = "EgressStateRepository";
|
||||
|
||||
class MongoEgressStateRepository implements IEgressStateRepository {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private toEntity = (doc: EgressStateDocument): EgressState => {
|
||||
return {
|
||||
id: toStringId(doc._id),
|
||||
status: doc.status,
|
||||
degradedSince: doc.degradedSince ? toDateString(doc.degradedSince) : null,
|
||||
lastRecoveredAt: doc.lastRecoveredAt ? toDateString(doc.lastRecoveredAt) : null,
|
||||
lastProbeAt: doc.lastProbeAt ? toDateString(doc.lastProbeAt) : null,
|
||||
lastProbeResults: (doc.lastProbeResults ?? []).map((result) => ({
|
||||
target: result.target,
|
||||
reachable: result.reachable,
|
||||
responseTime: result.responseTime ?? 0,
|
||||
...(result.message !== undefined && { message: result.message }),
|
||||
})),
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
};
|
||||
};
|
||||
|
||||
findSingleton = async () => {
|
||||
// Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write.
|
||||
const existing = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
|
||||
if (existing) return this.toEntity(existing);
|
||||
try {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $setOnInsert: { singleton: true, status: "ok" } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
} catch (error: unknown) {
|
||||
// Two processes can race the very first insert; the unique index rejects the loser, who reads the winner's row.
|
||||
if ((error as { code?: number }).code === 11000) {
|
||||
const doc = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
|
||||
if (doc) return this.toEntity(doc);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
reset = async () => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $set: { status: "ok", degradedSince: null, lastProbeAt: null, lastProbeResults: [] }, $setOnInsert: { singleton: true } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
};
|
||||
|
||||
recordProbe = async (results: EgressProbeResult[], now: Date) => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true },
|
||||
{ $set: { lastProbeAt: now, lastProbeResults: results }, $setOnInsert: { singleton: true, status: "ok" } },
|
||||
{ upsert: true, new: true, setDefaultsOnInsert: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return this.toEntity(doc);
|
||||
};
|
||||
|
||||
markDegraded = async (results: EgressProbeResult[], now: Date) => {
|
||||
// Only the caller that flips ok -> degraded sees a document back; everyone else sees null.
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true, status: "ok" },
|
||||
{ $set: { status: "degraded", degradedSince: now, lastProbeAt: now, lastProbeResults: results } },
|
||||
{ new: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return doc ? this.toEntity(doc) : null;
|
||||
};
|
||||
|
||||
markRecovered = async (results: EgressProbeResult[], now: Date) => {
|
||||
const doc = await EgressStateModel.findOneAndUpdate(
|
||||
{ singleton: true, status: "degraded" },
|
||||
{ $set: { status: "ok", lastRecoveredAt: now, lastProbeAt: now, lastProbeResults: results } },
|
||||
{ new: true }
|
||||
).lean<EgressStateDocument>();
|
||||
return doc ? this.toEntity(doc) : null;
|
||||
};
|
||||
}
|
||||
|
||||
export default MongoEgressStateRepository;
|
||||
@@ -0,0 +1,31 @@
|
||||
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import type { EgressState } from "@/domain/egress/egress.type.js";
|
||||
|
||||
export interface IEgressStateService {
|
||||
getState(): Promise<EgressState>;
|
||||
reset(): Promise<EgressState>;
|
||||
}
|
||||
|
||||
const SERVICE_NAME = "EgressStateService";
|
||||
|
||||
// API-process view of the instance-global egress state. Probe-driven transitions are owned by the
|
||||
// worker (egress.service.ts); this exists so the controller never touches the repositories.
|
||||
export class EgressStateService implements IEgressStateService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
constructor(
|
||||
private egressStateRepository: IEgressStateRepository,
|
||||
private jobsRepository: IJobsRepository
|
||||
) {}
|
||||
|
||||
getState = async (): Promise<EgressState> => {
|
||||
return this.egressStateRepository.findSingleton();
|
||||
};
|
||||
|
||||
// Called when the feature is switched on or off: drop any pending recovery job and start from "ok".
|
||||
reset = async (): Promise<EgressState> => {
|
||||
await this.jobsRepository.deleteGlobalJob("egress");
|
||||
return this.egressStateRepository.reset();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
import type { Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
|
||||
import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
|
||||
import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { jobId, type Job, type JobSeed } from "@/domain/jobs/job.type.js";
|
||||
import type { INetworkService } from "@/service/networkService.js";
|
||||
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
|
||||
import type { ILogger } from "@/utils/logger.js";
|
||||
import {
|
||||
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
|
||||
DEFAULT_EGRESS_TARGETS,
|
||||
type EgressAssessment,
|
||||
type EgressProbeResult,
|
||||
type EgressStatus,
|
||||
} from "@/domain/egress/egress.type.js";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
import { parseEgressTarget, type EgressTarget } from "@/utils/egressTarget.js";
|
||||
|
||||
const SERVICE_NAME = "EgressService";
|
||||
const PROBE_TIMEOUT_MS = 5000;
|
||||
// How long a settled assessment answers for later failures. An assessment still in flight is shared
|
||||
// for as long as it takes, so this only bounds how stale a reused answer can be.
|
||||
const ASSESSMENT_REUSE_MS = 5000;
|
||||
// Global job row (refId null) that re-probes while degraded. Inserted when egress degrades, removed on recovery.
|
||||
const RECOVERY_JOB_TYPE = "egress" as const;
|
||||
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
|
||||
|
||||
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
|
||||
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
|
||||
|
||||
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
|
||||
const PROBE_MONITOR_ID = "egress-probe";
|
||||
const PROBE_TEAM_ID = "system";
|
||||
|
||||
export interface IEgressService {
|
||||
probeTargets(targets: string[]): Promise<EgressProbeResult[]>;
|
||||
// Called by the check producer when a check fails. Null when the feature is off or the assessment itself failed.
|
||||
assessAfterFailure(): Promise<EgressStatus | null>;
|
||||
// Handler for the "egress" job: re-probes while degraded and removes the job once egress is back.
|
||||
checkRecovery(job: Job): Promise<void>;
|
||||
}
|
||||
|
||||
type CachedAssessment = {
|
||||
value: Promise<EgressAssessment | null>;
|
||||
expiresAt: number | null; // null while the assessment is still in flight
|
||||
};
|
||||
|
||||
export class EgressService implements IEgressService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
|
||||
private assessment: CachedAssessment | null = null;
|
||||
|
||||
constructor(
|
||||
private settingsService: ISettingsService,
|
||||
private egressStateRepository: IEgressStateRepository,
|
||||
private jobsRepository: IJobsRepository,
|
||||
private networkService: INetworkService,
|
||||
private proxyResolver: IProxyResolver,
|
||||
private logger: ILogger
|
||||
) {}
|
||||
|
||||
// ****************************
|
||||
// Probing
|
||||
// ****************************
|
||||
|
||||
// Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor.
|
||||
// proxyMode "inherit" lets the resolver apply the global proxy to http probes, as it does for monitors.
|
||||
private toProbeMonitor = (target: EgressTarget, raw: string): Monitor => {
|
||||
const base = {
|
||||
id: PROBE_MONITOR_ID,
|
||||
teamId: PROBE_TEAM_ID,
|
||||
name: `Egress probe: ${raw}`,
|
||||
method: "GET",
|
||||
proxyMode: "inherit",
|
||||
useAdvancedMatching: false,
|
||||
ignoreTlsErrors: false,
|
||||
customUpCodes: [],
|
||||
};
|
||||
|
||||
switch (target.kind) {
|
||||
case "http":
|
||||
return { ...base, type: "http", url: target.url } as unknown as Monitor;
|
||||
case "port":
|
||||
return { ...base, type: "port", url: target.host, port: target.port } as unknown as Monitor;
|
||||
case "ping":
|
||||
return { ...base, type: "ping", url: target.host } as unknown as Monitor;
|
||||
}
|
||||
};
|
||||
|
||||
private probeTarget = async (raw: string): Promise<EgressProbeResult> => {
|
||||
const target = parseEgressTarget(raw);
|
||||
if (!target) {
|
||||
// Validation rejects these; a stored value can still be stale, and it must not count as reachable.
|
||||
return { target: raw, reachable: false, responseTime: 0, message: "Invalid egress target" };
|
||||
}
|
||||
|
||||
const monitor = this.toProbeMonitor(target, raw);
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
|
||||
const { response, responseTime, error } = await timeRequest(async () => {
|
||||
const proxyUrl = await this.proxyResolver.resolve(monitor);
|
||||
return Promise.race([
|
||||
this.networkService.requestStatus(monitor, { proxyUrl }),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS);
|
||||
}),
|
||||
]).finally(() => clearTimeout(timer));
|
||||
});
|
||||
|
||||
if (error || !response) {
|
||||
return {
|
||||
target: raw,
|
||||
reachable: false,
|
||||
responseTime,
|
||||
message: error instanceof Error ? error.message : error ? String(error) : "No response",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
target: raw,
|
||||
// Any HTTP response, including 4xx/5xx, proves the instance can reach the target; only a transport
|
||||
// failure counts against egress. Ping and port providers have no such distinction.
|
||||
reachable: response.status === true || (target.kind === "http" && isHttpStatusCode(response.code)),
|
||||
responseTime: response.responseTime ?? responseTime,
|
||||
message: response.message,
|
||||
};
|
||||
};
|
||||
|
||||
probeTargets = async (targets: string[]): Promise<EgressProbeResult[]> => {
|
||||
return await Promise.all(targets.map((target) => this.probeTarget(target)));
|
||||
};
|
||||
|
||||
private resolveTargets = (configured: string[] | undefined): string[] => {
|
||||
const targets = (configured ?? []).map((target) => target.trim()).filter((target) => target.length > 0);
|
||||
// An empty list would make "all unreachable" vacuously true, so fall back to the defaults.
|
||||
return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
|
||||
};
|
||||
|
||||
private toIntervalMs = (seconds: number | undefined): number => {
|
||||
const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
|
||||
return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
|
||||
};
|
||||
|
||||
// ****************************
|
||||
// Event-triggered assessment
|
||||
// ****************************
|
||||
|
||||
assessAfterFailure = async (): Promise<EgressStatus | null> => {
|
||||
const assessment = await this.sharedAssessment();
|
||||
return assessment === "disabled" ? null : assessment;
|
||||
};
|
||||
|
||||
// When egress is down every monitor fails at once. Concurrent failures share the in-flight assessment
|
||||
// however long it takes, and a settled one answers for ASSESSMENT_REUSE_MS. Same shape as ProxyResolver's cache.
|
||||
private sharedAssessment = (): Promise<EgressAssessment | null> => {
|
||||
const cached = this.assessment;
|
||||
if (cached && (cached.expiresAt === null || cached.expiresAt > Date.now())) {
|
||||
return cached.value;
|
||||
}
|
||||
|
||||
const entry: CachedAssessment = { value: this.assess(), expiresAt: null };
|
||||
this.assessment = entry;
|
||||
void entry.value.then((result) => {
|
||||
if (this.assessment !== entry) return;
|
||||
if (result === null) {
|
||||
this.assessment = null; // an internal failure is not cached, so the next failing check retries
|
||||
} else {
|
||||
entry.expiresAt = Date.now() + ASSESSMENT_REUSE_MS;
|
||||
}
|
||||
});
|
||||
return entry.value;
|
||||
};
|
||||
|
||||
private assess = async (): Promise<EgressAssessment | null> => {
|
||||
try {
|
||||
const settings = await this.settingsService.getCachedDBSettings();
|
||||
if (!settings.egressCheckEnabled) {
|
||||
return "disabled";
|
||||
}
|
||||
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (state.status === "degraded") {
|
||||
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
|
||||
return "degraded";
|
||||
}
|
||||
|
||||
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
|
||||
const now = new Date();
|
||||
|
||||
if (results.some((result) => result.reachable)) {
|
||||
await this.egressStateRepository.recordProbe(results, now);
|
||||
return "ok";
|
||||
}
|
||||
|
||||
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
|
||||
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
|
||||
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
|
||||
const degraded = await this.egressStateRepository.markDegraded(results, now);
|
||||
this.logger.warn({
|
||||
message: degraded
|
||||
? "Instance egress degraded: every reliability target is unreachable. Monitor failures will not be counted until it recovers"
|
||||
: "Instance egress degraded (transition already recorded by another worker)",
|
||||
service: SERVICE_NAME,
|
||||
method: "assessAfterFailure",
|
||||
details: { results },
|
||||
});
|
||||
return "degraded";
|
||||
} catch (error: unknown) {
|
||||
// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
service: SERVICE_NAME,
|
||||
method: "assessAfterFailure",
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
// ****************************
|
||||
// Recovery job
|
||||
// ****************************
|
||||
|
||||
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
|
||||
// so re-arming while degraded never pushes a pending run back.
|
||||
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
|
||||
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
|
||||
const seed: JobSeed = {
|
||||
id: RECOVERY_JOB_ID,
|
||||
type: RECOVERY_JOB_TYPE,
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: Date.now() + intervalMs,
|
||||
intervalMs,
|
||||
};
|
||||
if (reschedule) {
|
||||
await this.jobsRepository.upsertCleanupJob(seed); // global-row upsert that also sets the schedule
|
||||
} else {
|
||||
await this.jobsRepository.upsertJob(seed);
|
||||
}
|
||||
};
|
||||
|
||||
// Removes the row only if nobody has rescheduled it since this run claimed it. Another worker may have seen
|
||||
// egress fail again in the meantime and started a new episode; its schedule must survive.
|
||||
private releaseRecoveryJob = async (job: Job) => {
|
||||
await this.jobsRepository.deleteGlobalJobIfUnchanged(RECOVERY_JOB_TYPE, job.nextScheduledAt);
|
||||
};
|
||||
|
||||
// Runs on the queue at the configured interval while degraded. Errors propagate so the queue records
|
||||
// the failure and retries with its usual backoff.
|
||||
checkRecovery = async (job: Job): Promise<void> => {
|
||||
const settings = await this.settingsService.getCachedDBSettings();
|
||||
const state = await this.egressStateRepository.findSingleton();
|
||||
if (!settings.egressCheckEnabled || state.status !== "degraded") {
|
||||
// Feature switched off, recovery recorded by another worker, or state reset: nothing left to poll for.
|
||||
await this.releaseRecoveryJob(job);
|
||||
return;
|
||||
}
|
||||
|
||||
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
|
||||
const now = new Date();
|
||||
|
||||
if (!results.some((result) => result.reachable)) {
|
||||
await this.egressStateRepository.recordProbe(results, now);
|
||||
return;
|
||||
}
|
||||
|
||||
const recovered = await this.egressStateRepository.markRecovered(results, now);
|
||||
await this.releaseRecoveryJob(job);
|
||||
if (!recovered) {
|
||||
// Another process performed the transition.
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.info({
|
||||
message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`,
|
||||
service: SERVICE_NAME,
|
||||
method: "checkRecovery",
|
||||
details: { results },
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Egress self-check: detects loss of the instance's own outbound connectivity so that
|
||||
// a burst of simultaneous monitor failures is not reported as a burst of real outages.
|
||||
|
||||
export const EgressStatuses = ["ok", "degraded"] as const;
|
||||
export type EgressStatus = (typeof EgressStatuses)[number];
|
||||
// Outcome of one assessment: a status, or "disabled" when the feature is switched off (distinct from an internal failure).
|
||||
export type EgressAssessment = EgressStatus | "disabled";
|
||||
|
||||
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
|
||||
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
|
||||
export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
|
||||
export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
|
||||
export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
|
||||
export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
|
||||
export const MAX_EGRESS_TARGETS = 10;
|
||||
|
||||
// Instance-global singleton persisted in MongoDB so that every worker process and the API
|
||||
// process share one view of the state (the queue worker may run separately from the API).
|
||||
export interface EgressState {
|
||||
id: string;
|
||||
status: EgressStatus;
|
||||
degradedSince: string | null; // ISO timestamp of the current or most recent degraded entry
|
||||
lastRecoveredAt: string | null; // ISO timestamp of the most recent recovery
|
||||
lastProbeAt: string | null; // ISO timestamp of the most recent reliability probe
|
||||
lastProbeResults: EgressProbeResult[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface EgressProbeResult {
|
||||
target: string;
|
||||
reachable: boolean;
|
||||
responseTime: number; // ms
|
||||
message?: string;
|
||||
}
|
||||
@@ -47,6 +47,18 @@ const JobSchema = new Schema<JobDocument>(
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Pending checks, inserted but not yet evaluated
|
||||
pendingChecks: {
|
||||
type: [
|
||||
{
|
||||
checkId: { type: String, required: true },
|
||||
createdAt: { type: Number, required: true },
|
||||
},
|
||||
],
|
||||
default: [],
|
||||
_id: false,
|
||||
},
|
||||
|
||||
// Observability
|
||||
runCount: {
|
||||
type: Number,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
|
||||
import { Job, JobSeed, JobType, PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export type JobPageQuery = {
|
||||
page?: number;
|
||||
@@ -32,9 +32,12 @@ export interface IJobsRepository {
|
||||
recordOneShot(id: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
|
||||
// Hand off to evaluator // This is accomplished pushing newly inserted Check IDs onto the evaluation row
|
||||
// ********************
|
||||
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
|
||||
upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
|
||||
|
||||
// Removes processed checks from the evaluate row. Returns false when this worker no longer holds the lease.
|
||||
pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Job CRUD
|
||||
@@ -61,6 +64,11 @@ export interface IJobsRepository {
|
||||
// Delete a single job of type
|
||||
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
|
||||
|
||||
// Global rows (refId null) are addressed by their canonical _id, see jobId
|
||||
deleteGlobalJob(type: JobType): Promise<boolean>;
|
||||
// Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged)
|
||||
deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise<boolean>;
|
||||
|
||||
// Delete jobs that reference monitors that no longer exist
|
||||
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
|
||||
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId, PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
|
||||
// Cap for rearm interval, never want to wait more than 15s.
|
||||
const REARM_JITTER_MAX_MS = 15_000;
|
||||
@@ -18,6 +18,7 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
intervalMs: doc.intervalMs,
|
||||
lockedBy: doc.lockedBy,
|
||||
lockedUntil: doc.lockedUntil,
|
||||
pendingChecks: doc.pendingChecks ?? [],
|
||||
runCount: doc.runCount,
|
||||
failCount: doc.failCount,
|
||||
lastFinishedAt: doc.lastFinishedAt,
|
||||
@@ -107,6 +108,8 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
failCount: 0,
|
||||
lastFailReason: null,
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
@@ -144,22 +147,27 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: PARKED,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
[
|
||||
{
|
||||
$set: {
|
||||
// Re-arm if more checks were pushed while this run held the lease; otherwise park
|
||||
nextScheduledAt: {
|
||||
$cond: [{ $gt: [{ $size: { $ifNull: ["$pendingChecks", []] } }, 0] }, now, PARKED],
|
||||
},
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
failCount: 0,
|
||||
lastFailReason: null,
|
||||
runCount: { $add: ["$runCount", 1] },
|
||||
},
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
]
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
upsertEvaluate = async (monitorId: string, now: number) => {
|
||||
upsertEvaluate = async (monitorId: string, pending: PendingCheck[], now: number) => {
|
||||
const filter = { type: "evaluate", refId: monitorId };
|
||||
const update = {
|
||||
$setOnInsert: {
|
||||
@@ -177,6 +185,9 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
$min: {
|
||||
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
||||
},
|
||||
$addToSet: {
|
||||
pendingChecks: { $each: pending }, // accumulate pending checks; a retried arm never duplicates an entry
|
||||
},
|
||||
};
|
||||
try {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
@@ -191,6 +202,18 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
// Returns whether this worker still holds the lease. false means another worker has claimed the row and the caller must stop.
|
||||
pullEvaluated = async (id: string, checkIds: string[]) => {
|
||||
if (checkIds.length === 0) return true;
|
||||
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
|
||||
{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
|
||||
);
|
||||
return res.matchedCount === 1;
|
||||
};
|
||||
|
||||
upsertJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
@@ -278,6 +301,16 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteGlobalJob = async (type: JobType) => {
|
||||
const res = await JobModel.deleteOne({ _id: jobId(type, null) });
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteGlobalJobIfUnchanged = async (type: JobType, nextScheduledAt: number) => {
|
||||
const res = await JobModel.deleteOne({ _id: jobId(type, null), nextScheduledAt });
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
|
||||
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
|
||||
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
|
||||
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"] as const;
|
||||
export type JobType = (typeof JOB_TYPES)[number];
|
||||
export const LOCK_MS = 60_000;
|
||||
export const BACKOFF_MS = 5_000;
|
||||
export const PARKED = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
|
||||
export type PendingCheck = {
|
||||
checkId: string;
|
||||
createdAt: number; // epoch ms, copied from Check.createdAt
|
||||
};
|
||||
|
||||
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
|
||||
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
|
||||
|
||||
export type Job = {
|
||||
@@ -21,6 +26,9 @@ export type Job = {
|
||||
lockedBy: string | null; // workerId holding the lease
|
||||
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
|
||||
|
||||
// Evaluate these rows only. These are checks that are inserted but not yet evaluated
|
||||
pendingChecks: PendingCheck[];
|
||||
|
||||
// observability (replaces what getMetrics/getJobs)
|
||||
runCount: number;
|
||||
failCount: number;
|
||||
|
||||
@@ -259,7 +259,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
||||
const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature];
|
||||
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
|
||||
const threshold = monitor.tempAlertThreshold;
|
||||
if (maxTemp >= threshold) {
|
||||
if (maxTemp > threshold) {
|
||||
breaches.push({
|
||||
metric: "temp",
|
||||
currentValue: maxTemp,
|
||||
|
||||
@@ -179,6 +179,9 @@ export class NotificationsService implements INotificationsService {
|
||||
deleteById = async (id: string, teamId: string): Promise<Notification> => {
|
||||
await this.monitorsRepository.removeNotificationFromMonitors(id);
|
||||
const deleted = await this.notificationsRepository.deleteById(id, teamId);
|
||||
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles.
|
||||
// After the team-scoped delete, so a request for another team's id (404 above) cannot touch instance settings.
|
||||
await this.settingsService.removeEgressNotification(id);
|
||||
return deleted;
|
||||
};
|
||||
}
|
||||
@@ -7,12 +7,14 @@ import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"
|
||||
import { ICheckService } from "../domain/checks/check.service.js";
|
||||
import { DockerLog } from "@/domain/docker/docker-log.type.js";
|
||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
const SERVICE_NAME = "BufferService";
|
||||
|
||||
export interface IBufferService {
|
||||
addToBuffer(check: Check): void;
|
||||
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
|
||||
addDockerLogToBuffer(dockerLog: DockerLog): void;
|
||||
ingestChecks(checks: Check[]): Promise<void>;
|
||||
scheduleNextFlush(): void;
|
||||
flushBuffer(): Promise<void>;
|
||||
flushGeoBuffer(): Promise<void>;
|
||||
@@ -34,6 +36,9 @@ export class BufferService implements IBufferService {
|
||||
private dockerLogsService: IDockerLogsService;
|
||||
private jobsRepository: IJobsRepository;
|
||||
|
||||
// Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated.
|
||||
private unarmed: Map<string, PendingCheck[]> = new Map();
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
checkService: ICheckService,
|
||||
@@ -99,6 +104,38 @@ export class BufferService implements IBufferService {
|
||||
}
|
||||
}
|
||||
|
||||
ingestChecks = async (checks: Check[]) => {
|
||||
if (checks.length > 0) await this.checksService.createChecks(checks);
|
||||
|
||||
// Group by monitor, starting from any entries a previous flush failed to arm
|
||||
const byMonitor = this.unarmed;
|
||||
this.unarmed = new Map();
|
||||
for (const check of checks) {
|
||||
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
|
||||
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
|
||||
byMonitor.set(check.metadata.monitorId, pending);
|
||||
}
|
||||
if (byMonitor.size === 0) return;
|
||||
|
||||
const now = Date.now();
|
||||
const ops: Promise<boolean>[] = [];
|
||||
for (const [monitorId, pendingChecks] of byMonitor) {
|
||||
ops.push(
|
||||
this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
|
||||
// The checks are already stored, so a failed arm is retried on the next flush rather than dropped
|
||||
this.unarmed.set(monitorId, pendingChecks);
|
||||
this.logger.error({
|
||||
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "ingestChecks",
|
||||
});
|
||||
return false;
|
||||
})
|
||||
);
|
||||
}
|
||||
await Promise.all(ops);
|
||||
};
|
||||
|
||||
scheduleNextFlush() {
|
||||
if (this.bufferTimer) {
|
||||
clearTimeout(this.bufferTimer);
|
||||
@@ -121,8 +158,9 @@ export class BufferService implements IBufferService {
|
||||
}
|
||||
}, this.BUFFER_TIMEOUT);
|
||||
}
|
||||
|
||||
async flushBuffer() {
|
||||
if (this.buffer.length === 0) {
|
||||
if (this.buffer.length === 0 && this.unarmed.size === 0) {
|
||||
return;
|
||||
}
|
||||
// Take the batch first so a write that drains it can't be appended to mid-flush
|
||||
@@ -134,11 +172,7 @@ export class BufferService implements IBufferService {
|
||||
service: this.SERVICE_NAME,
|
||||
method: "flushBuffer",
|
||||
});
|
||||
await this.checksService.createChecks(batch);
|
||||
// Need to evaluate checks when they are flushed
|
||||
const monitorIds = [...new Set(batch.map((check) => check.metadata.monitorId))];
|
||||
const now = Date.now();
|
||||
await Promise.all(monitorIds.map((monitorId) => this.jobsRepository.upsertEvaluate(monitorId, now)));
|
||||
await this.ingestChecks(batch);
|
||||
} catch (error: unknown) {
|
||||
this.logger.error({
|
||||
message: error instanceof Error ? error.message : "Unknown error",
|
||||
|
||||
@@ -22,7 +22,7 @@ export class PageSpeedProvider implements IStatusProvider<PageSpeedStatusPayload
|
||||
const { url, strategy } = monitor;
|
||||
try {
|
||||
if (!url) throw new Error("URL is required for PageSpeed monitor");
|
||||
const dbSettings = await this.settingsService.getDBSettings();
|
||||
const dbSettings = await this.settingsService.getCachedDBSettings();
|
||||
const apiKey = dbSettings?.pagespeedApiKey;
|
||||
const resolvedStrategy = strategy ?? DefaultPageSpeedStrategy;
|
||||
let pageSpeedUrl = `https://pagespeedonline.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(
|
||||
|
||||
@@ -4,25 +4,33 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js";
|
||||
import { MonitorStatusResponse } from "@/types/network.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import ping from "ping";
|
||||
import * as net from "net";
|
||||
import { timeRequest } from "@/service/network/utils.js";
|
||||
const SERVICE_NAME = "PingProvider";
|
||||
|
||||
type Ping = typeof ping;
|
||||
type NetType = typeof net;
|
||||
|
||||
export class PingProvider implements IStatusProvider<PingStatusPayload> {
|
||||
readonly type = "ping";
|
||||
|
||||
constructor(private ping: Ping) {}
|
||||
constructor(
|
||||
private ping: Ping,
|
||||
private net: NetType
|
||||
) {}
|
||||
|
||||
supports(type: MonitorType): boolean {
|
||||
return type === "ping";
|
||||
}
|
||||
|
||||
private sanitizeHost(url: string): string {
|
||||
return url
|
||||
.replace(/^https?:\/\//, "")
|
||||
.replace(/\/.*$/, "")
|
||||
.replace(/:.*/, "");
|
||||
const host = url.replace(/^https?:\/\//, "").replace(/\/.*$/, "");
|
||||
// "[2001:db8::1]" or "[2001:db8::1]:53": the address is inside the brackets
|
||||
const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host);
|
||||
if (bracketed) return bracketed[1]!;
|
||||
// A bare IPv6 address contains colons that are not a port separator
|
||||
if (this.net.isIPv6(host)) return host;
|
||||
return host.replace(/:.*/, "");
|
||||
}
|
||||
|
||||
async handle(monitor: Monitor): Promise<MonitorStatusResponse<PingStatusPayload>> {
|
||||
|
||||
@@ -73,7 +73,7 @@ export class ProxyResolver implements IProxyResolver {
|
||||
};
|
||||
|
||||
private resolveInherit = async (monitor: Monitor): Promise<string | undefined> => {
|
||||
const settings = await this.getCached("settings", () => this.settingsService.getDBSettings());
|
||||
const settings = await this.settingsService.getCachedDBSettings();
|
||||
if (!settings.globalProxyEnabled) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import { z } from "zod";
|
||||
import { dnsHostnameRegex } from "@/api/validation/shared.js";
|
||||
|
||||
// A reliability target for the egress self-check, in one of three shapes. Parsed here so the
|
||||
// settings validator and the probe agree on exactly what is accepted.
|
||||
export type EgressTarget =
|
||||
| { kind: "http"; url: string } // http(s):// URL, fetched
|
||||
| { kind: "port"; host: string; port: number } // host:port or [ipv6]:port, TCP connect
|
||||
| { kind: "ping"; host: string }; // bare hostname or IP address, ICMP
|
||||
|
||||
// "[ipv6]:port" or "host:port" where host has no colons, so a bare IPv6 address is never read as host:port.
|
||||
const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s[\]]+)):(\d{1,5})$/;
|
||||
const BRACKETED_PATTERN = /^\[([^\]]+)\]$/;
|
||||
|
||||
const isIpv6 = (value: string): boolean => z.ipv6().safeParse(value).success;
|
||||
const isHostOrIpv4 = (value: string): boolean => value === "localhost" || z.ipv4().safeParse(value).success || dnsHostnameRegex.test(value);
|
||||
|
||||
const parseHttp = (target: string): EgressTarget | null => {
|
||||
try {
|
||||
const { protocol, hostname } = new URL(target);
|
||||
return (protocol === "http:" || protocol === "https:") && hostname.length > 0 ? { kind: "http", url: target } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const parseEgressTarget = (raw: string): EgressTarget | null => {
|
||||
const target = raw.trim();
|
||||
if (target.length === 0) return null;
|
||||
|
||||
if (/^https?:\/\//i.test(target)) return parseHttp(target);
|
||||
|
||||
const hostPort = HOST_PORT_PATTERN.exec(target);
|
||||
if (hostPort) {
|
||||
const port = Number(hostPort[3]);
|
||||
if (port < 1 || port > 65535) return null;
|
||||
const bracketedHost = hostPort[1];
|
||||
if (bracketedHost !== undefined) return isIpv6(bracketedHost) ? { kind: "port", host: bracketedHost, port } : null;
|
||||
const host = hostPort[2]!;
|
||||
return isHostOrIpv4(host) ? { kind: "port", host, port } : null;
|
||||
}
|
||||
|
||||
const bracketed = BRACKETED_PATTERN.exec(target);
|
||||
if (bracketed) return isIpv6(bracketed[1]!) ? { kind: "ping", host: bracketed[1]! } : null;
|
||||
|
||||
if (isIpv6(target) || isHostOrIpv4(target)) return { kind: "ping", host: target };
|
||||
return null;
|
||||
};
|
||||
@@ -28,6 +28,7 @@ const CONCURRENCY: Record<JobType, number> = {
|
||||
evaluate: 20,
|
||||
"cleanup-orphaned": 1,
|
||||
"cleanup-retention": 1,
|
||||
egress: 1,
|
||||
};
|
||||
|
||||
export interface DBQueueWorkerDependencies {
|
||||
@@ -61,6 +62,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
evaluate: 0,
|
||||
"cleanup-orphaned": 0,
|
||||
"cleanup-retention": 0,
|
||||
egress: 0,
|
||||
};
|
||||
|
||||
private checksRepository: IChecksRepository;
|
||||
@@ -118,19 +120,41 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
// ********************
|
||||
|
||||
private runEvaluate = async (job: Job) => {
|
||||
if (!job.refId) return;
|
||||
if (!job.refId || job.pendingChecks.length === 0) return;
|
||||
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
|
||||
|
||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||
if (!monitor) return;
|
||||
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
|
||||
if (!monitor) {
|
||||
await this.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
|
||||
return;
|
||||
}
|
||||
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
|
||||
|
||||
let current = monitor;
|
||||
for (const check of checks) {
|
||||
const status = this.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
|
||||
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
|
||||
const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
if (!leaseHeld) {
|
||||
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
|
||||
this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" });
|
||||
return;
|
||||
}
|
||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||
}
|
||||
|
||||
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
|
||||
const found = new Set(checks.map((check) => check.id));
|
||||
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
|
||||
if (missing.length > 0) {
|
||||
this.logger.warn({
|
||||
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runEvaluate",
|
||||
});
|
||||
await this.jobsRepository.pullEvaluated(job.id, missing);
|
||||
}
|
||||
};
|
||||
|
||||
private runGeoCheck = async (job: Job) => {
|
||||
@@ -175,6 +199,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
case "cleanup-retention":
|
||||
await this.helper.getCleanupRetentionJob()(); // Get job and execute
|
||||
break;
|
||||
case "egress":
|
||||
await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
|
||||
break;
|
||||
}
|
||||
|
||||
if (job.intervalMs === null) {
|
||||
|
||||
@@ -11,10 +11,13 @@ import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
|
||||
import { ILogger } from "@/utils/logger.js";
|
||||
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
|
||||
import { IEgressService } from "@/domain/egress/egress.service.js";
|
||||
import type { Job } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export interface IWorkerHelper {
|
||||
getCleanupOrphanedJob(): () => Promise<void>;
|
||||
getCleanupRetentionJob(): () => Promise<void>;
|
||||
getEgressRecoveryJob(): (job: Job) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface MonitorActionDecision {
|
||||
@@ -45,6 +48,7 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
private incidentsRepository: IIncidentsRepository;
|
||||
private geoChecksRepository: IGeoChecksRepository;
|
||||
private dockerLogsRepository: IDockerLogsRepository;
|
||||
private egressService: IEgressService;
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
@@ -57,7 +61,8 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
checksRepository: IChecksRepository,
|
||||
incidentsRepository: IIncidentsRepository,
|
||||
geoChecksRepository: IGeoChecksRepository,
|
||||
dockerLogsRepository: IDockerLogsRepository
|
||||
dockerLogsRepository: IDockerLogsRepository,
|
||||
egressService: IEgressService
|
||||
) {
|
||||
this.logger = logger;
|
||||
this.checkService = checkService;
|
||||
@@ -70,8 +75,17 @@ export class WorkerHelper implements IWorkerHelper {
|
||||
this.incidentsRepository = incidentsRepository;
|
||||
this.geoChecksRepository = geoChecksRepository;
|
||||
this.dockerLogsRepository = dockerLogsRepository;
|
||||
this.egressService = egressService;
|
||||
}
|
||||
|
||||
// Runs while instance egress is degraded; the service removes the row once egress is back.
|
||||
// Errors propagate so the queue records the failure and retries with its usual backoff.
|
||||
getEgressRecoveryJob = () => {
|
||||
return async (job: Job) => {
|
||||
await this.egressService.checkRecovery(job);
|
||||
};
|
||||
};
|
||||
|
||||
getCleanupOrphanedJob = () => {
|
||||
return async () => {
|
||||
try {
|
||||
|
||||
@@ -41,6 +41,7 @@ export class JobScheduler implements IJobScheduler {
|
||||
evaluate: POLL_MS,
|
||||
"cleanup-orphaned": POLL_MS,
|
||||
"cleanup-retention": POLL_MS,
|
||||
egress: POLL_MS,
|
||||
};
|
||||
// Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd.
|
||||
protected toCheckJob = (monitor: MonitorScheduleFields, now: number, immediate = false): JobSeed => ({
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoEgressStateRepository from "../../src/domain/egress/egress-state.repository.mongo.ts";
|
||||
import { EgressStateModel } from "../../src/domain/egress/egress-state.model.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The transitions here are the guarantees several worker processes rely on: exactly one
|
||||
// caller flips ok -> degraded (and degraded -> ok), and a plain read never writes. Both are
|
||||
// MongoDB filter/update semantics, so they need a live engine.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await EgressStateModel.init(); // unique index on singleton
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await EgressStateModel.deleteMany({});
|
||||
});
|
||||
|
||||
const probe = (target: string, reachable: boolean) => ({ target, reachable, responseTime: 5, message: reachable ? "Success" : "timeout" });
|
||||
const NOW = new Date("2026-01-01T10:00:00.000Z");
|
||||
const LATER = new Date("2026-01-01T10:05:00.000Z");
|
||||
|
||||
describe("MongoEgressStateRepository", () => {
|
||||
const repo = new MongoEgressStateRepository();
|
||||
|
||||
describe("findSingleton", () => {
|
||||
it("creates the ok singleton on first call and returns the same document afterwards", async () => {
|
||||
const first = await repo.findSingleton();
|
||||
const second = await repo.findSingleton();
|
||||
|
||||
expect(first.status).toBe("ok");
|
||||
expect(second.id).toBe(first.id);
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
|
||||
it("does not bump updatedAt on a read", async () => {
|
||||
const created = await repo.findSingleton();
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
const read = await repo.findSingleton();
|
||||
|
||||
expect(read.updatedAt).toBe(created.updatedAt);
|
||||
});
|
||||
|
||||
it("survives two processes racing the first insert", async () => {
|
||||
const [a, b] = await Promise.all([repo.findSingleton(), repo.findSingleton()]);
|
||||
|
||||
expect(a.id).toBe(b.id);
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("markDegraded / markRecovered", () => {
|
||||
it("markDegraded does nothing when the singleton has never been created (callers read it first)", async () => {
|
||||
expect(await repo.markDegraded([probe("1.1.1.1", false)], NOW)).toBeNull();
|
||||
expect(await EgressStateModel.countDocuments()).toBe(0);
|
||||
});
|
||||
|
||||
it("lets exactly one caller perform each transition", async () => {
|
||||
await repo.findSingleton();
|
||||
|
||||
const [d1, d2] = await Promise.all([repo.markDegraded([probe("1.1.1.1", false)], NOW), repo.markDegraded([probe("1.1.1.1", false)], NOW)]);
|
||||
expect([d1, d2].filter((result) => result !== null)).toHaveLength(1);
|
||||
expect((await repo.findSingleton()).status).toBe("degraded");
|
||||
|
||||
const [r1, r2] = await Promise.all([repo.markRecovered([probe("1.1.1.1", true)], LATER), repo.markRecovered([probe("1.1.1.1", true)], LATER)]);
|
||||
expect([r1, r2].filter((result) => result !== null)).toHaveLength(1);
|
||||
const state = await repo.findSingleton();
|
||||
expect(state.status).toBe("ok");
|
||||
expect(state.degradedSince).toBe(NOW.toISOString());
|
||||
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
|
||||
});
|
||||
|
||||
it("markRecovered is a no-op while ok and markDegraded is a no-op while degraded", async () => {
|
||||
await repo.findSingleton();
|
||||
expect(await repo.markRecovered([probe("1.1.1.1", true)], NOW)).toBeNull();
|
||||
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
expect(await repo.markDegraded([probe("1.1.1.1", false)], LATER)).toBeNull();
|
||||
expect((await repo.findSingleton()).degradedSince).toBe(NOW.toISOString());
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordProbe", () => {
|
||||
it("stores the probe without changing the status", async () => {
|
||||
await repo.findSingleton();
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
|
||||
const state = await repo.recordProbe([probe("1.1.1.1", false), probe("8.8.8.8", false)], LATER);
|
||||
|
||||
expect(state.status).toBe("degraded");
|
||||
expect(state.lastProbeAt).toBe(LATER.toISOString());
|
||||
expect(state.lastProbeResults.map((result) => result.target)).toEqual(["1.1.1.1", "8.8.8.8"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reset", () => {
|
||||
it("returns a degraded singleton to ok with no episode and keeps the recovery history", async () => {
|
||||
await repo.findSingleton();
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
|
||||
await repo.markRecovered([probe("1.1.1.1", true)], LATER);
|
||||
await repo.markDegraded([probe("1.1.1.1", false)], LATER);
|
||||
|
||||
const state = await repo.reset();
|
||||
|
||||
expect(state.status).toBe("ok");
|
||||
expect(state.degradedSince).toBeNull();
|
||||
expect(state.lastProbeAt).toBeNull();
|
||||
expect(state.lastProbeResults).toEqual([]);
|
||||
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
|
||||
});
|
||||
|
||||
it("creates the singleton when none exists", async () => {
|
||||
const state = await repo.reset();
|
||||
|
||||
expect(state.status).toBe("ok");
|
||||
expect(await EgressStateModel.countDocuments()).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,200 @@
|
||||
import { describe, expect, it, jest, beforeAll, afterAll, beforeEach, afterEach } from "@jest/globals";
|
||||
import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
|
||||
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
||||
import { CheckModel } from "../../src/domain/checks/check.model.ts";
|
||||
import JobModel from "../../src/domain/jobs/job.model.ts";
|
||||
import { CheckService } from "../../src/domain/checks/check.service.ts";
|
||||
import { BufferService } from "../../src/service/bufferService.ts";
|
||||
import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts";
|
||||
import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts";
|
||||
import { createMockLogger } from "../helpers/createMockLogger.ts";
|
||||
import type { Job } from "../../src/domain/jobs/job.type.ts";
|
||||
import type { Check } from "../../src/domain/checks/check.type.ts";
|
||||
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
|
||||
import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||
import type { ILogger } from "../../src/utils/logger.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// Regression test for the dropped-check bug in the check-to-evaluate handoff.
|
||||
//
|
||||
// The evaluate step used to be driven by a timestamp cursor: the evaluator read checks with
|
||||
// createdAt strictly after the monitor's lastEvaluatedAt and advanced that cursor to each
|
||||
// check's createdAt. Each worker's BufferService flushes its own in-memory batch on its own
|
||||
// timer, so with two processing workers a later check produced on worker B could be flushed and
|
||||
// evaluated before an earlier check produced on worker A had been flushed. When A's batch landed,
|
||||
// its check was older than the cursor and the evaluate query never returned it. The check existed
|
||||
// in the collection but never reached the status window, stats, incidents or notifications.
|
||||
//
|
||||
// The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each
|
||||
// is applied, so a late batch is always evaluated. This drives the real time-series query, the
|
||||
// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an
|
||||
// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's.
|
||||
|
||||
let mongod: MongoMemoryServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
mongod = await MongoMemoryServer.create();
|
||||
await mongoose.connect(mongod.getUri());
|
||||
await CheckModel.createCollection(); // timeseries collections must exist before insert
|
||||
await JobModel.init();
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await mongoose.disconnect();
|
||||
await mongod.stop();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await CheckModel.deleteMany({});
|
||||
await JobModel.deleteMany({});
|
||||
});
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const MONITOR_ID = new mongoose.Types.ObjectId().toString();
|
||||
const TEAM_ID = new mongoose.Types.ObjectId().toString();
|
||||
const T0 = Date.parse("2026-01-01T00:00:00.000Z");
|
||||
const INTERVAL_MS = 30_000;
|
||||
|
||||
const makeMonitor = (): Monitor =>
|
||||
({
|
||||
id: MONITOR_ID,
|
||||
teamId: TEAM_ID,
|
||||
type: "http",
|
||||
interval: INTERVAL_MS,
|
||||
status: "up",
|
||||
isActive: true,
|
||||
statusWindow: [],
|
||||
statusWindowSize: 5,
|
||||
statusWindowThreshold: 60,
|
||||
recentChecks: [],
|
||||
lastEvaluatedAt: 0,
|
||||
}) as unknown as Monitor;
|
||||
|
||||
const makeStatusResponse = (): MonitorStatusResponse => ({
|
||||
monitorId: MONITOR_ID,
|
||||
teamId: TEAM_ID,
|
||||
type: "http",
|
||||
status: true,
|
||||
code: 200,
|
||||
message: "OK",
|
||||
responseTime: 100,
|
||||
});
|
||||
|
||||
// Production stamps createdAt at the moment the probe ran, on the producing worker; pin it so
|
||||
// the two workers' checks have a known order regardless of wall clock.
|
||||
const producedAt = (check: Check, at: number): Check => {
|
||||
const createdAt = new Date(at).toISOString();
|
||||
return { ...check, createdAt, updatedAt: createdAt };
|
||||
};
|
||||
|
||||
const stubSettingsService = { getSettings: () => ({ nodeEnv: "production" }) };
|
||||
const stubGeoChecksService = { createGeoChecks: jest.fn<any>().mockResolvedValue([]) };
|
||||
const stubDockerLogsService = { createDockerLogs: jest.fn<any>().mockResolvedValue(0) };
|
||||
|
||||
describe("Evaluate cursor with more than one processing worker", () => {
|
||||
let logger: ILogger;
|
||||
let monitorsRepository: InMemoryMonitorsRepository;
|
||||
let checksRepository: MongoChecksRepository;
|
||||
let checkService: CheckService;
|
||||
let bufferA: BufferService;
|
||||
let bufferB: BufferService;
|
||||
let evaluatedCheckIds: string[];
|
||||
let evaluator: DBQueueWorker;
|
||||
let evaluatorJobsRepository: MongoJobsRepository;
|
||||
|
||||
// One BufferService per worker process, each flushing into the same collection and jobs table.
|
||||
const createWorkerBuffer = (workerId: string) =>
|
||||
new BufferService(
|
||||
logger,
|
||||
checkService,
|
||||
stubGeoChecksService as any,
|
||||
stubDockerLogsService as any,
|
||||
stubSettingsService as any,
|
||||
new MongoJobsRepository(workerId)
|
||||
);
|
||||
|
||||
// Claim every due evaluate job and run it through the worker's real per-job path.
|
||||
const runDueEvaluateJobs = async () => {
|
||||
const jobs = await evaluatorJobsRepository.claimDueBatch("evaluate", 20, Date.now());
|
||||
const runJob = (evaluator as unknown as { runJob: (job: Job) => Promise<void> }).runJob;
|
||||
for (const job of jobs) await runJob(job);
|
||||
return jobs.length;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
logger = createMockLogger() as unknown as ILogger;
|
||||
monitorsRepository = new InMemoryMonitorsRepository();
|
||||
monitorsRepository.seed(makeMonitor());
|
||||
checksRepository = new MongoChecksRepository(logger);
|
||||
checkService = new CheckService(monitorsRepository, logger, checksRepository);
|
||||
bufferA = createWorkerBuffer("worker-a");
|
||||
bufferB = createWorkerBuffer("worker-b");
|
||||
|
||||
evaluatedCheckIds = [];
|
||||
evaluatorJobsRepository = new MongoJobsRepository("worker-b");
|
||||
evaluator = new DBQueueWorker({
|
||||
logger,
|
||||
isDbConnected: () => true,
|
||||
jobsRepository: evaluatorJobsRepository,
|
||||
monitorsRepository,
|
||||
checksRepository,
|
||||
checkService,
|
||||
bufferService: bufferB,
|
||||
checkProducer: { produce: jest.fn<any>() },
|
||||
checkEvaluator: {
|
||||
evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
|
||||
evaluatedCheckIds.push(check.id);
|
||||
return {
|
||||
monitor,
|
||||
status,
|
||||
check,
|
||||
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 },
|
||||
decision: {},
|
||||
};
|
||||
},
|
||||
} as any,
|
||||
geoCheckPipeline: { run: jest.fn<any>() },
|
||||
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
|
||||
helper: {} as any,
|
||||
queueWorkersRepository: {} as any,
|
||||
queueMode: "worker",
|
||||
queuePrimaryProcesses: true,
|
||||
workerId: "worker-b",
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await bufferA.shutdown();
|
||||
await bufferB.shutdown();
|
||||
});
|
||||
|
||||
it("evaluates every persisted check even when a later worker's batch flushes first", async () => {
|
||||
// t=0: worker A holds the check job, probes, and buffers the result. Its flush timer has not fired.
|
||||
const c1 = producedAt(checkService.toCheck(makeStatusResponse())!, T0);
|
||||
bufferA.addToBuffer(c1);
|
||||
|
||||
// t=30s: worker B claims the next run of the same job, probes, and buffers the result.
|
||||
const c2 = producedAt(checkService.toCheck(makeStatusResponse())!, T0 + INTERVAL_MS);
|
||||
bufferB.addToBuffer(c2);
|
||||
|
||||
// t=45s: worker B's flush timer fires first. c2 is inserted and an evaluate job is queued.
|
||||
await bufferB.flushBuffer();
|
||||
expect(await runDueEvaluateJobs()).toBe(1);
|
||||
expect(evaluatedCheckIds).toEqual([c2.id]);
|
||||
|
||||
// t=60s: worker A's flush timer fires. c1 is inserted and the evaluate job is pulled due again.
|
||||
await bufferA.flushBuffer();
|
||||
expect(await runDueEvaluateJobs()).toBe(1);
|
||||
|
||||
// Both checks are durably stored and visible in history.
|
||||
const stored = await CheckModel.find({ "metadata.monitorId": new mongoose.Types.ObjectId(MONITOR_ID) }).lean();
|
||||
expect(stored.map((doc) => String(doc._id)).sort()).toEqual([c1.id, c2.id].sort());
|
||||
|
||||
// Every stored check must reach the evaluator exactly once.
|
||||
expect(evaluatedCheckIds).toHaveLength(2);
|
||||
expect(evaluatedCheckIds).toContain(c1.id);
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import mongoose from "mongoose";
|
||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
||||
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
|
||||
import { BACKOFF_MS, LOCK_MS, type Job, type JobType } from "../../src/domain/jobs/job.type.ts";
|
||||
import { BACKOFF_MS, LOCK_MS, PARKED, type Job, type JobType, type PendingCheck } from "../../src/domain/jobs/job.type.ts";
|
||||
|
||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
|
||||
@@ -58,6 +58,9 @@ const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string
|
||||
|
||||
const WORKER_ID = "worker-under-test";
|
||||
|
||||
// Pending entries as ingest pushes them: one per inserted check, stamped with the check's createdAt.
|
||||
const pending = (...checkIds: string[]): PendingCheck[] => checkIds.map((checkId, i) => ({ checkId, createdAt: NOW + i }));
|
||||
|
||||
describe("MongoJobsRepository", () => {
|
||||
let repo: MongoJobsRepository;
|
||||
|
||||
@@ -180,7 +183,7 @@ describe("MongoJobsRepository", () => {
|
||||
|
||||
it("does not release a lock owned by another worker (fenced no-op)", async () => {
|
||||
// A worker whose lease expired and was reclaimed must not clobber the new owner's lock.
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7 });
|
||||
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7, failCount: 2, lastFailReason: "timeout" });
|
||||
|
||||
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000);
|
||||
|
||||
@@ -189,6 +192,72 @@ describe("MongoJobsRepository", () => {
|
||||
expect(row?.lockedBy).toBe("other-worker"); // lock untouched
|
||||
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
|
||||
expect(row?.runCount).toBe(7); // not double-counted
|
||||
expect(row?.failCount).toBe(2); // failure state untouched
|
||||
expect(row?.lastFailReason).toBe("timeout");
|
||||
});
|
||||
|
||||
it("clears the failure state so failCount counts consecutive failures", async () => {
|
||||
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, failCount: 3, lastFailReason: "connection refused" });
|
||||
|
||||
await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000);
|
||||
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.failCount).toBe(0);
|
||||
expect(row?.lastFailReason).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── recordOneShot (park after a single run) ─────────────────────────────────
|
||||
describe("recordOneShot", () => {
|
||||
it("parks the job, releases the lease, counts the run and clears the failure state", async () => {
|
||||
await seedJob({
|
||||
intervalMs: null,
|
||||
lockedBy: ownedBy(repo),
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
runCount: 1,
|
||||
failCount: 2,
|
||||
lastFailReason: "connection refused",
|
||||
});
|
||||
|
||||
const ok = await repo.recordOneShot("check:mon-1", NOW + 1_000);
|
||||
|
||||
expect(ok).toBe(true);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(PARKED);
|
||||
expect(row?.lockedBy).toBeNull();
|
||||
expect(row?.lockedUntil).toBeNull();
|
||||
expect(row?.lastFinishedAt).toBe(NOW + 1_000);
|
||||
expect(row?.runCount).toBe(2);
|
||||
expect(row?.failCount).toBe(0);
|
||||
expect(row?.lastFailReason).toBeNull();
|
||||
});
|
||||
|
||||
it("does not touch a lock owned by another worker (fenced no-op)", async () => {
|
||||
await seedJob({ intervalMs: null, lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, failCount: 2, lastFailReason: "timeout" });
|
||||
|
||||
expect(await repo.recordOneShot("check:mon-1", NOW + 1_000)).toBe(false);
|
||||
const row = await readRow("check:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW); // not parked
|
||||
expect(row?.lockedBy).toBe("other-worker");
|
||||
expect(row?.failCount).toBe(2);
|
||||
expect(row?.lastFailReason).toBe("timeout");
|
||||
});
|
||||
|
||||
it("re-arms instead of parking when checks were pushed while the run held the lease", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
lockedBy: ownedBy(repo),
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
pendingChecks: pending("c-late"), // arrived mid-run, not yet pulled
|
||||
});
|
||||
|
||||
expect(await repo.recordOneShot("evaluate:mon-1", NOW + 1_000)).toBe(true);
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // due now, not PARKED
|
||||
expect(row?.lockedBy).toBeNull();
|
||||
expect(row?.runCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -276,10 +345,10 @@ describe("MongoJobsRepository", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── upsertEvaluate (handoff + $min coalescing) ───────────────────────────────
|
||||
// ── upsertEvaluate (handoff: $push ids + $min due time) ──────────────────────
|
||||
describe("upsertEvaluate", () => {
|
||||
it("creates a single evaluate row keyed by monitor", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
it("creates a single evaluate row keyed by monitor carrying the pushed ids", async () => {
|
||||
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.type).toBe("evaluate");
|
||||
@@ -287,11 +356,12 @@ describe("MongoJobsRepository", () => {
|
||||
expect(row?.nextScheduledAt).toBe(NOW);
|
||||
expect(row?.intervalMs).toBeNull();
|
||||
expect(row?.isActive).toBe(true);
|
||||
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
|
||||
});
|
||||
|
||||
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
||||
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
|
||||
await repo.upsertEvaluate("mon-1", pending("c2"), NOW + 9_000);
|
||||
|
||||
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
|
||||
expect(rows).toHaveLength(1);
|
||||
@@ -299,19 +369,117 @@ describe("MongoJobsRepository", () => {
|
||||
});
|
||||
|
||||
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 9_000);
|
||||
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.nextScheduledAt).toBe(NOW);
|
||||
});
|
||||
|
||||
it("concurrent upserts for one monitor produce exactly one row", async () => {
|
||||
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
||||
await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW)));
|
||||
it("never duplicates an entry when the same batch is armed twice (retry after a failed arm)", async () => {
|
||||
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
|
||||
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW + 1_000);
|
||||
|
||||
const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" });
|
||||
expect(count).toBe(1);
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
|
||||
});
|
||||
|
||||
it("accumulates ids across flushes so a late batch is never lost", async () => {
|
||||
await repo.upsertEvaluate("mon-1", pending("c2"), NOW); // later check, flushed first
|
||||
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 15_000); // earlier check, flushed later
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2", "c1"]);
|
||||
});
|
||||
|
||||
it("keeps pushing onto a row that is currently leased by a worker", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
lockedBy: "other-worker",
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
pendingChecks: pending("c1"),
|
||||
});
|
||||
|
||||
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.lockedBy).toBe("other-worker"); // lease untouched
|
||||
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1", "c2"]);
|
||||
});
|
||||
|
||||
it("concurrent upserts for one monitor produce exactly one row holding every id", async () => {
|
||||
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
||||
await Promise.all(workers.map((w, i) => w.upsertEvaluate("mon-1", pending(`c${i}`), NOW)));
|
||||
|
||||
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].pendingChecks).toHaveLength(20);
|
||||
});
|
||||
});
|
||||
|
||||
// ── pullEvaluated (evaluator removes applied ids) ───────────────────────────
|
||||
describe("pullEvaluated", () => {
|
||||
it("removes only the listed ids and leaves the rest", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
lockedBy: ownedBy(repo),
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
pendingChecks: pending("c1", "c2", "c3"),
|
||||
});
|
||||
|
||||
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1", "c3"])).toBe(true);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2"]);
|
||||
});
|
||||
|
||||
it("is fenced on the lease: another worker's row is untouched", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
lockedBy: "other-worker",
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
pendingChecks: pending("c1"),
|
||||
});
|
||||
|
||||
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(false);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1"]);
|
||||
});
|
||||
|
||||
it("reports the lease as held even when the ids were already pulled", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
lockedBy: ownedBy(repo),
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
pendingChecks: [],
|
||||
});
|
||||
|
||||
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(true);
|
||||
});
|
||||
|
||||
it("is a no-op for an empty id list", async () => {
|
||||
await seedJob({
|
||||
_id: "evaluate:mon-1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
lockedBy: ownedBy(repo),
|
||||
lockedUntil: NOW + LOCK_MS,
|
||||
pendingChecks: pending("c1"),
|
||||
});
|
||||
|
||||
expect(await repo.pullEvaluated("evaluate:mon-1", [])).toBe(true);
|
||||
|
||||
const row = await readRow("evaluate:mon-1");
|
||||
expect(row?.pendingChecks).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -327,6 +495,7 @@ describe("MongoJobsRepository", () => {
|
||||
intervalMs: 60_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
pendingChecks: [],
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
@@ -358,6 +527,7 @@ describe("MongoJobsRepository", () => {
|
||||
intervalMs: 86_400_000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
pendingChecks: [],
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
@@ -438,7 +608,7 @@ describe("MongoJobsRepository", () => {
|
||||
it("deleteById drops every row for the monitor", async () => {
|
||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||
await repo.upsertEvaluate("mon-1", NOW);
|
||||
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
|
||||
|
||||
await repo.deleteById("mon-1");
|
||||
|
||||
@@ -533,4 +703,47 @@ describe("MongoJobsRepository", () => {
|
||||
expect(await repo.countDueBacklog(NOW)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── global rows: egress recovery job ──────────────────────────────────────
|
||||
|
||||
describe("global rows", () => {
|
||||
const repo = new MongoJobsRepository(WORKER_ID);
|
||||
const egressRow = (nextScheduledAt: number) =>
|
||||
seedJob({ _id: "egress", type: "egress" as JobType, refId: null, nextScheduledAt, intervalMs: 30_000 });
|
||||
|
||||
it("deleteGlobalJob removes the row by its canonical _id and leaves other global rows alone", async () => {
|
||||
await egressRow(NOW);
|
||||
await seedJob({ _id: "cleanup-orphaned", type: "cleanup-orphaned" as JobType, refId: null });
|
||||
|
||||
expect(await repo.deleteGlobalJob("egress")).toBe(true);
|
||||
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
expect(await readRow("cleanup-orphaned")).not.toBeNull();
|
||||
expect(await repo.deleteGlobalJob("egress")).toBe(false);
|
||||
});
|
||||
|
||||
it("deleteGlobalJobIfUnchanged removes the row only while nextScheduledAt matches the claimed value", async () => {
|
||||
await egressRow(NOW);
|
||||
|
||||
// Another worker started a new episode and rescheduled the row after this run claimed it
|
||||
await JobModel.updateOne({ _id: "egress" }, { $set: { nextScheduledAt: NOW + 30_000 } });
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW)).toBe(false);
|
||||
expect(await readRow("egress")).not.toBeNull();
|
||||
|
||||
// Nobody touched it: the row goes
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW + 30_000)).toBe(true);
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
});
|
||||
|
||||
it("recordSuccess after a refused conditional delete is a no-op once the lock has moved on", async () => {
|
||||
await egressRow(NOW);
|
||||
const [claimed] = await repo.claimDueBatch("egress", 1, NOW);
|
||||
expect(claimed?.id).toBe("egress");
|
||||
|
||||
// Row deleted by the handler (recovered), then recordSuccess runs from the queue: nothing to update
|
||||
expect(await repo.deleteGlobalJobIfUnchanged("egress", claimed!.nextScheduledAt)).toBe(true);
|
||||
expect(await repo.recordSuccess("egress", claimed!.nextScheduledAt, 30_000, NOW + 1_000)).toBe(false);
|
||||
expect(await readRow("egress")).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -33,7 +33,7 @@ const createMockHttpProvider = () =>
|
||||
|
||||
const createMockSettingsService = (apiKey?: string) =>
|
||||
({
|
||||
getDBSettings: jest.fn().mockResolvedValue({ pagespeedApiKey: apiKey }),
|
||||
getCachedDBSettings: jest.fn().mockResolvedValue({ pagespeedApiKey: apiKey }),
|
||||
}) as unknown as jest.Mocked<ISettingsService>;
|
||||
|
||||
const createProvider = (opts?: { apiKey?: string }) => {
|
||||
@@ -154,7 +154,7 @@ describe("PageSpeedProvider", () => {
|
||||
const logger = createMockLogger();
|
||||
const httpProvider = createMockHttpProvider();
|
||||
const settingsService = {
|
||||
getDBSettings: jest.fn().mockResolvedValue(null),
|
||||
getCachedDBSettings: jest.fn().mockResolvedValue(null),
|
||||
} as unknown as jest.Mocked<ISettingsService>;
|
||||
const provider = new PageSpeedProvider(httpProvider, settingsService, logger as any);
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { PingProvider } from "../../../../src/service/network/PingProvider.ts";
|
||||
import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts";
|
||||
import type { Monitor } from "../../../../src/domain/monitors/monitor.type.ts";
|
||||
import * as net from "net";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -27,7 +28,7 @@ const createMockPing = (result?: Partial<{ alive: boolean; time: number | string
|
||||
// ── Contract ─────────────────────────────────────────────────────────────────
|
||||
|
||||
testStatusProviderContract("PingProvider", {
|
||||
create: () => new PingProvider(createMockPing() as any),
|
||||
create: () => new PingProvider(createMockPing() as any, net),
|
||||
supportedType: "ping",
|
||||
unsupportedType: "http",
|
||||
makeMonitor: () => makeMonitor(),
|
||||
@@ -37,7 +38,7 @@ testStatusProviderContract("PingProvider", {
|
||||
|
||||
describe("PingProvider", () => {
|
||||
it("returns success when host is alive", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -55,7 +56,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("returns failure when host is not alive", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -69,7 +70,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("parses string time values", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -77,7 +78,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("defaults responseTime to 0 for non-numeric time", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -85,7 +86,7 @@ describe("PingProvider", () => {
|
||||
});
|
||||
|
||||
it("defaults status to false when alive is undefined", async () => {
|
||||
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any);
|
||||
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any, net);
|
||||
|
||||
const result = await provider.handle(makeMonitor());
|
||||
|
||||
@@ -94,7 +95,7 @@ describe("PingProvider", () => {
|
||||
|
||||
it("sanitizes URL by stripping protocol and path", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "https://example.com/path?query=1" }));
|
||||
|
||||
@@ -103,36 +104,54 @@ describe("PingProvider", () => {
|
||||
|
||||
it("sanitizes URL by stripping port", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "example.com:8080" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("example.com");
|
||||
});
|
||||
|
||||
it("keeps a bare IPv6 address intact", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
|
||||
});
|
||||
|
||||
it("unwraps a bracketed IPv6 address and drops its port", async () => {
|
||||
const mockPing = createMockPing();
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" }));
|
||||
|
||||
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
|
||||
});
|
||||
|
||||
it("throws AppError when url is missing", async () => {
|
||||
const provider = new PingProvider(createMockPing() as any);
|
||||
const provider = new PingProvider(createMockPing() as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor({ url: "" }))).rejects.toThrow("URL is required for ping monitor");
|
||||
});
|
||||
|
||||
it("throws AppError when ping.probe rejects", async () => {
|
||||
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(new Error("Host unreachable")) } };
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("Host unreachable");
|
||||
});
|
||||
|
||||
it("throws AppError when probe returns no response", async () => {
|
||||
const mockPing = { promise: { probe: jest.fn().mockResolvedValue(null) } };
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("No response from ping");
|
||||
});
|
||||
|
||||
it("throws AppError with stringified message for non-Error throws", async () => {
|
||||
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(42) } };
|
||||
const provider = new PingProvider(mockPing as any);
|
||||
const provider = new PingProvider(mockPing as any, net);
|
||||
|
||||
await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
|
||||
});
|
||||
|
||||
@@ -45,6 +45,8 @@ const makeCheck = (overrides?: Partial<Check>): Check =>
|
||||
statusCode: 200,
|
||||
responseTime: 100,
|
||||
message: "OK",
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
...overrides,
|
||||
}) as Check;
|
||||
|
||||
@@ -392,9 +394,44 @@ describe("BufferService", () => {
|
||||
|
||||
await service.flushBuffer();
|
||||
|
||||
const createdAt = Date.parse("2026-01-01T00:00:00.000Z");
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-1", expect.any(Number));
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", expect.any(Number));
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith(
|
||||
"mon-1",
|
||||
[
|
||||
{ checkId: "c1", createdAt },
|
||||
{ checkId: "c2", createdAt },
|
||||
],
|
||||
expect.any(Number)
|
||||
);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
|
||||
});
|
||||
|
||||
it("retries a failed arm on the next flush without re-storing the checks", async () => {
|
||||
const { service, checkService, jobsRepository, logger } = createService();
|
||||
(jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed"));
|
||||
service.addToBuffer(makeCheck({ id: "c1" }));
|
||||
|
||||
await service.flushBuffer();
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
|
||||
|
||||
await service.flushBuffer(); // buffer is empty, only the retry runs
|
||||
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number));
|
||||
|
||||
await service.flushBuffer(); // nothing left to retry
|
||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("ingestChecks with an empty batch neither writes nor arms", async () => {
|
||||
const { service, checkService, jobsRepository } = createService();
|
||||
|
||||
await service.ingestChecks([]);
|
||||
|
||||
expect(checkService.createChecks).not.toHaveBeenCalled();
|
||||
expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not arm the evaluate stage when the check write fails", async () => {
|
||||
|
||||
@@ -9,7 +9,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
// ── Notes ──────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
|
||||
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
|
||||
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and
|
||||
// renews their locks, and registers itself in the worker heartbeat. The repository
|
||||
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
|
||||
// test only the orchestration around it, with the repository fully mocked.
|
||||
@@ -56,6 +56,7 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
intervalMs: 60000,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
pendingChecks: [],
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
@@ -63,12 +64,22 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// An evaluate row as ingest leaves it: one pending entry per stored check
|
||||
const makeEvaluateJob = (...checkIds: string[]): Job =>
|
||||
makeJob({
|
||||
id: "evaluate:m1",
|
||||
type: "evaluate",
|
||||
intervalMs: null,
|
||||
pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })),
|
||||
});
|
||||
|
||||
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
|
||||
const jobsRepository = {
|
||||
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
|
||||
upsertJob: jest.fn<any>().mockResolvedValue(true),
|
||||
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
|
||||
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
|
||||
pullEvaluated: jest.fn<any>().mockResolvedValue(true),
|
||||
recordSuccess: jest.fn<any>().mockResolvedValue(true),
|
||||
recordFailure: jest.fn<any>().mockResolvedValue(true),
|
||||
recordOneShot: jest.fn<any>().mockResolvedValue(true),
|
||||
@@ -92,7 +103,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
};
|
||||
const checkService = {
|
||||
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
|
||||
toLastEvaluatedAt: jest.fn<any>().mockReturnValue(12345),
|
||||
};
|
||||
const bufferService = {
|
||||
addToBuffer: jest.fn<any>(),
|
||||
@@ -113,6 +123,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
};
|
||||
const queueWorkersRepository = {
|
||||
upsert: jest.fn<any>().mockResolvedValue(undefined),
|
||||
@@ -226,7 +237,7 @@ describe("DBQueueWorker", () => {
|
||||
const { mocks } = await start({ queueMode: "worker" });
|
||||
|
||||
const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
|
||||
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
|
||||
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"]));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
|
||||
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
|
||||
@@ -368,27 +379,78 @@ describe("DBQueueWorker", () => {
|
||||
});
|
||||
|
||||
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const job = makeEvaluateJob("c1");
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluate job dispatches each unevaluated check and advances lastEvaluatedAt", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
it("evaluate job loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
|
||||
const job = makeEvaluateJob("c1");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||
});
|
||||
|
||||
expect(checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks);
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
|
||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
|
||||
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||
// The monitor cursor is gone: nothing writes lastEvaluatedAt
|
||||
expect(mocks.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("evaluate job with no pending checks does nothing", async () => {
|
||||
const job = makeEvaluateJob();
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
expect(mocks.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
|
||||
expect(mocks.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
||||
});
|
||||
|
||||
it("evaluate job for a deleted monitor drops every pending id", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2");
|
||||
const monitorsRepository = { findByIdLean: jest.fn<any>().mockResolvedValue(null), updateById: jest.fn<any>() };
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, monitorsRepository },
|
||||
});
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).not.toHaveBeenCalled();
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
|
||||
});
|
||||
|
||||
it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
|
||||
const job = makeEvaluateJob("c1", "c2", "c3");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) };
|
||||
const jobsRepository = {
|
||||
...createWorker().mocks.jobsRepository,
|
||||
claimDueBatch: claimOnce(job),
|
||||
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository, checksRepository } });
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
|
||||
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" }));
|
||||
});
|
||||
|
||||
it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
|
||||
const job = makeEvaluateJob("c1", "c-gone");
|
||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up
|
||||
const { mocks } = await start({
|
||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||
});
|
||||
|
||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]);
|
||||
});
|
||||
|
||||
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
|
||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
||||
const job = makeEvaluateJob("c1", "c2");
|
||||
const initialMonitor = makeMonitor();
|
||||
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
|
||||
const monitorsRepository = {
|
||||
@@ -440,6 +502,20 @@ describe("DBQueueWorker", () => {
|
||||
expect(retentionFn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("egress job runs the helper's egress recovery function with the claimed job", async () => {
|
||||
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
|
||||
const egressFn = jest.fn<any>().mockResolvedValue(undefined);
|
||||
const helper = {
|
||||
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
|
||||
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
|
||||
};
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
|
||||
|
||||
expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
|
||||
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
|
||||
});
|
||||
|
||||
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
|
||||
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
|
||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
import { afterEach, describe, expect, it, jest } from "@jest/globals";
|
||||
import { EgressService } from "../../../src/domain/egress/egress.service.ts";
|
||||
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
|
||||
import type { Job } from "../../../src/domain/jobs/job.type.ts";
|
||||
import { createMockLogger } from "../../helpers/createMockLogger.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
id: "egress-1",
|
||||
status: "ok",
|
||||
degradedSince: null,
|
||||
lastRecoveredAt: null,
|
||||
lastProbeAt: null,
|
||||
lastProbeResults: [],
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const makeSettings = (overrides?: Record<string, unknown>) => ({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: ["notif-1"],
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// The claimed "egress" job row as the queue hands it to the handler
|
||||
const makeJob = (overrides?: Partial<Job>): Job => ({
|
||||
id: "egress",
|
||||
type: "egress",
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: 1_000,
|
||||
intervalMs: 30_000,
|
||||
lockedBy: "worker-1",
|
||||
lockedUntil: 61_000,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// requestStatus mock that answers per target url
|
||||
const statusFor = (reachableTargets: string[]) =>
|
||||
jest.fn().mockImplementation(async (monitor: any) => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: reachableTargets.includes(monitor.url),
|
||||
code: reachableTargets.includes(monitor.url) ? 200 : 5000,
|
||||
message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed",
|
||||
responseTime: 12,
|
||||
}));
|
||||
|
||||
const degradedRepository = (overrides?: Record<string, any>) => ({
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
|
||||
markDegraded: jest.fn().mockResolvedValue(null),
|
||||
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
reset: jest.fn(),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const createService = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings()) },
|
||||
egressStateRepository: {
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState()),
|
||||
recordProbe: jest.fn().mockResolvedValue(makeState()),
|
||||
markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
|
||||
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })),
|
||||
reset: jest.fn(),
|
||||
},
|
||||
jobsRepository: {
|
||||
upsertJob: jest.fn().mockResolvedValue(true),
|
||||
upsertCleanupJob: jest.fn().mockResolvedValue(true),
|
||||
deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true),
|
||||
},
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) },
|
||||
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
|
||||
logger: createMockLogger(),
|
||||
...overrides,
|
||||
};
|
||||
const service = new EgressService(
|
||||
defaults.settingsService as any,
|
||||
defaults.egressStateRepository as any,
|
||||
defaults.jobsRepository as any,
|
||||
defaults.networkService as any,
|
||||
defaults.proxyResolver as any,
|
||||
defaults.logger as any
|
||||
);
|
||||
return { service, defaults };
|
||||
};
|
||||
|
||||
const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) =>
|
||||
(defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("EgressService", () => {
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
// ── probeTargets ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("probeTargets", () => {
|
||||
it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]);
|
||||
|
||||
const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any);
|
||||
expect(monitors).toEqual([
|
||||
expect.objectContaining({ type: "ping", url: "1.1.1.1" }),
|
||||
expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }),
|
||||
expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }),
|
||||
expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }),
|
||||
expect.objectContaining({ type: "ping", url: "2606:4700::1111" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => {
|
||||
const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128");
|
||||
const { service, defaults } = createService({ proxyResolver: { resolve } });
|
||||
|
||||
await service.probeTargets(["https://example.com/health"]);
|
||||
|
||||
expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" }));
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), {
|
||||
proxyUrl: "http://proxy.internal:3128",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports reachable only when the provider says status is true", async () => {
|
||||
const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } });
|
||||
|
||||
const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }),
|
||||
expect.objectContaining({ target: "8.8.8.8", reachable: false }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => {
|
||||
const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({
|
||||
monitorId: monitor.id,
|
||||
teamId: monitor.teamId,
|
||||
type: monitor.type,
|
||||
status: false,
|
||||
code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR
|
||||
message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED",
|
||||
responseTime: 3,
|
||||
}));
|
||||
const { service } = createService({ networkService: { requestStatus } });
|
||||
|
||||
const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }),
|
||||
expect.objectContaining({ target: "https://example.com/refused", reachable: false }),
|
||||
expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts
|
||||
]);
|
||||
});
|
||||
|
||||
it("counts an unparseable stored target as unreachable without probing", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
const results = await service.probeTargets(["[1.1.1.1"]);
|
||||
|
||||
expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]);
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("counts a thrown provider error as unreachable", async () => {
|
||||
const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } });
|
||||
|
||||
const results = await service.probeTargets(["1.1.1.1"]);
|
||||
|
||||
expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]);
|
||||
});
|
||||
|
||||
it("counts a hung provider as unreachable once the probe timeout elapses", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
|
||||
|
||||
const pending = service.probeTargets(["1.1.1.1"]);
|
||||
await jest.advanceTimersByTimeAsync(5000);
|
||||
const results = await pending;
|
||||
|
||||
expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── assessAfterFailure ────────────────────────────────────────────────────
|
||||
|
||||
describe("assessAfterFailure", () => {
|
||||
it("returns null and does not probe or record when the feature is disabled", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
|
||||
});
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns ok and records the probe when any target is reachable", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("ok");
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith(
|
||||
[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })],
|
||||
expect.any(Date)
|
||||
);
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith(
|
||||
[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
|
||||
expect.any(Date)
|
||||
);
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
|
||||
id: "egress",
|
||||
type: "egress",
|
||||
refId: null,
|
||||
isActive: true,
|
||||
nextScheduledAt: expect.any(Number),
|
||||
intervalMs: 30_000,
|
||||
});
|
||||
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
|
||||
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
|
||||
const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
expect(scheduleOrder).toBeLessThan(transitionOrder);
|
||||
});
|
||||
|
||||
it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => {
|
||||
const { service, defaults } = createService({
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
egressStateRepository: degradedRepository({
|
||||
findSingleton: jest.fn().mockResolvedValue(makeState()),
|
||||
markDegraded: jest.fn().mockResolvedValue(null),
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
|
||||
});
|
||||
|
||||
it("does not mark degraded when the recovery job cannot be scheduled", async () => {
|
||||
const { service, defaults } = createService({
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
jobsRepository: {
|
||||
upsertJob: jest.fn(),
|
||||
upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")),
|
||||
deleteGlobalJobIfUnchanged: jest.fn(),
|
||||
},
|
||||
});
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
|
||||
const { service, defaults } = createService({ egressStateRepository: degradedRepository() });
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBe("degraded");
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
|
||||
});
|
||||
|
||||
it("falls back to the default targets when the configured list is empty", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
|
||||
});
|
||||
|
||||
await service.assessAfterFailure();
|
||||
|
||||
expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]);
|
||||
});
|
||||
|
||||
it("returns null and logs when the assessment itself fails", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockRejectedValue(new Error("db down")) },
|
||||
});
|
||||
|
||||
const result = await service.assessAfterFailure();
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── shared assessment ─────────────────────────────────────────────────────
|
||||
|
||||
describe("shared assessment", () => {
|
||||
it("shares one probe between concurrent failures", async () => {
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
|
||||
|
||||
expect(results).toEqual(["ok", "ok", "ok"]);
|
||||
expect(defaults.settingsService.getCachedDBSettings).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps sharing an in-flight assessment that outlives the reuse window", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
|
||||
|
||||
const first = service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call
|
||||
const second = service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle
|
||||
|
||||
expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]);
|
||||
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds
|
||||
});
|
||||
|
||||
it("reuses a settled assessment within the reuse window and probes again after it", async () => {
|
||||
jest.useFakeTimers();
|
||||
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
|
||||
|
||||
await service.assessAfterFailure();
|
||||
await jest.advanceTimersByTimeAsync(1_000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
|
||||
await jest.advanceTimersByTimeAsync(5_000);
|
||||
await service.assessAfterFailure();
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("reuses a disabled result instead of re-reading settings for every failure", async () => {
|
||||
const getCachedDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false }));
|
||||
const { service } = createService({ settingsService: { getCachedDBSettings } });
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
|
||||
expect(getCachedDBSettings).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not cache an internal failure", async () => {
|
||||
const getCachedDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
|
||||
const { service, defaults } = createService({ settingsService: { getCachedDBSettings } });
|
||||
|
||||
expect(await service.assessAfterFailure()).toBeNull();
|
||||
expect(await service.assessAfterFailure()).toBe("ok");
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── checkRecovery (the "egress" job) ──────────────────────────────────────
|
||||
|
||||
describe("checkRecovery", () => {
|
||||
it("marks recovered, releases the job and logs when a target becomes reachable", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository(),
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
|
||||
});
|
||||
const job = makeJob({ nextScheduledAt: 123_456 });
|
||||
|
||||
await service.checkRecovery(job);
|
||||
|
||||
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith(
|
||||
[expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
|
||||
expect.any(Date)
|
||||
);
|
||||
// Conditional on the schedule this run claimed, so a concurrent new episode keeps its job
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456);
|
||||
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }));
|
||||
});
|
||||
|
||||
it("releases the job but does not log a recovery when another process already recorded it", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }),
|
||||
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
|
||||
});
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
expect(defaults.logger.info).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })
|
||||
);
|
||||
});
|
||||
|
||||
it("records the probe and keeps the job while every target stays unreachable", async () => {
|
||||
const { service, defaults } = createService({
|
||||
egressStateRepository: degradedRepository(),
|
||||
networkService: { requestStatus: statusFor([]) },
|
||||
});
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("releases the job without probing when the persisted state is no longer degraded", async () => {
|
||||
const { service, defaults } = createService();
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
});
|
||||
|
||||
it("releases the job without probing when the feature has been disabled mid-episode", async () => {
|
||||
const { service, defaults } = createService({
|
||||
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
|
||||
egressStateRepository: degradedRepository(),
|
||||
});
|
||||
|
||||
await service.checkRecovery(makeJob());
|
||||
|
||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
|
||||
});
|
||||
|
||||
it("lets errors propagate so the queue records the failure and retries", async () => {
|
||||
const { service } = createService({
|
||||
egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }),
|
||||
});
|
||||
|
||||
await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, expect, it, jest } from "@jest/globals";
|
||||
import { EgressStateService } from "../../../src/domain/egress/egress-state.service.ts";
|
||||
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const createEgressStateRepo = () => ({
|
||||
findSingleton: jest.fn(),
|
||||
recordProbe: jest.fn(),
|
||||
markDegraded: jest.fn(),
|
||||
markRecovered: jest.fn(),
|
||||
reset: jest.fn(),
|
||||
});
|
||||
|
||||
const createJobsRepo = () => ({
|
||||
deleteGlobalJob: jest.fn().mockResolvedValue(true),
|
||||
});
|
||||
|
||||
const createService = () => {
|
||||
const egressStateRepository = createEgressStateRepo();
|
||||
const jobsRepository = createJobsRepo();
|
||||
const service = new EgressStateService(egressStateRepository as any, jobsRepository as any);
|
||||
return { service, egressStateRepository, jobsRepository };
|
||||
};
|
||||
|
||||
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
|
||||
id: "egress-1",
|
||||
status: "ok",
|
||||
degradedSince: null,
|
||||
lastRecoveredAt: null,
|
||||
lastProbeAt: null,
|
||||
lastProbeResults: [],
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("EgressStateService", () => {
|
||||
describe("getState", () => {
|
||||
it("returns the singleton state from the repository", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
const state = makeState({
|
||||
status: "degraded",
|
||||
degradedSince: "2026-01-01T00:05:00Z",
|
||||
lastProbeAt: "2026-01-01T00:05:00Z",
|
||||
lastProbeResults: [{ target: "1.1.1.1", reachable: false, responseTime: 0, message: "timeout" }],
|
||||
});
|
||||
(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(state);
|
||||
|
||||
const result = await service.getState();
|
||||
|
||||
expect(egressStateRepository.findSingleton).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual(state);
|
||||
});
|
||||
|
||||
it("does not call any transition method", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(makeState());
|
||||
|
||||
await service.getState();
|
||||
|
||||
expect(egressStateRepository.recordProbe).not.toHaveBeenCalled();
|
||||
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("propagates repository errors", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
(egressStateRepository.findSingleton as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
await expect(service.getState()).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
|
||||
describe("reset", () => {
|
||||
it("removes any pending recovery job and returns the state to ok", async () => {
|
||||
const { service, egressStateRepository, jobsRepository } = createService();
|
||||
const reset = makeState();
|
||||
(egressStateRepository.reset as jest.Mock).mockResolvedValue(reset);
|
||||
|
||||
const result = await service.reset();
|
||||
|
||||
expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress");
|
||||
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual(reset);
|
||||
});
|
||||
|
||||
it("does not touch the probe-driven transitions", async () => {
|
||||
const { service, egressStateRepository } = createService();
|
||||
(egressStateRepository.reset as jest.Mock).mockResolvedValue(makeState());
|
||||
|
||||
await service.reset();
|
||||
|
||||
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
|
||||
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -143,6 +143,21 @@ describe("JobScheduler", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("addJob", () => {
|
||||
it("preserves an arbitrary monitor interval in the scheduled job", async () => {
|
||||
const { scheduler, mocks } = createScheduler();
|
||||
|
||||
await scheduler.addJob("m1", makeMonitor({ interval: 75000 }));
|
||||
|
||||
expect(mocks.jobsRepository.upsertJob).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
id: "check:m1",
|
||||
intervalMs: 75000,
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// Regression: a heartbeat upsert racing shutdown's deleteById could resurrect the
|
||||
// registry row, leaving a ghost worker that inflates the alive count until TTL.
|
||||
// shutdown() must await the in-flight beat and issue the delete as the LAST write.
|
||||
|
||||
@@ -685,6 +685,22 @@ describe("NotificationMessageBuilder", () => {
|
||||
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
|
||||
});
|
||||
|
||||
// A temperature exactly AT the threshold must not report a breach here, matching
|
||||
// StatusService.computeHardwareStatus's own strict `>` comparison (cpu/memory/disk
|
||||
// already agree with this file on `>`; temp alone used to diverge with `>=`, so an
|
||||
// incident-driving check at exactly the threshold never fired but the notification
|
||||
// content for a breach some OTHER metric triggered could still list temp as breaching).
|
||||
it("does not report temperature breach when exactly at threshold", () => {
|
||||
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 });
|
||||
const response = makeStatusResponse({
|
||||
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [80] } as any }),
|
||||
} as any);
|
||||
|
||||
const breaches = builder.extractThresholdBreaches(monitor, response);
|
||||
|
||||
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips temperature check when tempAlertThreshold is undefined", () => {
|
||||
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined });
|
||||
const response = makeStatusResponse({
|
||||
|
||||
@@ -28,6 +28,7 @@ const createMonitorsRepo = () => ({
|
||||
|
||||
const createSettingsService = (clientHost = "https://app.example.com") => ({
|
||||
getSettings: jest.fn().mockReturnValue({ clientHost }),
|
||||
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
|
||||
});
|
||||
|
||||
const createMessageBuilder = () => ({
|
||||
@@ -371,5 +372,26 @@ describe("NotificationsService", () => {
|
||||
expect(result).toBe(deleted);
|
||||
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
|
||||
});
|
||||
|
||||
it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => {
|
||||
const { service, notificationsRepository, settingsService } = createService();
|
||||
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
|
||||
|
||||
await service.deleteById("notif-1", "team-1");
|
||||
|
||||
expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1");
|
||||
const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!;
|
||||
expect(deleteOrder).toBeLessThan(cascadeOrder);
|
||||
});
|
||||
|
||||
it("does not touch app settings when the delete is refused", async () => {
|
||||
const { service, notificationsRepository, settingsService } = createService();
|
||||
(notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found"));
|
||||
|
||||
await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found");
|
||||
|
||||
expect(settingsService.removeEgressNotification).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -27,7 +27,7 @@ const makeProxy = (overrides?: Partial<Proxy>): Proxy =>
|
||||
const createResolver = (overrides?: Record<string, any>) => {
|
||||
const defaults = {
|
||||
proxiesRepository: { findByIdOrNull: jest.fn<() => Promise<Proxy | null>>().mockResolvedValue(makeProxy()) },
|
||||
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: "p1" }) },
|
||||
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: "p1" }) },
|
||||
logger: createMockLogger(),
|
||||
ttlMs: 60_000,
|
||||
...overrides,
|
||||
@@ -46,7 +46,7 @@ describe("ProxyResolver", () => {
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(defaults.proxiesRepository.findByIdOrNull).not.toHaveBeenCalled();
|
||||
expect(defaults.settingsService.getDBSettings).not.toHaveBeenCalled();
|
||||
expect(defaults.settingsService.getCachedDBSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns undefined for proxyMode 'none' without calling settings", async () => {
|
||||
@@ -55,7 +55,7 @@ describe("ProxyResolver", () => {
|
||||
const result = await resolver.resolve(makeMonitor({ proxyMode: "none" }));
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(defaults.settingsService.getDBSettings).not.toHaveBeenCalled();
|
||||
expect(defaults.settingsService.getCachedDBSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── custom mode ───────────────────────────────────────────────────────────
|
||||
@@ -94,7 +94,7 @@ describe("ProxyResolver", () => {
|
||||
|
||||
it("returns undefined when the global proxy is disabled, without a repo call", async () => {
|
||||
const { resolver, defaults } = createResolver({
|
||||
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: false, globalProxyId: "p1" }) },
|
||||
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: false, globalProxyId: "p1" }) },
|
||||
});
|
||||
|
||||
const result = await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
|
||||
@@ -114,7 +114,7 @@ describe("ProxyResolver", () => {
|
||||
|
||||
it("returns undefined when the global proxy is enabled but globalProxyId is null, without a repo call", async () => {
|
||||
const { resolver, defaults } = createResolver({
|
||||
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: null }) },
|
||||
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: null }) },
|
||||
});
|
||||
|
||||
const result = await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
|
||||
@@ -140,7 +140,7 @@ describe("ProxyResolver", () => {
|
||||
const result = await resolver.resolve(makeMonitor({ proxyMode: "garbage" as Monitor["proxyMode"] }));
|
||||
|
||||
expect(result).toBe("http://proxy.example.com:8080");
|
||||
expect(defaults.settingsService.getDBSettings).toHaveBeenCalled();
|
||||
expect(defaults.settingsService.getCachedDBSettings).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// ── never throws ──────────────────────────────────────────────────────────
|
||||
@@ -158,7 +158,7 @@ describe("ProxyResolver", () => {
|
||||
|
||||
it("returns undefined and warns when the settings service throws", async () => {
|
||||
const { resolver, defaults } = createResolver({
|
||||
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockRejectedValue(new Error("db down")) },
|
||||
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockRejectedValue(new Error("db down")) },
|
||||
});
|
||||
|
||||
const result = await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
|
||||
@@ -175,7 +175,6 @@ describe("ProxyResolver", () => {
|
||||
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
|
||||
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
|
||||
|
||||
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.proxiesRepository.findByIdOrNull).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -185,7 +184,6 @@ describe("ProxyResolver", () => {
|
||||
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
|
||||
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
|
||||
|
||||
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(2);
|
||||
expect(defaults.proxiesRepository.findByIdOrNull).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
@@ -209,7 +207,6 @@ describe("ProxyResolver", () => {
|
||||
resolver.resolve(makeMonitor({ proxyMode: "inherit" })),
|
||||
]);
|
||||
|
||||
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
|
||||
expect(defaults.proxiesRepository.findByIdOrNull).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
|
||||
@@ -43,12 +43,13 @@ const createSettingsRepo = () =>
|
||||
create: jest.fn().mockResolvedValue(makeSettings()),
|
||||
findSingleton: jest.fn().mockResolvedValue(makeSettings()),
|
||||
update: jest.fn().mockResolvedValue(makeSettings()),
|
||||
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
|
||||
deleteLegacy: jest.fn().mockResolvedValue(true),
|
||||
}) as unknown as jest.Mocked<ISettingsRepository>;
|
||||
|
||||
const createService = (envOverrides?: Partial<ValidatedEnv>) => {
|
||||
const createService = (envOverrides?: Partial<ValidatedEnv>, dbSettingsTTLMs?: number) => {
|
||||
const env = makeEnv(envOverrides);
|
||||
const service = new SettingsService(env);
|
||||
const service = new SettingsService(env, dbSettingsTTLMs);
|
||||
const settingsRepository = createSettingsRepo();
|
||||
service.setRepository(settingsRepository);
|
||||
return { service, settingsRepository, env };
|
||||
@@ -171,6 +172,66 @@ describe("SettingsService", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ── getCachedDBSettings ─────────────────────────────────────────────────
|
||||
|
||||
describe("getCachedDBSettings", () => {
|
||||
it("serves a second call within the TTL from cache", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
|
||||
const first = await service.getCachedDBSettings();
|
||||
const second = await service.getCachedDBSettings();
|
||||
|
||||
expect(second).toBe(first);
|
||||
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("refetches after the TTL expires", async () => {
|
||||
const { service, settingsRepository } = createService(undefined, 0);
|
||||
|
||||
await service.getCachedDBSettings();
|
||||
await service.getCachedDBSettings();
|
||||
|
||||
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("shares one in-flight fetch between concurrent calls", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
|
||||
await Promise.all([service.getCachedDBSettings(), service.getCachedDBSettings(), service.getCachedDBSettings()]);
|
||||
|
||||
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not cache a rejected fetch, the next call retries", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
(settingsRepository.findSingleton as jest.Mock).mockRejectedValueOnce(new Error("db down")).mockResolvedValueOnce(makeSettings());
|
||||
|
||||
await expect(service.getCachedDBSettings()).rejects.toThrow("db down");
|
||||
await expect(service.getCachedDBSettings()).resolves.toEqual(makeSettings());
|
||||
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("refetches after updateDbSettings so the write is visible immediately", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
|
||||
await service.getCachedDBSettings();
|
||||
await service.updateDbSettings({ checkTTL: 60 });
|
||||
await service.getCachedDBSettings();
|
||||
|
||||
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("refetches after removeEgressNotification", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
|
||||
await service.getCachedDBSettings();
|
||||
await service.removeEgressNotification("notif-1");
|
||||
await service.getCachedDBSettings();
|
||||
|
||||
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── updateDbSettings ────────────────────────────────────────────────────
|
||||
|
||||
describe("updateDbSettings", () => {
|
||||
@@ -192,4 +253,20 @@ describe("SettingsService", () => {
|
||||
await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized");
|
||||
});
|
||||
});
|
||||
|
||||
describe("removeEgressNotification", () => {
|
||||
it("delegates the pull to the repository", async () => {
|
||||
const { service, settingsRepository } = createService();
|
||||
|
||||
await service.removeEgressNotification("notif-1");
|
||||
|
||||
expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1");
|
||||
});
|
||||
|
||||
it("throws when repository is not set", async () => {
|
||||
const service = new SettingsService(makeEnv());
|
||||
|
||||
await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -41,6 +41,9 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
const checkService = {
|
||||
deleteOlderThan: jest.fn().mockResolvedValue(0),
|
||||
};
|
||||
const egressService = {
|
||||
checkRecovery: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
const defaults = {
|
||||
logger: createMockLogger(),
|
||||
@@ -54,6 +57,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
incidentsRepository,
|
||||
geoChecksRepository,
|
||||
dockerLogsRepository,
|
||||
egressService,
|
||||
...overrides,
|
||||
};
|
||||
|
||||
@@ -68,7 +72,8 @@ const createHelper = (overrides?: Record<string, unknown>) => {
|
||||
defaults.checksRepository as any,
|
||||
defaults.incidentsRepository as any,
|
||||
defaults.geoChecksRepository as any,
|
||||
defaults.dockerLogsRepository as any
|
||||
defaults.dockerLogsRepository as any,
|
||||
defaults.egressService as any
|
||||
);
|
||||
return { helper, defaults };
|
||||
};
|
||||
@@ -184,4 +189,22 @@ describe("WorkerHelper", () => {
|
||||
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
|
||||
});
|
||||
});
|
||||
|
||||
describe("getEgressRecoveryJob", () => {
|
||||
it("returns a function that hands the claimed job to the egress service", async () => {
|
||||
const { helper, defaults } = createHelper();
|
||||
const job = { id: "egress", type: "egress", refId: null, nextScheduledAt: 1000 } as any;
|
||||
|
||||
await helper.getEgressRecoveryJob()(job);
|
||||
|
||||
expect(defaults.egressService.checkRecovery).toHaveBeenCalledWith(job);
|
||||
});
|
||||
|
||||
it("lets the egress service's error propagate so the queue records the failure", async () => {
|
||||
const { helper, defaults } = createHelper();
|
||||
(defaults.egressService.checkRecovery as jest.Mock).mockRejectedValue(new Error("db down"));
|
||||
|
||||
await expect(helper.getEgressRecoveryJob()({ id: "egress" } as any)).rejects.toThrow("db down");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { parseEgressTarget } from "../../../src/utils/egressTarget.ts";
|
||||
|
||||
describe("parseEgressTarget", () => {
|
||||
it("parses http(s) URLs", () => {
|
||||
expect(parseEgressTarget("https://example.com/health")).toEqual({ kind: "http", url: "https://example.com/health" });
|
||||
expect(parseEgressTarget("HTTP://example.com")).toEqual({ kind: "http", url: "HTTP://example.com" });
|
||||
expect(parseEgressTarget("http://[2606:4700::1111]:8080/")).toEqual({ kind: "http", url: "http://[2606:4700::1111]:8080/" });
|
||||
});
|
||||
|
||||
it("parses host:port with a hostname, IPv4 address, localhost or bracketed IPv6 address", () => {
|
||||
expect(parseEgressTarget("dns.google:53")).toEqual({ kind: "port", host: "dns.google", port: 53 });
|
||||
expect(parseEgressTarget("8.8.8.8:53")).toEqual({ kind: "port", host: "8.8.8.8", port: 53 });
|
||||
expect(parseEgressTarget("localhost:8080")).toEqual({ kind: "port", host: "localhost", port: 8080 });
|
||||
expect(parseEgressTarget("[2606:4700::1111]:443")).toEqual({ kind: "port", host: "2606:4700::1111", port: 443 });
|
||||
});
|
||||
|
||||
it("parses bare hosts and addresses as ping targets", () => {
|
||||
expect(parseEgressTarget("1.1.1.1")).toEqual({ kind: "ping", host: "1.1.1.1" });
|
||||
expect(parseEgressTarget("example.com")).toEqual({ kind: "ping", host: "example.com" });
|
||||
expect(parseEgressTarget("2606:4700:4700::1111")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
|
||||
expect(parseEgressTarget("[2606:4700:4700::1111]")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
|
||||
expect(parseEgressTarget(" 1.1.1.1 ")).toEqual({ kind: "ping", host: "1.1.1.1" });
|
||||
});
|
||||
|
||||
it("rejects anything the probe could not reach", () => {
|
||||
for (const target of [
|
||||
"",
|
||||
"ftp://example.com",
|
||||
"http://",
|
||||
"bad host",
|
||||
"1.1.1.1;rm",
|
||||
"[1.1.1.1",
|
||||
"1.1.1.1]",
|
||||
"[::1",
|
||||
"::1]",
|
||||
"[[::1]]",
|
||||
"[not-an-address]:53",
|
||||
"[1.1.1.1]:53",
|
||||
"999.1.1.1",
|
||||
"1.1.1.1:0",
|
||||
"1.1.1.1:70000",
|
||||
"1.1.1.1:",
|
||||
"host-without-tld",
|
||||
]) {
|
||||
expect(parseEgressTarget(target)).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts";
|
||||
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts";
|
||||
|
||||
const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8";
|
||||
|
||||
describe("settingsValidation", () => {
|
||||
describe("updateAppSettingsBodyValidation egress fields", () => {
|
||||
it("accepts a full set of valid egress settings", () => {
|
||||
const parsed = updateAppSettingsBodyValidation.parse({
|
||||
egressCheckEnabled: true,
|
||||
egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"],
|
||||
egressPollIntervalSeconds: 30,
|
||||
egressNotifications: [validNotificationId],
|
||||
});
|
||||
|
||||
expect(parsed.egressCheckEnabled).toBe(true);
|
||||
expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]);
|
||||
expect(parsed.egressPollIntervalSeconds).toBe(30);
|
||||
expect(parsed.egressNotifications).toEqual([validNotificationId]);
|
||||
});
|
||||
|
||||
it("accepts a body with no egress fields", () => {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ checkTTL: 30 })).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts an empty target list (the worker falls back to the default targets)", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: [] }).egressCheckTargets).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects more targets than the maximum", () => {
|
||||
const targets = Array.from({ length: MAX_EGRESS_TARGETS + 1 }, (_, i) => `10.0.0.${i}`);
|
||||
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow();
|
||||
});
|
||||
|
||||
it("accepts IPv6 targets, bare or bracketed with a port", () => {
|
||||
const targets = ["2606:4700:4700::1111", "[2606:4700:4700::1111]", "[2606:4700:4700::1111]:53", "localhost", "localhost:8080"];
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets }).egressCheckTargets).toEqual(targets);
|
||||
});
|
||||
|
||||
it("rejects targets that are not a host, host:port or http(s) URL", () => {
|
||||
for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm", "http://"]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects malformed addresses that the probe could never reach", () => {
|
||||
for (const target of [
|
||||
"[1.1.1.1",
|
||||
"1.1.1.1]",
|
||||
"[::1",
|
||||
"::1]",
|
||||
"[[::1]]",
|
||||
"[not-an-address]:53",
|
||||
"999.1.1.1",
|
||||
"1.1.1.1:0",
|
||||
"1.1.1.1:70000",
|
||||
"[::1]:",
|
||||
]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts the poll interval boundaries", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MIN_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
|
||||
MAX_EGRESS_POLL_INTERVAL_SECONDS
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects out-of-range and non-integer poll intervals", () => {
|
||||
for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts an empty notification list", () => {
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
|
||||
});
|
||||
|
||||
it("accepts upper-case hex notification ids", () => {
|
||||
const id = "64B7F0C2A1D2E3F4A5B6C7D8";
|
||||
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [id] }).egressNotifications).toEqual([id]);
|
||||
});
|
||||
|
||||
it("rejects notification ids that are not 24-character hex", () => {
|
||||
for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a non-boolean enabled flag", () => {
|
||||
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckEnabled: "yes" })).toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user