Compare commits

..
Author SHA1 Message Date
Alexander Holliday 5ad82d1e02 Merge pull request #3953 from bluewave-labs/develop
develop -> demo
2026-09-12 15:04:34 -07:00
Alexander Holliday 54eaef725a Merge pull request #3888 from bluewave-labs/develop
develop -> demo
2026-08-26 12:08:23 -07:00
Alexander Holliday 2a2226456c Merge pull request #3884 from bluewave-labs/develop
develop -> demo
2026-08-26 10:42:45 -07:00
Alexander Holliday b8d4e4fc59 Merge pull request #3874 from bluewave-labs/develop
develop -> demo
2026-08-26 10:37:20 -07:00
Alexander Holliday 24cf1ec6c8 Merge pull request #3806 from bluewave-labs/develop
develop -> demo
2026-07-28 09:49:24 -07:00
Alexander Holliday a76f837803 Merge pull request #3759 from bluewave-labs/develop
develop -> demo
2026-07-07 10:19:16 -07:00
Alexander Holliday 9da2653d7b Merge pull request #3752 from bluewave-labs/develop
develop -> demo
2026-07-02 11:07:25 -07:00
Alexander Holliday 0092c9c006 Merge pull request #3746 from bluewave-labs/develop
develop -> demo
2026-07-02 10:15:41 -07:00
Alexander Holliday 348502834c Merge pull request #3745 from bluewave-labs/develop
develop -> demo
2026-07-02 08:50:31 -07:00
Alexander Holliday 1e007551ef Merge pull request #3706 from bluewave-labs/develop
develop -> demo
2026-06-05 12:03:44 -07:00
Alexander Holliday 07c62970b5 Merge pull request #3701 from bluewave-labs/develop
develop -> demo
2026-06-05 10:38:35 -07:00
Alexander Holliday f0123d8052 Merge pull request #3698 from bluewave-labs/develop
develop -> demo
2026-06-05 10:32:50 -07:00
Alexander Holliday de7d5b38ef Merge pull request #3608 from bluewave-labs/develop
develop -> demo
2026-04-29 13:03:15 -07:00
Alexander Holliday e51a55b14d Merge pull request #3562 from bluewave-labs/develop
develop -> demo
2026-04-23 10:13:33 -07:00
Alexander Holliday 1a5e8dc77d Merge pull request #3393 from bluewave-labs/develop
develop->demo 3.5.1 staging
2026-03-06 12:50:53 -08:00
Alexander Holliday 3b9e162315 Merge pull request #3367 from bluewave-labs/develop
develop -> demo
2026-03-04 10:30:22 -08:00
Alexander Holliday 89dec1c82b Merge pull request #3318 from bluewave-labs/develop
develop -> demo
2026-02-20 12:05:30 -08:00
Alexander Holliday 1a44080b84 Merge pull request #3317 from bluewave-labs/develop
develop -> demo
2026-02-20 10:32:55 -08:00
Alexander Holliday 762615b258 Merge pull request #3210 from bluewave-labs/develop
develop -> demo
2026-01-28 12:59:50 -08:00
Alexander Holliday f9036f0863 Merge pull request #3206 from bluewave-labs/develop
develop -> demo
2026-01-27 13:19:44 -08:00
Alexander Holliday 990f353040 Merge pull request #3203 from bluewave-labs/develop
develop -> demo
2026-01-27 13:05:13 -08:00
Alexander Holliday 37836e56b8 Merge pull request #3201 from bluewave-labs/develop
develop -> demo
2026-01-27 12:52:48 -08:00
Alexander Holliday 5e4060cfe3 Merge pull request #3198 from bluewave-labs/develop
develop -> demo
2026-01-26 20:08:38 -08:00
Alexander Holliday c90bef4cf8 Merge pull request #3196 from bluewave-labs/develop
develop->demo
2026-01-26 19:40:56 -08:00
Alexander Holliday 61bcae9aff Merge pull request #3194 from bluewave-labs/develop
develop -> demo
2026-01-26 15:58:39 -08:00
Alexander Holliday aa8dec6ca1 Merge pull request #3192 from bluewave-labs/develop
develop -> demo
2026-01-26 15:48:07 -08:00
Alexander Holliday e9bd8150fa Merge pull request #2965 from bluewave-labs/develop
develop -> demo
2025-09-18 13:41:13 -07:00
Alexander Holliday ce2808ccbc Merge pull request #2961 from bluewave-labs/develop
develop -> demo
2025-09-18 12:04:14 -07:00
Alexander Holliday 58692f8b72 Merge pull request #2934 from bluewave-labs/develop
develop -> demo
2025-09-08 20:32:53 -07:00
Alexander Holliday 421f1635ef Merge pull request #2930 from bluewave-labs/develop
develop->demo
2025-09-08 07:24:53 -07:00
Alexander Holliday b9f24f290d Merge pull request #2922 from bluewave-labs/develop
develop -> demo
2025-09-04 15:47:24 -07:00
Alexander Holliday a84cb95c50 Merge pull request #2903 from bluewave-labs/develop
develop -> demo
2025-09-01 09:00:47 -07:00
Alexander Holliday ac6e4ded73 Merge pull request #2891 from bluewave-labs/develop
develop -> demo
2025-08-30 19:32:05 -07:00
Alexander Holliday 5a6d59576a Merge pull request #2886 from bluewave-labs/develop
develop->demo
2025-08-30 16:24:11 -07:00
Alexander Holliday 3a34ee89d4 Merge pull request #2859 from bluewave-labs/develop
develop -> demo
2025-08-27 15:11:13 -07:00
Alexander Holliday ba2218490b Merge pull request #2856 from bluewave-labs/develop
develop -> demo
2025-08-25 14:38:31 -07:00
Alexander Holliday 188966db49 Merge pull request #2835 from bluewave-labs/develop
develop -> demo
2025-08-21 10:13:59 -07:00
Alexander Holliday 35b10b0a23 Merge pull request #2832 from bluewave-labs/develop
develop -> demo
2025-08-21 09:16:12 -07:00
Alexander Holliday 082d44cac7 Merge pull request #2825 from bluewave-labs/develop
develop->demo
2025-08-20 14:05:17 -07:00
Alexander Holliday a88f5def87 Merge pull request #2796 from bluewave-labs/develop
develop -> demo
2025-08-15 09:19:00 -07:00
Alexander Holliday 033a8e6ca1 Merge pull request #2786 from bluewave-labs/develop
develop -> demo
2025-08-14 12:49:50 -07:00
Alexander Holliday bd7016483d Merge pull request #2783 from bluewave-labs/develop
develop -> demo
2025-08-14 11:27:49 -07:00
Alexander Holliday 6bb885c98b Merge pull request #2781 from bluewave-labs/develop
develop -> demo
2025-08-14 10:53:11 -07:00
Alexander Holliday 2ca441548b Merge pull request #2779 from bluewave-labs/develop
develop -> Demo
2025-08-14 10:07:26 -07:00
Alexander Holliday 53461bbb59 Merge pull request #2776 from bluewave-labs/develop
Develop -> Demo
2025-08-14 09:56:38 -07:00
Alexander Holliday 82ad0bee9d Merge pull request #2648 from bluewave-labs/develop
develop -> demo
2025-07-20 19:50:19 -07:00
Alexander Holliday 250793ccdd Merge pull request #2611 from bluewave-labs/develop
Develop -> Demo
2025-07-14 14:23:10 -07:00
Alexander Holliday 93b3cc8b68 Merge pull request #2585 from bluewave-labs/develop
bump version
2025-07-08 09:51:51 -07:00
Alexander Holliday 19f7c30876 Merge pull request #2583 from bluewave-labs/develop
develop -> demo
2025-07-07 15:39:20 -07:00
Alexander Holliday c3cfdb68a5 Merge pull request #2546 from bluewave-labs/master
Master -> Demo
2025-06-25 12:43:06 +08:00
Alexander Holliday bad7a3c7ea Merge pull request #2537 from bluewave-labs/master
Master -> Demo for demo deploy
2025-06-25 09:40:06 +08:00
Alexander Holliday 4ef0ef53ed Merge pull request #2521 from bluewave-labs/develop
develop -> demo
2025-06-24 10:11:15 +08:00
Alexander Holliday 07da6ce4ce Merge pull request #2507 from bluewave-labs/develop
develop -> demo
2025-06-23 09:21:52 +08:00
Alexander Holliday 48b604bef9 Merge pull request #2377 from bluewave-labs/develop
Develop -> Demo
2025-05-30 15:12:06 -07:00
Alexander Holliday f14cd68fcc Merge pull request #2371 from bluewave-labs/develop
develop -> demo
2025-05-30 14:05:08 -07:00
Alexander Holliday b06aa07b3d Merge pull request #2364 from bluewave-labs/develop
develop -> demo
2025-05-28 09:50:13 -07:00
Alexander Holliday 1ba63985dd Merge pull request #2341 from bluewave-labs/develop
Develop -> Demo
2025-05-26 16:38:10 -07:00
Alexander Holliday 8968d01781 Merge pull request #2337 from bluewave-labs/develop
Develop -> Demo
2025-05-26 16:12:42 -07:00
Alexander Holliday daa63e7c54 Merge pull request #2310 from bluewave-labs/develop
develop -> demo
2025-05-19 12:45:04 -07:00
Alexander Holliday 305586d55b Merge pull request #2266 from bluewave-labs/develop
hotfix: remove version from returned settings
2025-05-13 13:53:23 -07:00
Alexander Holliday 881fbdeeb4 Merge pull request #2263 from bluewave-labs/develop
develop->demo
2025-05-13 13:35:45 -07:00
Alexander Holliday dd109c0e5c Merge pull request #2228 from bluewave-labs/develop
Demo deploy
2025-05-09 09:39:55 -07:00
73 changed files with 128 additions and 3069 deletions

No files matched your search

+1 -1
View File
@@ -92,7 +92,7 @@ JWT_SECRET="$(openssl rand -hex 32)" docker compose up -d
Then open http://localhost:52345. If the app is reached at another origin (domain or LAN IP), set `CLIENT_HOST` accordingly. To build the image yourself, run `docker build -f docker/Dockerfile -t checkmate .` from a checkout. For TLS, put any reverse proxy (Caddy, Traefik, nginx) in front of port 52345.
There are also 1-click installation options like [Repocloud](https://repocloud.io/details/Checkmate),
[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [Easypanel](https://easypanel.io/templates/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). Note that the Helm chart has not yet been migrated to the all-in-one image: it still deploys the legacy `checkmate-client`, `checkmate-backend` and `checkmate-mongo` images, pinned at v3.8.1.
[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). Note that the Helm chart has not yet been migrated to the all-in-one image: it still deploys the legacy `checkmate-client`, `checkmate-backend` and `checkmate-mongo` images, pinned at v3.8.1.
### Configuration
@@ -8,11 +8,8 @@ import { typographyLevels } from "@/Utils/Theme/Palette";
import useMediaQuery from "@mui/material/useMediaQuery";
import { LAYOUT } from "@/Utils/Theme/constants";
import { formatDuration } from "@/Utils/TimeUtils";
import { useTranslation } from "react-i18next";
export const MonitorStatus = ({ monitor }: { monitor: Monitor }) => {
const theme = useTheme();
const { t, i18n } = useTranslation();
const isSmall = useMediaQuery(theme.breakpoints.down("md"));
if (!monitor) {
@@ -55,12 +52,7 @@ export const MonitorStatus = ({ monitor }: { monitor: Monitor }) => {
<>
<Dot />
<Typography>
{t("pages.common.monitors.statBoxes.checkingEvery", {
interval: formatDuration(monitor.interval, {
long: true,
locale: i18n.resolvedLanguage,
}),
})}
Checking every {formatDuration(monitor?.interval, true)}
</Typography>
</>
)}
@@ -5,10 +5,9 @@ import {
type FieldValues,
} from "react-hook-form";
import { Autocomplete } from "@/Components/inputs";
import type { AutocompleteProps } from "@mui/material/Autocomplete";
export interface AutocompleteOption {
id: string | number;
id: string;
name: string;
}
@@ -17,9 +16,6 @@ interface FormAutocompleteProps<T extends FieldValues, O extends AutocompleteOpt
options: O[];
fieldLabel?: string;
placeholder?: string;
filterOptions?: AutocompleteProps<O, false, false, false>["filterOptions"];
getOptionDisabled?: AutocompleteProps<O, false, false, false>["getOptionDisabled"];
disableClearable?: boolean;
}
export const FormAutocompleteField = <
@@ -30,9 +26,6 @@ export const FormAutocompleteField = <
options,
fieldLabel,
placeholder,
filterOptions,
getOptionDisabled,
disableClearable,
}: FormAutocompleteProps<T, O>) => {
const { control } = useFormContext<T>();
return (
@@ -46,14 +39,10 @@ export const FormAutocompleteField = <
isOptionEqualToValue={(option, value) => option.id === value.id}
value={options.find((o) => o.id === field.value) ?? null}
onChange={(_: unknown, newValue: O | null) => {
if (newValue === null && disableClearable) return;
field.onChange(newValue?.id ?? "");
}}
fieldLabel={fieldLabel}
placeholder={placeholder}
filterOptions={filterOptions}
getOptionDisabled={getOptionDisabled}
disableClearable={disableClearable}
error={!!fieldState.error}
helperText={fieldState.error?.message ?? ""}
/>
@@ -21,7 +21,7 @@ export const MonitorStatBoxes = ({
domainExpiry,
}: MonitorStatBoxesProps) => {
const theme = useTheme();
const { t, i18n } = useTranslation();
const { t } = useTranslation();
if (!monitorStats || !monitor) {
return null;
}
@@ -29,16 +29,13 @@ export const MonitorStatBoxes = ({
const timeOfLastFailure = monitorStats?.timeOfLastFailure || 0;
const timeSinceLastFailure = timeOfLastFailure > 0 ? Date.now() - timeOfLastFailure : 0;
// Determine time since last check
const timeOfLastCheck = monitorStats?.lastCheckTimestamp || 0;
const timeSinceLastCheck = Date.now() - timeOfLastCheck || 0;
const streakTime = formatDuration(timeSinceLastFailure);
const lastCheckTime = timeOfLastCheck
? formatDuration(Date.now() - timeOfLastCheck, {
long: true,
relative: true,
locale: i18n.resolvedLanguage,
})
: t("common.labels.na");
const lastCheckTime = formatDuration(timeSinceLastCheck);
const isActive =
monitor?.status === "up" ||
monitor?.status === "paused" ||
-6
View File
@@ -20,18 +20,12 @@ import {
Tag,
Waypoints,
Container,
// LayoutDashboard,
} from "lucide-react";
export const getMenu = (t: Function) => {
return [
{
group: t("components.sidebar.menu.groups.monitoring"),
items: [
// {
// name: t("components.sidebar.menu.dashboard"),
// path: "dashboard",
// icon: <Icon icon={LayoutDashboard} />,
// },
{
name: t("components.sidebar.menu.uptime"),
path: "uptime",
-10
View File
@@ -2,10 +2,6 @@ import { createSlice, type PayloadAction } from "@reduxjs/toolkit";
type ThemeMode = "light" | "dark";
type ChartType = "histogram" | "line";
export const dashboardCardKeys = ["monitorStatus", "monitorsByType"] as const;
export type DashboardCardKey = (typeof dashboardCardKeys)[number];
type TableName =
| "monitors"
| "team"
@@ -39,7 +35,6 @@ interface UIState {
language: string;
starPromptOpen: boolean;
chartType: ChartType;
dashboardVisibleCards: DashboardCardKey[];
}
const initialMode: ThemeMode = window?.matchMedia?.("(prefers-color-scheme: dark)")
@@ -79,7 +74,6 @@ const initialState: UIState = {
language: "en",
starPromptOpen: true,
chartType: "histogram",
dashboardVisibleCards: [...dashboardCardKeys],
};
const uiSlice = createSlice({
@@ -123,9 +117,6 @@ const uiSlice = createSlice({
setChartType: (state, action: PayloadAction<ChartType>) => {
state.chartType = action.payload;
},
setDashboardVisibleCards: (state, action: PayloadAction<DashboardCardKey[]>) => {
state.dashboardVisibleCards = action.payload;
},
},
});
@@ -142,5 +133,4 @@ export const {
setLanguage,
setStarPromptOpen,
setChartType,
setDashboardVisibleCards,
} = uiSlice.actions;
+12 -60
View File
@@ -38,7 +38,6 @@ import {
PageSpeedStrategies,
DnsRecordTypes,
MonitorIntervalOptions,
MIN_MONITOR_INTERVAL_MS,
DefaultMonitorMatchMethod,
MonitorMatchMethods,
GeoCheckIntervalOptions,
@@ -59,14 +58,9 @@ import { FormTextField } from "@/Components/inputs/forms/FormTextField";
import { FormRadioGroup } from "@/Components/inputs/forms/FormRadioGroupField";
import { FormMultiSelectField } from "@/Components/inputs/forms/FormMultiSelectField";
import { FormSelectField } from "@/Components/inputs/forms/FormSelectField";
import {
FormAutocompleteField,
type AutocompleteOption,
} from "@/Components/inputs/forms/FormAutoCompleteField";
import { FormSliderField } from "@/Components/inputs/forms/FormSliderField";
import { FormSwitchField } from "@/Components/inputs/forms/FormSwitchField";
import type { ProxyResponse } from "@/Types/Proxy";
import { formatDuration } from "@/Utils/TimeUtils";
const httpMethodOptions = HttpMethods.map((method) => ({
value: method,
@@ -81,40 +75,6 @@ const maskedInputSx = {
"&:not(.Mui-focused) textarea": { WebkitTextSecurity: "disc" },
};
interface IntervalOption extends AutocompleteOption {
id: number;
}
const INTERVAL_UNIT_MULTIPLIERS = [1000, 60000, 3600000, 86400000] as const;
const toIntervalOption = (
intervalMs: number,
locale: string | undefined
): IntervalOption => ({
id: intervalMs,
name: formatDuration(intervalMs, { long: true, locale }),
});
const filterIntervalOptions = (
options: IntervalOption[],
inputValue: string,
locale: string | undefined
): IntervalOption[] => {
const input = inputValue.trim();
if (input === "" || options.some((option) => option.name === input)) {
return options;
}
const amount = Number(input);
if (!Number.isFinite(amount) || amount <= 0) {
return [];
}
return INTERVAL_UNIT_MULTIPLIERS.map((multiplier) => amount * multiplier)
.filter(Number.isSafeInteger)
.map((ms) => toIntervalOption(ms, locale));
};
interface GeneralSettingsConfig {
urlLabel: string;
urlPlaceholder: string;
@@ -292,7 +252,7 @@ const getGeneralSettingsConfig = (
const CreateMonitorPage = () => {
const theme = useTheme();
const { t, i18n } = useTranslation();
const { t } = useTranslation();
const { monitorId } = useParams();
const location = useLocation();
const navigate = useNavigate();
@@ -354,7 +314,6 @@ const CreateMonitorPage = () => {
const watchedProxyMode = watch("proxyMode") as ProxyMode | undefined;
const watchedUseAdvancedMatching = watch("useAdvancedMatching") as boolean;
const watchGeoCheckEnabled = watch("geoCheckEnabled") as boolean;
const watchedInterval = watch("interval");
// Steps without an advanced section drop it, so the last step is the form's.
const totalSteps = monitorStepCount(watchedType);
@@ -512,18 +471,16 @@ const CreateMonitorPage = () => {
[t]
);
const intervalOptions = useMemo(() => {
const options = MonitorIntervalOptions.map(({ value }) =>
toIntervalOption(value, i18n.resolvedLanguage)
);
if (
typeof watchedInterval === "number" &&
!options.some((option) => option.id === watchedInterval)
) {
options.push(toIntervalOption(watchedInterval, i18n.resolvedLanguage));
}
return options;
}, [watchedInterval, i18n.resolvedLanguage]);
const intervalOptions = useMemo(
() =>
MonitorIntervalOptions.map((option) => ({
value: option.value,
label: t(
`pages.createMonitor.form.frequency.option.frequency.value.${option.labelKey}`
),
})),
[t]
);
const matchMethodOptions = useMemo(
() =>
@@ -807,17 +764,12 @@ const CreateMonitorPage = () => {
title={t("pages.createMonitor.form.frequency.title")}
subtitle={t("pages.createMonitor.form.frequency.description")}
rightContent={
<FormAutocompleteField
<FormSelectField
name="interval"
fieldLabel={t(
"pages.createMonitor.form.frequency.option.frequency.label"
)}
options={intervalOptions}
filterOptions={(options, { inputValue }) =>
filterIntervalOptions(options, inputValue, i18n.resolvedLanguage)
}
getOptionDisabled={(option) => option.id < MIN_MONITOR_INTERVAL_MS}
disableClearable
/>
}
/>
@@ -1,54 +0,0 @@
import Box from "@mui/material/Box";
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { useTheme } from "@mui/material/styles";
import { LAYOUT } from "@/Utils/Theme/constants";
import { typographyLevels } from "@/Utils/Theme/Palette";
interface DashboardCardProps {
title: string;
topRight?: React.ReactNode;
children: React.ReactNode;
}
export const DashboardCard = ({ title, topRight, children }: DashboardCardProps) => {
const theme = useTheme();
return (
<Box
bgcolor={theme.palette.background.paper}
border={1}
borderColor={theme.palette.divider}
borderRadius={`${theme.shape.borderRadius}px`}
padding={theme.spacing(LAYOUT.MD)}
display="flex"
flexDirection="column"
gap={theme.spacing(LAYOUT.SM)}
height="100%"
boxSizing="border-box"
>
<Stack
direction="row"
alignItems="center"
justifyContent="space-between"
>
<Typography
fontSize={typographyLevels.m}
fontWeight={500}
color={theme.palette.text.primary}
>
{title}
</Typography>
{topRight && (
<Typography
fontSize={typographyLevels.s}
color={theme.palette.text.secondary}
>
{topRight}
</Typography>
)}
</Stack>
{children}
</Box>
);
};
@@ -1,71 +0,0 @@
import { useState } from "react";
import Stack from "@mui/material/Stack";
import FormControlLabel from "@mui/material/FormControlLabel";
import { useTranslation } from "react-i18next";
import { Checkbox, Dialog } from "@/Components/inputs";
import { type DashboardCardKey, dashboardCardKeys } from "@/Features/UI/uiSlice";
export type { DashboardCardKey };
interface EditCardsModalProps {
open: boolean;
visibleCards: Set<DashboardCardKey>;
onClose: () => void;
onSave: (visibleCards: Set<DashboardCardKey>) => void;
}
export const EditCardsModal = ({
open,
visibleCards,
onClose,
onSave,
}: EditCardsModalProps) => {
const { t } = useTranslation();
const [draft, setDraft] = useState<Set<DashboardCardKey>>(new Set(visibleCards));
const handleClose = () => {
setDraft(new Set(visibleCards));
onClose();
};
const handleToggle = (key: DashboardCardKey) => {
setDraft((prev) => {
const next = new Set(prev);
if (next.has(key)) {
next.delete(key);
} else {
next.add(key);
}
return next;
});
};
return (
<Dialog
open={open}
title={t("pages.dashboard.editCardsModal.title")}
content={t("pages.dashboard.editCardsModal.description")}
onCancel={handleClose}
onConfirm={() => onSave(draft)}
confirmText={t("pages.dashboard.editCardsModal.done")}
maxWidth="xs"
fullWidth
>
<Stack>
{dashboardCardKeys.map((key) => (
<FormControlLabel
key={key}
label={t(`pages.dashboard.cards.${key}`)}
control={
<Checkbox
checked={draft.has(key)}
disabled={draft.has(key) && draft.size === 1}
onChange={() => handleToggle(key)}
/>
}
/>
))}
</Stack>
</Dialog>
);
};
@@ -1,28 +0,0 @@
import { LAYOUT } from "@/Utils/Theme/constants";
import Box from "@mui/material/Box";
import { useTheme } from "@mui/material/styles";
interface ProgressBarProps {
fillRatio: number;
}
export const ProgressBar = ({ fillRatio }: ProgressBarProps) => {
const theme = useTheme();
const clamped = Math.min(1, Math.max(0, fillRatio));
return (
<Box
height={theme.spacing(LAYOUT.XS)}
borderRadius={theme.shape.borderRadius}
bgcolor={theme.palette.action.disabledBackground}
overflow="hidden"
>
<Box
height="100%"
width={`${clamped * 100}%`}
borderRadius={theme.shape.borderRadius}
bgcolor={theme.palette.primary.main}
/>
</Box>
);
};
@@ -1,42 +0,0 @@
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { useTheme } from "@mui/material/styles";
import { typographyLevels } from "@/Utils/Theme/Palette";
import { SPACING } from "@/Utils/Theme/constants";
import { ProgressBar } from "@/Pages/Dashboard/components/ProgressBar";
interface BarRowProps {
label: string;
value: string;
fillRatio: number;
valueColor?: string;
}
export const BarRow = ({ label, value, fillRatio, valueColor }: BarRowProps) => {
const theme = useTheme();
return (
<Stack gap={theme.spacing(SPACING.LG)}>
<Stack
direction="row"
justifyContent="space-between"
alignItems="center"
>
<Typography
fontSize={typographyLevels.m}
color={theme.palette.text.primary}
>
{label}
</Typography>
<Typography
fontSize={typographyLevels.m}
color={valueColor ?? theme.palette.primary.main}
fontWeight={500}
>
{value}
</Typography>
</Stack>
<ProgressBar fillRatio={fillRatio} />
</Stack>
);
};
@@ -1,58 +0,0 @@
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { useTheme } from "@mui/material/styles";
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
import { ProgressBar } from "@/Pages/Dashboard/components/ProgressBar";
import { typographyLevels } from "@/Utils/Theme/Palette";
import { SPACING } from "@/Utils/Theme/constants";
import { useTranslation } from "react-i18next";
import type { MonitorsSummary } from "@/Types/Monitor";
interface MonitorStatusCardProps {
summary: MonitorsSummary | null;
}
export const MonitorStatusCard = ({ summary }: MonitorStatusCardProps) => {
const theme = useTheme();
const { t } = useTranslation();
const up = summary?.upMonitors ?? 0;
const total = summary?.totalMonitors ?? 0;
const ratio = total > 0 ? up / total : 0;
return (
<DashboardCard title={t("pages.dashboard.cards.monitorStatus")}>
{total === 0 ? (
<Typography
fontSize={typographyLevels.m}
color={theme.palette.text.disabled}
>
{t("pages.dashboard.noData")}
</Typography>
) : (
<Stack gap={theme.spacing(SPACING.SM)}>
<Stack
direction="row"
alignItems="baseline"
gap={theme.spacing(SPACING.XS)}
>
<Typography
fontSize={typographyLevels.xxl}
fontWeight={500}
color={theme.palette.text.primary}
>
{up}
</Typography>
<Typography
fontSize={typographyLevels.m}
color={theme.palette.text.secondary}
>
{t("pages.dashboard.monitorStatus.ofUp", { total })}
</Typography>
</Stack>
<ProgressBar fillRatio={ratio} />
</Stack>
)}
</DashboardCard>
);
};
@@ -1,46 +0,0 @@
import Stack from "@mui/material/Stack";
import Typography from "@mui/material/Typography";
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
import { BarRow } from "@/Pages/Dashboard/components/cards/BarRow";
import { useTranslation } from "react-i18next";
import { useTheme } from "@mui/material/styles";
import type { MonitorTypeCount } from "@/Types/Monitor";
import { getMonitorTypeLabel } from "@/Types/Monitor";
import { LAYOUT } from "@/Utils/Theme/constants";
import { typographyLevels } from "@/Utils/Theme/Palette";
interface MonitorsByTypeCardProps {
monitorsByType: MonitorTypeCount[];
}
export const MonitorsByTypeCard = ({ monitorsByType }: MonitorsByTypeCardProps) => {
const { t } = useTranslation();
const theme = useTheme();
const maxCount = monitorsByType[0]?.count ?? 1;
return (
<DashboardCard title={t("pages.dashboard.cards.monitorsByType")}>
{monitorsByType.length === 0 ? (
<Typography
fontSize={typographyLevels.m}
color={theme.palette.text.disabled}
>
{t("pages.dashboard.noData")}
</Typography>
) : (
<Stack gap={theme.spacing(LAYOUT.SM)}>
{monitorsByType.map(({ type, count }) => (
<BarRow
key={type}
label={getMonitorTypeLabel(type, t)}
value={String(count)}
fillRatio={count / maxCount}
valueColor={theme.palette.text.primary}
/>
))}
</Stack>
)}
</DashboardCard>
);
};
-99
View File
@@ -1,99 +0,0 @@
import { useMemo, useState, type ReactNode } from "react";
import Box from "@mui/material/Box";
import Stack from "@mui/material/Stack";
import { useTheme } from "@mui/material/styles";
import { useTranslation } from "react-i18next";
import { useSelector, useDispatch } from "react-redux";
import { BasePage } from "@/Components/design-elements/BasePage";
import { Button } from "@/Components/inputs";
import { LAYOUT } from "@/Utils/Theme/constants";
import { useMonitorListController } from "@/Hooks/useMonitorListController";
import { MonitorTypes, type MonitorTypeCount } from "@/Types/Monitor";
import {
setDashboardVisibleCards,
dashboardCardKeys,
type DashboardCardKey,
} from "@/Features/UI/uiSlice";
import type { RootState, AppDispatch } from "@/store";
import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard";
import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard";
import { EditCardsModal } from "@/Pages/Dashboard/components/EditCardsModal";
const Dashboard = () => {
const theme = useTheme();
const { t } = useTranslation();
const dispatch = useDispatch<AppDispatch>();
const [editOpen, setEditOpen] = useState(false);
const visibleCardsList = useSelector(
(state: RootState) => state.ui.dashboardVisibleCards
);
const visibleCards = useMemo(() => new Set(visibleCardsList), [visibleCardsList]);
const { monitors, summary } = useMonitorListController({
types: [...MonitorTypes],
checksLimit: 1,
refreshInterval: 30000,
rowsPerPageTable: "monitors",
rowsPerPageDefault: 100,
});
const monitorsByType = useMemo<MonitorTypeCount[]>(() => {
const countMap = new Map<MonitorTypeCount["type"], number>();
(monitors ?? []).forEach((m) =>
countMap.set(m.type, (countMap.get(m.type) ?? 0) + 1)
);
return [...countMap.entries()]
.map(([type, count]) => ({ type, count }))
.sort((a, b) => b.count - a.count);
}, [monitors]);
const cardComponents: Record<DashboardCardKey, ReactNode> = {
monitorStatus: <MonitorStatusCard summary={summary} />,
monitorsByType: <MonitorsByTypeCard monitorsByType={monitorsByType} />,
};
return (
<BasePage headerKey="dashboard">
<Stack gap={theme.spacing(LAYOUT.SM)}>
<Stack
direction="row"
justifyContent="flex-end"
>
<Button
variant="outlined"
size="small"
onClick={() => setEditOpen(true)}
>
{t("pages.dashboard.editCards")}
</Button>
</Stack>
<Box
display="grid"
gridTemplateColumns={{ xs: "1fr", md: "1fr 1fr" }}
gap={theme.spacing(LAYOUT.MD)}
>
{dashboardCardKeys
.filter((key) => visibleCards.has(key))
.map((key) => (
<div key={key}>{cardComponents[key]}</div>
))}
</Box>
</Stack>
<EditCardsModal
open={editOpen}
visibleCards={visibleCards}
onClose={() => setEditOpen(false)}
onSave={(next) => {
dispatch(setDashboardVisibleCards([...next]));
setEditOpen(false);
}}
/>
</BasePage>
);
};
export default Dashboard;
+1 -1
View File
@@ -8,7 +8,7 @@ import { useTranslation } from "react-i18next";
const LogsPage = () => {
const { t } = useTranslation();
const [activeTab, setActiveTab] = useState<number>(0);
const [activeTab, setActiveTab] = useState<number>(1);
return (
<BasePage headerKey="logs">
<Tabs
-7
View File
@@ -8,9 +8,6 @@ import AuthRegister from "@/Pages/Auth/Register";
import AuthForgotPassword from "@/Pages/Auth/Recovery";
import AuthSetNewPassword from "@/Pages/Auth/SetNewPassword";
// Dashboard
import Dashboard from "@/Pages/Dashboard";
// Uptime
import Uptime from "@/Pages/Uptime/Monitors";
import UptimeDetails from "@/Pages/Uptime/Details";
@@ -95,10 +92,6 @@ const Routes = () => {
path="/"
element={<Navigate to="/uptime" />}
/>
<Route
path="dashboard"
element={<Dashboard />}
/>
<Route
path="/uptime"
element={<Uptime />}
-14
View File
@@ -60,15 +60,6 @@ export const monitorTypeLabelKey: Record<SelectableMonitorType, string> &
pagespeed: "optionPagespeed",
};
export const getMonitorTypeLabel = (
type: MonitorType,
t: (key: string) => string
): string => {
const monitorLabel = monitorTypeLabelKey[type];
return monitorLabel ? t(`pages.common.monitors.monitorTypes.${monitorLabel}`) : type;
};
export const MIN_MONITOR_INTERVAL_MS = 10000;
export const MonitorIntervalOptions = [
{ value: 15000, labelKey: "fifteenSeconds" },
{ value: 30000, labelKey: "thirtySeconds" },
@@ -192,11 +183,6 @@ export interface MonitorsSummary {
breachedMonitors: number;
}
export interface MonitorTypeCount {
type: MonitorType;
count: number;
}
export interface MonitorsWithChecksResponse {
count: number;
monitors: MonitorWithChecks[];
+3 -46
View File
@@ -104,52 +104,9 @@ export const formatMs = (ms: number, hasSpace: boolean = true): string => {
return formatted;
};
type DurationUnit = "day" | "hour" | "minute" | "second";
interface FormatDurationOptions {
long?: boolean;
relative?: boolean;
hasSpace?: boolean;
locale?: string;
}
const DURATION_UNITS: Array<{ unit: DurationUnit; milliseconds: number }> = [
{ unit: "day", milliseconds: MS_PER_DAY },
{ unit: "hour", milliseconds: MS_PER_HOUR },
{ unit: "minute", milliseconds: MS_PER_MINUTE },
{ unit: "second", milliseconds: MS_PER_SECOND },
];
const FALLBACK_DURATION_UNIT = DURATION_UNITS[DURATION_UNITS.length - 1];
const getDurationUnit = (ms: number) =>
DURATION_UNITS.find(({ milliseconds }) => Math.abs(ms) >= milliseconds) ??
FALLBACK_DURATION_UNIT;
const getExactDurationUnit = (ms: number) =>
DURATION_UNITS.find(({ milliseconds }) => ms !== 0 && ms % milliseconds === 0) ??
FALLBACK_DURATION_UNIT;
export const formatDuration = (
ms: number,
{ long = false, relative = false, hasSpace = true, locale }: FormatDurationOptions = {}
): string => {
if (relative) {
const { unit, milliseconds } = getDurationUnit(ms);
const value = Math.sign(ms) * Math.floor(Math.abs(ms) / milliseconds);
return new Intl.RelativeTimeFormat(locale, {
numeric: "always",
style: long ? "long" : "narrow",
}).format(-value, unit);
}
if (long) {
const { unit, milliseconds } = getExactDurationUnit(ms);
return new Intl.NumberFormat(locale, {
style: "unit",
unit,
unitDisplay: "long",
}).format(ms / milliseconds);
export const formatDuration = (ms: number, verbose: boolean = false, hasSpace = true) => {
if (verbose) {
return prettyMilliseconds(ms, { verbose: true });
}
const formatted = prettyMilliseconds(ms, {
+1 -3
View File
@@ -6,7 +6,6 @@ import {
HttpMethods,
PageSpeedStrategies,
ProxyModes,
MIN_MONITOR_INTERVAL_MS,
type MonitorType,
} from "@/Types/Monitor";
import { ALL_HTTP_STATUS_CODES } from "@/Utils/statusCode";
@@ -28,8 +27,7 @@ const baseSchema = z.object({
description: z.string().optional(),
interval: z
.number()
.int("Interval must be a whole number of milliseconds")
.min(MIN_MONITOR_INTERVAL_MS, "Interval must be at least 10 seconds")
.min(15000, "Interval must be at least 15 seconds")
.register(monitorStepRegistry, { step: 1 }),
notifications: z.array(z.string()).register(monitorStepRegistry, { step: 1 }),
tags: z.array(z.string()).register(monitorStepRegistry, { step: 1 }),
-1
View File
@@ -448,7 +448,6 @@
},
"statBoxes": {
"activeFor": "Aktiv seit",
"checkingEvery": "Check alle {{interval}}",
"certificateExpiry": "Zertifikatablauf",
"lastCheck": "Letzter Check",
"lastResponseTime": "Letzte Antwortzeit",
+1 -23
View File
@@ -226,8 +226,7 @@
"alerting": "Alerting",
"reporting": "Reporting",
"configuration": "Configuration"
},
"dashboard": "Dashboard"
}
},
"bottomMenu": {
"support": "Support",
@@ -548,7 +547,6 @@
},
"statBoxes": {
"activeFor": "Active for",
"checkingEvery": "Checking every {{interval}}",
"certificateExpiry": "Certificate expiry",
"domainExpiry": "Domain expiry",
"lastCheck": "Last check",
@@ -1909,26 +1907,6 @@
"title": "Uptime monitors",
"description": "Watch HTTP endpoints, pings, containers, and ports — and get alerted the moment something goes down."
}
},
"dashboard": {
"noData": "No data available at this moment.",
"monitorStatus": {
"ofUp": "of {{total}} up"
},
"cards": {
"monitorStatus": "Monitor status",
"monitorsByType": "Monitors by type"
},
"header": {
"title": "Dashboard",
"description": "See every monitor type in one place and spot what needs attention."
},
"editCards": "Edit cards",
"editCardsModal": {
"title": "Edit cards",
"description": "Select the cards to display on your dashboard.",
"done": "Done"
}
}
}
}
-24
View File
@@ -13294,30 +13294,6 @@
"globalProxyId": {
"type": "string",
"nullable": true
},
"egressCheckEnabled": {
"type": "boolean"
},
"egressCheckTargets": {
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 253
},
"maxItems": 10
},
"egressPollIntervalSeconds": {
"type": "integer",
"minimum": 5,
"maximum": 600
},
"egressNotifications": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[0-9a-fA-F]{24}$"
}
}
}
}
@@ -6,7 +6,6 @@ import { AppError } from "@/utils/AppError.js";
import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
import { IEmailService } from "@/service/emailService.js";
import { IProxiesService } from "@/domain/proxies/proxy.service.js";
import { IEgressStateService } from "@/domain/egress/egress-state.service.js";
import { Settings } from "@/domain/app-settings/app-settings.type.js";
export interface ISettingsController {
@@ -19,17 +18,10 @@ class SettingsController implements ISettingsController {
private settingsService: ISettingsService;
private emailService: IEmailService;
private proxiesService: IProxiesService;
private egressStateService: IEgressStateService;
constructor(
settingsService: ISettingsService,
emailService: IEmailService,
proxiesService: IProxiesService,
egressStateService: IEgressStateService
) {
constructor(settingsService: ISettingsService, emailService: IEmailService, proxiesService: IProxiesService) {
this.settingsService = settingsService;
this.emailService = emailService;
this.proxiesService = proxiesService;
this.egressStateService = egressStateService;
}
buildAppSettings = async (dbSettings: Settings) => {
@@ -78,14 +70,7 @@ class SettingsController implements ISettingsController {
}
}
const previousSettings = await this.settingsService.getDBSettings();
const updatedSettings = await this.settingsService.updateDbSettings(validatedBody);
// Switching the egress check on or off starts from a clean state: no degraded episode, no pending recovery job
if (validatedBody.egressCheckEnabled !== undefined && validatedBody.egressCheckEnabled !== previousSettings.egressCheckEnabled) {
await this.egressStateService.reset();
}
const returnSettings = await this.buildAppSettings(updatedSettings);
return res.status(200).json({
success: true,
@@ -1,17 +1,6 @@
import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js";
import { parseEgressTarget } from "@/utils/egressTarget.js";
import { z } from "zod";
// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL.
// Parsed with the same function the probe uses, so nothing the validator accepts can fail to probe.
const egressTargetValidation = z
.string()
.trim()
.min(1)
.max(253)
.refine((target) => parseEgressTarget(target) !== null, "Enter a hostname or IP address, host:port, or http(s) URL");
//****************************************
// Settings Validations
//****************************************
@@ -53,11 +42,6 @@ export const updateAppSettingsBodyValidation = z
.optional(),
globalProxyEnabled: z.boolean().optional(),
globalProxyId: z.string().nullable().optional(),
egressCheckEnabled: z.boolean().optional(),
egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS
egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(),
egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(),
})
.strip()
.superRefine((body, ctx) => {
+1 -6
View File
@@ -36,12 +36,7 @@ export const initializeControllers = (apiServices: ApiServices): InitializedCont
return {
authController: new AuthController(apiServices.userService),
monitorController: new MonitorController(apiServices.monitorService, apiServices.notificationsService),
settingsController: new SettingsController(
apiServices.settingsService,
apiServices.emailService,
apiServices.proxiesService,
apiServices.egressStateService
),
settingsController: new SettingsController(apiServices.settingsService, apiServices.emailService, apiServices.proxiesService),
checkController: new CheckController(apiServices.checkService),
geoCheckController: new GeoCheckController(apiServices.geoChecksService),
inviteController: new InviteController(apiServices.inviteService),
-5
View File
@@ -7,7 +7,6 @@ import { ITagsService, TagsService } from "@/domain/tags/tag.service.js";
import { IUserService, UserService } from "@/domain/users/user.service.js";
import { IJobScheduler } from "@/worker/worker.interface.js";
import { ProxiesService, IProxiesService } from "@/domain/proxies/proxy.service.js";
import { EgressStateService, IEgressStateService } from "@/domain/egress/egress-state.service.js";
import { SharedServices } from "@/config/services.shared.js";
// Third-party
@@ -25,7 +24,6 @@ export interface ApiServices extends SharedServices {
tagsService: ITagsService;
diagnosticService: IDiagnosticService;
proxiesService: IProxiesService;
egressStateService: IEgressStateService;
}
export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => {
@@ -50,7 +48,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
maintenanceWindowsRepository,
jobsRepository,
proxiesRepository,
egressStateRepository,
} = shared;
const userService = new UserService({
@@ -103,7 +100,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
const tagsService = new TagsService(tagsRepository, monitorsRepository);
const diagnosticService = new DiagnosticService(db);
const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService);
const egressStateService = new EgressStateService(egressStateRepository, jobsRepository);
return {
...shared,
worker: jobScheduler,
@@ -115,6 +111,5 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A
tagsService,
diagnosticService,
proxiesService,
egressStateService,
};
};
-5
View File
@@ -56,7 +56,6 @@ import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js";
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.js";
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
// Mongo repository implementations
import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js";
@@ -77,7 +76,6 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js";
import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js";
import MongoEgressStateRepository from "@/domain/egress/egress-state.repository.mongo.js";
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
// Shared infrastructure + business services that both the API and the worker process construct.
@@ -116,7 +114,6 @@ export interface SharedServices {
teamsRepository: ITeamsRepository;
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
proxiesRepository: IProxiesRepository;
egressStateRepository: IEgressStateRepository;
}
export const buildShared = async ({
@@ -160,7 +157,6 @@ export const buildShared = async ({
const teamsRepository = new MongoTeamsRepository();
const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
const proxiesRepository = new MongoProxiesRepository();
const egressStateRepository = new MongoEgressStateRepository();
// Inject settings repository into settings service (now that DB is connected)
(settingsService as SettingsService).setRepository(settingsRepository);
@@ -250,7 +246,6 @@ export const buildShared = async ({
teamsRepository,
maintenanceWindowsRepository,
proxiesRepository,
egressStateRepository,
};
return sharedServices;
};
+3 -8
View File
@@ -28,7 +28,6 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
import { DBQueueWorker } from "@/worker/worker.db-queue.js";
import { ProxyResolver } from "@/service/network/ProxyResolver.js";
import { IEgressService, EgressService } from "@/domain/egress/egress.service.js";
// Network providers
import { PingProvider } from "@/service/network/PingProvider.js";
import { HttpProvider } from "@/service/network/HttpProvider.js";
@@ -47,7 +46,6 @@ export interface WorkerServices {
networkService: INetworkService;
bufferService: IBufferService;
statusService: IStatusService;
egressService: IEgressService;
}
export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise<WorkerServices> => {
@@ -72,13 +70,12 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
teamsRepository,
maintenanceWindowsRepository,
proxiesRepository,
egressStateRepository,
} = shared;
// ***********************
// Network providers
// ***********************
const pingProvider = new PingProvider(ping, net);
const pingProvider = new PingProvider(ping);
const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
const hardwareProvider = new HardwareProvider(httpProvider);
@@ -106,7 +103,6 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository);
const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
const monitorStatusPolicy = new MonitorStatusPolicy();
const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger);
// ***********************
// Reactors and dispatcher
@@ -149,8 +145,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
checksRepository,
incidentsRepository,
geoChecksRepository,
dockerLogsRepository,
egressService
dockerLogsRepository
);
const worker = await DBQueueWorker.create({
@@ -190,5 +185,5 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig
});
}
return { worker, networkService, bufferService, statusService, egressService };
return { worker, networkService, bufferService, statusService };
};
@@ -6,7 +6,6 @@ export interface ISettingsRepository {
findSingleton(): Promise<Settings | null>;
// update
update(settings: SettingsUpdate): Promise<Settings>;
removeEgressNotification(notificationId: string): Promise<void>; // $pull, so it cannot race a concurrent settings update
// delete
deleteLegacy: () => Promise<boolean>;
}
@@ -1,6 +1,5 @@
import { Schema, model, type Types } from "mongoose";
import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js";
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
interface AppSettingsDocument extends Omit<Settings, "id" | "createdAt" | "updatedAt"> {
_id: Types.ObjectId;
@@ -43,10 +42,6 @@ const AppSettingsSchema = new Schema<AppSettingsDocument>(
globalThresholds: { type: thresholdsSchema },
globalProxyEnabled: { type: Boolean, default: false },
globalProxyId: { type: String, default: null },
egressCheckEnabled: { type: Boolean, default: false },
egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] },
egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS },
egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId
},
{ timestamps: true }
);
@@ -3,7 +3,6 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito
import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js";
import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js";
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js";
class MongoSettingsRepository implements ISettingsRepository {
private toEntity = (doc: AppSettingsDocument): Settings => {
@@ -31,10 +30,6 @@ class MongoSettingsRepository implements ISettingsRepository {
globalThresholds: doc.globalThresholds ?? undefined,
globalProxyEnabled: doc.globalProxyEnabled ?? false,
globalProxyId: doc.globalProxyId ?? undefined,
egressCheckEnabled: doc.egressCheckEnabled ?? false,
egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS],
egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
egressNotifications: doc.egressNotifications ?? [],
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
};
@@ -84,10 +79,6 @@ class MongoSettingsRepository implements ISettingsRepository {
return this.toEntity(updatedSettings);
};
removeEgressNotification = async (notificationId: string) => {
await AppSettingsModel.updateMany({ egressNotifications: notificationId }, { $pull: { egressNotifications: notificationId } });
};
deleteLegacy = async () => {
const res = await AppSettingsModel.deleteMany({ version: { $exists: false } });
return res.deletedCount > 0;
@@ -23,27 +23,15 @@ export interface ISettingsService {
getSettings(): EnvConfig;
areStatusPageThemesEnabled(): boolean;
getDBSettings(): Promise<Settings>;
getCachedDBSettings(): Promise<Settings>;
updateDbSettings(newSettings: SettingsUpdate): Promise<Settings>;
removeEgressNotification(notificationId: string): Promise<void>;
}
const DB_SETTINGS_CACHE_TTL_MS = 60_000;
type CachedSettings = {
value: Promise<Settings>;
expiresAt: number;
};
export class SettingsService implements ISettingsService {
static SERVICE_NAME = SERVICE_NAME;
private settings: EnvConfig;
private settingsRepository: ISettingsRepository | null = null;
private dbSettingsCache: CachedSettings | null = null;
constructor(
env: ValidatedEnv,
private dbSettingsTTLMs: number = DB_SETTINGS_CACHE_TTL_MS
) {
constructor(env: ValidatedEnv) {
this.settings = {
jwtSecret: env.JWT_SECRET,
jwtTTL: env.TOKEN_TTL as StringValue,
@@ -87,14 +75,7 @@ export class SettingsService implements ISettingsService {
}
updateDbSettings = async (newSettings: SettingsUpdate) => {
const updated = await this.getRepository().update(newSettings);
this.dbSettingsCache = null;
return updated;
};
removeEgressNotification = async (notificationId: string) => {
await this.getRepository().removeEgressNotification(notificationId);
this.dbSettingsCache = null;
return await this.getRepository().update(newSettings);
};
getDBSettings = async () => {
@@ -114,17 +95,4 @@ export class SettingsService implements ISettingsService {
return settings;
};
getCachedDBSettings = async () => {
const hit = this.dbSettingsCache;
if (hit && hit.expiresAt > Date.now()) {
return hit.value;
}
const entry: CachedSettings = { value: this.getDBSettings(), expiresAt: Date.now() + this.dbSettingsTTLMs };
this.dbSettingsCache = entry;
entry.value.catch(() => {
if (this.dbSettingsCache === entry) this.dbSettingsCache = null;
});
return entry.value;
};
}
@@ -51,11 +51,6 @@ export interface Settings {
globalThresholds?: SettingsThresholds;
globalProxyEnabled: boolean;
globalProxyId?: string | null;
// Egress self-check (see domain/egress)
egressCheckEnabled: boolean;
egressCheckTargets: string[];
egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule
egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert
createdAt: string;
updatedAt: string;
}
@@ -9,7 +9,6 @@ import type {
PageSpeedChecksResult,
UptimeChecksResult,
} from "@/domain/checks/check.type.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { CheckFilter, DateRange } from "@/types/query.js";
@@ -44,7 +43,7 @@ export interface IChecksRepository {
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult | DockerChecksResult>;
findDockerContainerChecks(monitorId: string, containerName: string, dateRange: DateRange): Promise<Omit<DockerContainerStats, "restartsInRange">>;
findSummaryByTeamId(teamId: string, dateRange: DateRange): Promise<ChecksSummary>;
findUnevaluatedByMonitorId(monitorId: string, pending: PendingCheck[]): Promise<Check[]>;
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
getDailyStatusBuckets(monitorIds: string[], days: number, timezone: string): Promise<DailyCheckBucket[]>;
// update
@@ -33,7 +33,6 @@ import {
getDockerTotalChecks,
getDockerUpChecks,
} from "@/domain/checks/check.docker.aggregation.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
const SERVICE_NAME = "ChecksRepository";
@@ -361,23 +360,10 @@ class MongoChecksRepository implements IChecksRepository {
};
};
findUnevaluatedByMonitorId = async (monitorId: string, pending: PendingCheck[]) => {
if (pending.length === 0) return [];
const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
// Bound the query to the pending time range so the monitorId + createdAt index is used
let from = Number.POSITIVE_INFINITY;
let to = Number.NEGATIVE_INFINITY;
for (const entry of pending) {
if (entry.createdAt < from) from = entry.createdAt;
if (entry.createdAt > to) to = entry.createdAt;
}
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
const docs = await CheckModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
createdAt: { $gte: new Date(from), $lte: new Date(to) },
_id: { $in: ids },
createdAt: { $gt: new Date(since) },
})
.sort({ createdAt: 1 })
.lean<CheckDocument[]>();
@@ -12,6 +12,7 @@ const SERVICE_NAME = "checkService";
export interface ICheckService {
createChecks(checks: Check[]): Promise<Check[]>;
toLastEvaluatedAt(check: Check): number;
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
getChecksByMonitor(params: {
@@ -54,6 +55,10 @@ export class CheckService implements ICheckService {
return this.checksRepository.createChecks(checks);
};
toLastEvaluatedAt = (check: Check) => {
return new Date(check.createdAt).getTime();
};
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
@@ -1,40 +0,0 @@
import { Schema, model, type Types } from "mongoose";
import { EgressStatuses, type EgressProbeResult, type EgressState } from "@/domain/egress/egress.type.js";
interface EgressStateDocument extends Omit<EgressState, "id" | "degradedSince" | "lastRecoveredAt" | "lastProbeAt" | "createdAt" | "updatedAt"> {
_id: Types.ObjectId;
singleton: boolean;
degradedSince: Date | null;
lastRecoveredAt: Date | null;
lastProbeAt: Date | null;
createdAt: Date;
updatedAt: Date;
}
const probeResultSchema = new Schema<EgressProbeResult>(
{
target: { type: String, required: true },
reachable: { type: Boolean, required: true },
responseTime: { type: Number, default: 0 },
message: { type: String },
},
{ _id: false }
);
const EgressStateSchema = new Schema<EgressStateDocument>(
{
singleton: { type: Boolean, required: true, unique: true, default: true },
status: { type: String, enum: EgressStatuses, required: true, default: "ok" },
degradedSince: { type: Date, default: null },
lastRecoveredAt: { type: Date, default: null },
lastProbeAt: { type: Date, default: null },
lastProbeResults: { type: [probeResultSchema], default: () => [] },
},
{ timestamps: true }
);
const EgressStateModel = model<EgressStateDocument>("EgressState", EgressStateSchema);
export type { EgressStateDocument };
export { EgressStateModel };
export default EgressStateModel;
@@ -1,13 +0,0 @@
import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
export interface IEgressStateRepository {
// read
findSingleton(): Promise<EgressState>; // creates the "ok" singleton on first call, a plain read after that
// update
recordProbe(results: EgressProbeResult[], now: Date): Promise<EgressState>;
reset(): Promise<EgressState>; // back to "ok" with no episode; used when the feature is toggled
// Atomic transitions. Each returns the updated state when this caller performed the transition,
// or null when the state was already on the target side (another worker got there first).
markDegraded(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
markRecovered(results: EgressProbeResult[], now: Date): Promise<EgressState | null>;
}
@@ -1,88 +0,0 @@
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js";
import { EgressStateModel, type EgressStateDocument } from "@/domain/egress/egress-state.model.js";
import { toStringId, toDateString } from "@/utils/mongoMappers.js";
const SERVICE_NAME = "EgressStateRepository";
class MongoEgressStateRepository implements IEgressStateRepository {
static SERVICE_NAME = SERVICE_NAME;
private toEntity = (doc: EgressStateDocument): EgressState => {
return {
id: toStringId(doc._id),
status: doc.status,
degradedSince: doc.degradedSince ? toDateString(doc.degradedSince) : null,
lastRecoveredAt: doc.lastRecoveredAt ? toDateString(doc.lastRecoveredAt) : null,
lastProbeAt: doc.lastProbeAt ? toDateString(doc.lastProbeAt) : null,
lastProbeResults: (doc.lastProbeResults ?? []).map((result) => ({
target: result.target,
reachable: result.reachable,
responseTime: result.responseTime ?? 0,
...(result.message !== undefined && { message: result.message }),
})),
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
};
};
findSingleton = async () => {
// Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write.
const existing = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
if (existing) return this.toEntity(existing);
try {
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
{ $setOnInsert: { singleton: true, status: "ok" } },
{ upsert: true, new: true, setDefaultsOnInsert: true }
).lean<EgressStateDocument>();
return this.toEntity(doc);
} catch (error: unknown) {
// Two processes can race the very first insert; the unique index rejects the loser, who reads the winner's row.
if ((error as { code?: number }).code === 11000) {
const doc = await EgressStateModel.findOne({ singleton: true }).lean<EgressStateDocument>();
if (doc) return this.toEntity(doc);
}
throw error;
}
};
reset = async () => {
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
{ $set: { status: "ok", degradedSince: null, lastProbeAt: null, lastProbeResults: [] }, $setOnInsert: { singleton: true } },
{ upsert: true, new: true, setDefaultsOnInsert: true }
).lean<EgressStateDocument>();
return this.toEntity(doc);
};
recordProbe = async (results: EgressProbeResult[], now: Date) => {
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true },
{ $set: { lastProbeAt: now, lastProbeResults: results }, $setOnInsert: { singleton: true, status: "ok" } },
{ upsert: true, new: true, setDefaultsOnInsert: true }
).lean<EgressStateDocument>();
return this.toEntity(doc);
};
markDegraded = async (results: EgressProbeResult[], now: Date) => {
// Only the caller that flips ok -> degraded sees a document back; everyone else sees null.
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true, status: "ok" },
{ $set: { status: "degraded", degradedSince: now, lastProbeAt: now, lastProbeResults: results } },
{ new: true }
).lean<EgressStateDocument>();
return doc ? this.toEntity(doc) : null;
};
markRecovered = async (results: EgressProbeResult[], now: Date) => {
const doc = await EgressStateModel.findOneAndUpdate(
{ singleton: true, status: "degraded" },
{ $set: { status: "ok", lastRecoveredAt: now, lastProbeAt: now, lastProbeResults: results } },
{ new: true }
).lean<EgressStateDocument>();
return doc ? this.toEntity(doc) : null;
};
}
export default MongoEgressStateRepository;
@@ -1,31 +0,0 @@
import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import type { EgressState } from "@/domain/egress/egress.type.js";
export interface IEgressStateService {
getState(): Promise<EgressState>;
reset(): Promise<EgressState>;
}
const SERVICE_NAME = "EgressStateService";
// API-process view of the instance-global egress state. Probe-driven transitions are owned by the
// worker (egress.service.ts); this exists so the controller never touches the repositories.
export class EgressStateService implements IEgressStateService {
static SERVICE_NAME = SERVICE_NAME;
constructor(
private egressStateRepository: IEgressStateRepository,
private jobsRepository: IJobsRepository
) {}
getState = async (): Promise<EgressState> => {
return this.egressStateRepository.findSingleton();
};
// Called when the feature is switched on or off: drop any pending recovery job and start from "ok".
reset = async (): Promise<EgressState> => {
await this.jobsRepository.deleteGlobalJob("egress");
return this.egressStateRepository.reset();
};
}
-283
View File
@@ -1,283 +0,0 @@
import type { Monitor } from "@/domain/monitors/monitor.type.js";
import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js";
import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { jobId, type Job, type JobSeed } from "@/domain/jobs/job.type.js";
import type { INetworkService } from "@/service/networkService.js";
import type { IProxyResolver } from "@/service/network/ProxyResolver.js";
import type { ILogger } from "@/utils/logger.js";
import {
DEFAULT_EGRESS_POLL_INTERVAL_SECONDS,
DEFAULT_EGRESS_TARGETS,
type EgressAssessment,
type EgressProbeResult,
type EgressStatus,
} from "@/domain/egress/egress.type.js";
import { timeRequest } from "@/service/network/utils.js";
import { parseEgressTarget, type EgressTarget } from "@/utils/egressTarget.js";
const SERVICE_NAME = "EgressService";
const PROBE_TIMEOUT_MS = 5000;
// How long a settled assessment answers for later failures. An assessment still in flight is shared
// for as long as it takes, so this only bounds how stale a reused answer can be.
const ASSESSMENT_REUSE_MS = 5000;
// Global job row (refId null) that re-probes while degraded. Inserted when egress degrades, removed on recovery.
const RECOVERY_JOB_TYPE = "egress" as const;
const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null);
// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code.
const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599;
// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs.
const PROBE_MONITOR_ID = "egress-probe";
const PROBE_TEAM_ID = "system";
export interface IEgressService {
probeTargets(targets: string[]): Promise<EgressProbeResult[]>;
// Called by the check producer when a check fails. Null when the feature is off or the assessment itself failed.
assessAfterFailure(): Promise<EgressStatus | null>;
// Handler for the "egress" job: re-probes while degraded and removes the job once egress is back.
checkRecovery(job: Job): Promise<void>;
}
type CachedAssessment = {
value: Promise<EgressAssessment | null>;
expiresAt: number | null; // null while the assessment is still in flight
};
export class EgressService implements IEgressService {
static SERVICE_NAME = SERVICE_NAME;
private assessment: CachedAssessment | null = null;
constructor(
private settingsService: ISettingsService,
private egressStateRepository: IEgressStateRepository,
private jobsRepository: IJobsRepository,
private networkService: INetworkService,
private proxyResolver: IProxyResolver,
private logger: ILogger
) {}
// ****************************
// Probing
// ****************************
// Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor.
// proxyMode "inherit" lets the resolver apply the global proxy to http probes, as it does for monitors.
private toProbeMonitor = (target: EgressTarget, raw: string): Monitor => {
const base = {
id: PROBE_MONITOR_ID,
teamId: PROBE_TEAM_ID,
name: `Egress probe: ${raw}`,
method: "GET",
proxyMode: "inherit",
useAdvancedMatching: false,
ignoreTlsErrors: false,
customUpCodes: [],
};
switch (target.kind) {
case "http":
return { ...base, type: "http", url: target.url } as unknown as Monitor;
case "port":
return { ...base, type: "port", url: target.host, port: target.port } as unknown as Monitor;
case "ping":
return { ...base, type: "ping", url: target.host } as unknown as Monitor;
}
};
private probeTarget = async (raw: string): Promise<EgressProbeResult> => {
const target = parseEgressTarget(raw);
if (!target) {
// Validation rejects these; a stored value can still be stale, and it must not count as reachable.
return { target: raw, reachable: false, responseTime: 0, message: "Invalid egress target" };
}
const monitor = this.toProbeMonitor(target, raw);
let timer: NodeJS.Timeout | undefined;
const { response, responseTime, error } = await timeRequest(async () => {
const proxyUrl = await this.proxyResolver.resolve(monitor);
return Promise.race([
this.networkService.requestStatus(monitor, { proxyUrl }),
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS);
}),
]).finally(() => clearTimeout(timer));
});
if (error || !response) {
return {
target: raw,
reachable: false,
responseTime,
message: error instanceof Error ? error.message : error ? String(error) : "No response",
};
}
return {
target: raw,
// Any HTTP response, including 4xx/5xx, proves the instance can reach the target; only a transport
// failure counts against egress. Ping and port providers have no such distinction.
reachable: response.status === true || (target.kind === "http" && isHttpStatusCode(response.code)),
responseTime: response.responseTime ?? responseTime,
message: response.message,
};
};
probeTargets = async (targets: string[]): Promise<EgressProbeResult[]> => {
return await Promise.all(targets.map((target) => this.probeTarget(target)));
};
private resolveTargets = (configured: string[] | undefined): string[] => {
const targets = (configured ?? []).map((target) => target.trim()).filter((target) => target.length > 0);
// An empty list would make "all unreachable" vacuously true, so fall back to the defaults.
return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS];
};
private toIntervalMs = (seconds: number | undefined): number => {
const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0;
return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000;
};
// ****************************
// Event-triggered assessment
// ****************************
assessAfterFailure = async (): Promise<EgressStatus | null> => {
const assessment = await this.sharedAssessment();
return assessment === "disabled" ? null : assessment;
};
// When egress is down every monitor fails at once. Concurrent failures share the in-flight assessment
// however long it takes, and a settled one answers for ASSESSMENT_REUSE_MS. Same shape as ProxyResolver's cache.
private sharedAssessment = (): Promise<EgressAssessment | null> => {
const cached = this.assessment;
if (cached && (cached.expiresAt === null || cached.expiresAt > Date.now())) {
return cached.value;
}
const entry: CachedAssessment = { value: this.assess(), expiresAt: null };
this.assessment = entry;
void entry.value.then((result) => {
if (this.assessment !== entry) return;
if (result === null) {
this.assessment = null; // an internal failure is not cached, so the next failing check retries
} else {
entry.expiresAt = Date.now() + ASSESSMENT_REUSE_MS;
}
});
return entry.value;
};
private assess = async (): Promise<EgressAssessment | null> => {
try {
const settings = await this.settingsService.getCachedDBSettings();
if (!settings.egressCheckEnabled) {
return "disabled";
}
const state = await this.egressStateRepository.findSingleton();
if (state.status === "degraded") {
// Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false);
return "degraded";
}
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
const now = new Date();
if (results.some((result) => result.reachable)) {
await this.egressStateRepository.recordProbe(results, now);
return "ok";
}
// Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row
// while ok, which the job's first run removes, rather than a degraded state that nothing is polling.
await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true);
const degraded = await this.egressStateRepository.markDegraded(results, now);
this.logger.warn({
message: degraded
? "Instance egress degraded: every reliability target is unreachable. Monitor failures will not be counted until it recovers"
: "Instance egress degraded (transition already recorded by another worker)",
service: SERVICE_NAME,
method: "assessAfterFailure",
details: { results },
});
return "degraded";
} catch (error: unknown) {
// The egress check must never break check production; treat an internal failure as "unknown" and carry on.
this.logger.error({
message: error instanceof Error ? error.message : String(error),
service: SERVICE_NAME,
method: "assessAfterFailure",
stack: error instanceof Error ? error.stack : undefined,
});
return null;
}
};
// ****************************
// Recovery job
// ****************************
// reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing,
// so re-arming while degraded never pushes a pending run back.
private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => {
const intervalMs = this.toIntervalMs(pollIntervalSeconds);
const seed: JobSeed = {
id: RECOVERY_JOB_ID,
type: RECOVERY_JOB_TYPE,
refId: null,
isActive: true,
nextScheduledAt: Date.now() + intervalMs,
intervalMs,
};
if (reschedule) {
await this.jobsRepository.upsertCleanupJob(seed); // global-row upsert that also sets the schedule
} else {
await this.jobsRepository.upsertJob(seed);
}
};
// Removes the row only if nobody has rescheduled it since this run claimed it. Another worker may have seen
// egress fail again in the meantime and started a new episode; its schedule must survive.
private releaseRecoveryJob = async (job: Job) => {
await this.jobsRepository.deleteGlobalJobIfUnchanged(RECOVERY_JOB_TYPE, job.nextScheduledAt);
};
// Runs on the queue at the configured interval while degraded. Errors propagate so the queue records
// the failure and retries with its usual backoff.
checkRecovery = async (job: Job): Promise<void> => {
const settings = await this.settingsService.getCachedDBSettings();
const state = await this.egressStateRepository.findSingleton();
if (!settings.egressCheckEnabled || state.status !== "degraded") {
// Feature switched off, recovery recorded by another worker, or state reset: nothing left to poll for.
await this.releaseRecoveryJob(job);
return;
}
const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets));
const now = new Date();
if (!results.some((result) => result.reachable)) {
await this.egressStateRepository.recordProbe(results, now);
return;
}
const recovered = await this.egressStateRepository.markRecovered(results, now);
await this.releaseRecoveryJob(job);
if (!recovered) {
// Another process performed the transition.
return;
}
this.logger.info({
message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`,
service: SERVICE_NAME,
method: "checkRecovery",
details: { results },
});
};
}
-35
View File
@@ -1,35 +0,0 @@
// Egress self-check: detects loss of the instance's own outbound connectivity so that
// a burst of simultaneous monitor failures is not reported as a burst of real outages.
export const EgressStatuses = ["ok", "degraded"] as const;
export type EgressStatus = (typeof EgressStatuses)[number];
// Outcome of one assessment: a status, or "disabled" when the feature is switched off (distinct from an internal failure).
export type EgressAssessment = EgressStatus | "disabled";
// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect,
// http(s):// URLs are fetched. Any one target being reachable means egress is fine.
export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const;
export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30;
export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5;
export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600;
export const MAX_EGRESS_TARGETS = 10;
// Instance-global singleton persisted in MongoDB so that every worker process and the API
// process share one view of the state (the queue worker may run separately from the API).
export interface EgressState {
id: string;
status: EgressStatus;
degradedSince: string | null; // ISO timestamp of the current or most recent degraded entry
lastRecoveredAt: string | null; // ISO timestamp of the most recent recovery
lastProbeAt: string | null; // ISO timestamp of the most recent reliability probe
lastProbeResults: EgressProbeResult[];
createdAt: string;
updatedAt: string;
}
export interface EgressProbeResult {
target: string;
reachable: boolean;
responseTime: number; // ms
message?: string;
}
-12
View File
@@ -47,18 +47,6 @@ const JobSchema = new Schema<JobDocument>(
default: null,
},
// Pending checks, inserted but not yet evaluated
pendingChecks: {
type: [
{
checkId: { type: String, required: true },
createdAt: { type: Number, required: true },
},
],
default: [],
_id: false,
},
// Observability
runCount: {
type: Number,
@@ -1,4 +1,4 @@
import { Job, JobSeed, JobType, PendingCheck } from "@/domain/jobs/job.type.js";
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
export type JobPageQuery = {
page?: number;
@@ -32,12 +32,9 @@ export interface IJobsRepository {
recordOneShot(id: string, now: number): Promise<boolean>;
// ********************
// Hand off to evaluator // This is accomplished pushing newly inserted Check IDs onto the evaluation row
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
// ********************
upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
// Removes processed checks from the evaluate row. Returns false when this worker no longer holds the lease.
pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
// ********************
// Job CRUD
@@ -64,11 +61,6 @@ export interface IJobsRepository {
// Delete a single job of type
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
// Global rows (refId null) are addressed by their canonical _id, see jobId
deleteGlobalJob(type: JobType): Promise<boolean>;
// Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged)
deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise<boolean>;
// Delete jobs that reference monitors that no longer exist
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
+12 -45
View File
@@ -1,6 +1,6 @@
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId, PendingCheck } from "@/domain/jobs/job.type.js";
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
// Cap for rearm interval, never want to wait more than 15s.
const REARM_JITTER_MAX_MS = 15_000;
@@ -18,7 +18,6 @@ class MongoJobsRepository implements IJobsRepository {
intervalMs: doc.intervalMs,
lockedBy: doc.lockedBy,
lockedUntil: doc.lockedUntil,
pendingChecks: doc.pendingChecks ?? [],
runCount: doc.runCount,
failCount: doc.failCount,
lastFinishedAt: doc.lastFinishedAt,
@@ -108,8 +107,6 @@ class MongoJobsRepository implements IJobsRepository {
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
failCount: 0,
lastFailReason: null,
},
$inc: {
runCount: 1, // Increment run count
@@ -147,27 +144,22 @@ class MongoJobsRepository implements IJobsRepository {
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
[
{
$set: {
// Re-arm if more checks were pushed while this run held the lease; otherwise park
nextScheduledAt: {
$cond: [{ $gt: [{ $size: { $ifNull: ["$pendingChecks", []] } }, 0] }, now, PARKED],
},
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
failCount: 0,
lastFailReason: null,
runCount: { $add: ["$runCount", 1] },
},
{
$set: {
nextScheduledAt: PARKED,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
},
]
$inc: {
runCount: 1, // Increment run count
},
}
);
return res.modifiedCount === 1;
};
upsertEvaluate = async (monitorId: string, pending: PendingCheck[], now: number) => {
upsertEvaluate = async (monitorId: string, now: number) => {
const filter = { type: "evaluate", refId: monitorId };
const update = {
$setOnInsert: {
@@ -185,9 +177,6 @@ class MongoJobsRepository implements IJobsRepository {
$min: {
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
},
$addToSet: {
pendingChecks: { $each: pending }, // accumulate pending checks; a retried arm never duplicates an entry
},
};
try {
const res = await JobModel.updateOne(filter, update, { upsert: true });
@@ -202,18 +191,6 @@ class MongoJobsRepository implements IJobsRepository {
throw error;
}
};
// Returns whether this worker still holds the lease. false means another worker has claimed the row and the caller must stop.
pullEvaluated = async (id: string, checkIds: string[]) => {
if (checkIds.length === 0) return true;
const res = await JobModel.updateOne(
{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
);
return res.matchedCount === 1;
};
upsertJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
@@ -301,16 +278,6 @@ class MongoJobsRepository implements IJobsRepository {
return res.deletedCount > 0;
};
deleteGlobalJob = async (type: JobType) => {
const res = await JobModel.deleteOne({ _id: jobId(type, null) });
return res.deletedCount > 0;
};
deleteGlobalJobIfUnchanged = async (type: JobType, nextScheduledAt: number) => {
const res = await JobModel.deleteOne({ _id: jobId(type, null), nextScheduledAt });
return res.deletedCount > 0;
};
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
+2 -10
View File
@@ -1,15 +1,10 @@
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"] as const;
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
export type JobType = (typeof JOB_TYPES)[number];
export const LOCK_MS = 60_000;
export const BACKOFF_MS = 5_000;
export const PARKED = Number.MAX_SAFE_INTEGER;
export type PendingCheck = {
checkId: string;
createdAt: number; // epoch ms, copied from Check.createdAt
};
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
export type Job = {
@@ -26,9 +21,6 @@ export type Job = {
lockedBy: string | null; // workerId holding the lease
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
// Evaluate these rows only. These are checks that are inserted but not yet evaluated
pendingChecks: PendingCheck[];
// observability (replaces what getMetrics/getJobs)
runCount: number;
failCount: number;
@@ -259,7 +259,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature];
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
const threshold = monitor.tempAlertThreshold;
if (maxTemp > threshold) {
if (maxTemp >= threshold) {
breaches.push({
metric: "temp",
currentValue: maxTemp,
@@ -179,9 +179,6 @@ export class NotificationsService implements INotificationsService {
deleteById = async (id: string, teamId: string): Promise<Notification> => {
await this.monitorsRepository.removeNotificationFromMonitors(id);
const deleted = await this.notificationsRepository.deleteById(id, teamId);
// App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles.
// After the team-scoped delete, so a request for another team's id (404 above) cannot touch instance settings.
await this.settingsService.removeEgressNotification(id);
return deleted;
};
}
+6 -40
View File
@@ -7,14 +7,12 @@ import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"
import { ICheckService } from "../domain/checks/check.service.js";
import { DockerLog } from "@/domain/docker/docker-log.type.js";
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
const SERVICE_NAME = "BufferService";
export interface IBufferService {
addToBuffer(check: Check): void;
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
addDockerLogToBuffer(dockerLog: DockerLog): void;
ingestChecks(checks: Check[]): Promise<void>;
scheduleNextFlush(): void;
flushBuffer(): Promise<void>;
flushGeoBuffer(): Promise<void>;
@@ -36,9 +34,6 @@ export class BufferService implements IBufferService {
private dockerLogsService: IDockerLogsService;
private jobsRepository: IJobsRepository;
// Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated.
private unarmed: Map<string, PendingCheck[]> = new Map();
constructor(
logger: ILogger,
checkService: ICheckService,
@@ -104,38 +99,6 @@ export class BufferService implements IBufferService {
}
}
ingestChecks = async (checks: Check[]) => {
if (checks.length > 0) await this.checksService.createChecks(checks);
// Group by monitor, starting from any entries a previous flush failed to arm
const byMonitor = this.unarmed;
this.unarmed = new Map();
for (const check of checks) {
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
byMonitor.set(check.metadata.monitorId, pending);
}
if (byMonitor.size === 0) return;
const now = Date.now();
const ops: Promise<boolean>[] = [];
for (const [monitorId, pendingChecks] of byMonitor) {
ops.push(
this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
// The checks are already stored, so a failed arm is retried on the next flush rather than dropped
this.unarmed.set(monitorId, pendingChecks);
this.logger.error({
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`,
service: this.SERVICE_NAME,
method: "ingestChecks",
});
return false;
})
);
}
await Promise.all(ops);
};
scheduleNextFlush() {
if (this.bufferTimer) {
clearTimeout(this.bufferTimer);
@@ -158,9 +121,8 @@ export class BufferService implements IBufferService {
}
}, this.BUFFER_TIMEOUT);
}
async flushBuffer() {
if (this.buffer.length === 0 && this.unarmed.size === 0) {
if (this.buffer.length === 0) {
return;
}
// Take the batch first so a write that drains it can't be appended to mid-flush
@@ -172,7 +134,11 @@ export class BufferService implements IBufferService {
service: this.SERVICE_NAME,
method: "flushBuffer",
});
await this.ingestChecks(batch);
await this.checksService.createChecks(batch);
// Need to evaluate checks when they are flushed
const monitorIds = [...new Set(batch.map((check) => check.metadata.monitorId))];
const now = Date.now();
await Promise.all(monitorIds.map((monitorId) => this.jobsRepository.upsertEvaluate(monitorId, now)));
} catch (error: unknown) {
this.logger.error({
message: error instanceof Error ? error.message : "Unknown error",
@@ -22,7 +22,7 @@ export class PageSpeedProvider implements IStatusProvider<PageSpeedStatusPayload
const { url, strategy } = monitor;
try {
if (!url) throw new Error("URL is required for PageSpeed monitor");
const dbSettings = await this.settingsService.getCachedDBSettings();
const dbSettings = await this.settingsService.getDBSettings();
const apiKey = dbSettings?.pagespeedApiKey;
const resolvedStrategy = strategy ?? DefaultPageSpeedStrategy;
let pageSpeedUrl = `https://pagespeedonline.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(
+5 -13
View File
@@ -4,33 +4,25 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js";
import { MonitorStatusResponse } from "@/types/network.js";
import { AppError } from "@/utils/AppError.js";
import ping from "ping";
import * as net from "net";
import { timeRequest } from "@/service/network/utils.js";
const SERVICE_NAME = "PingProvider";
type Ping = typeof ping;
type NetType = typeof net;
export class PingProvider implements IStatusProvider<PingStatusPayload> {
readonly type = "ping";
constructor(
private ping: Ping,
private net: NetType
) {}
constructor(private ping: Ping) {}
supports(type: MonitorType): boolean {
return type === "ping";
}
private sanitizeHost(url: string): string {
const host = url.replace(/^https?:\/\//, "").replace(/\/.*$/, "");
// "[2001:db8::1]" or "[2001:db8::1]:53": the address is inside the brackets
const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host);
if (bracketed) return bracketed[1]!;
// A bare IPv6 address contains colons that are not a port separator
if (this.net.isIPv6(host)) return host;
return host.replace(/:.*/, "");
return url
.replace(/^https?:\/\//, "")
.replace(/\/.*$/, "")
.replace(/:.*/, "");
}
async handle(monitor: Monitor): Promise<MonitorStatusResponse<PingStatusPayload>> {
+1 -1
View File
@@ -73,7 +73,7 @@ export class ProxyResolver implements IProxyResolver {
};
private resolveInherit = async (monitor: Monitor): Promise<string | undefined> => {
const settings = await this.settingsService.getCachedDBSettings();
const settings = await this.getCached("settings", () => this.settingsService.getDBSettings());
if (!settings.globalProxyEnabled) {
return undefined;
}
-48
View File
@@ -1,48 +0,0 @@
import { z } from "zod";
import { dnsHostnameRegex } from "@/api/validation/shared.js";
// A reliability target for the egress self-check, in one of three shapes. Parsed here so the
// settings validator and the probe agree on exactly what is accepted.
export type EgressTarget =
| { kind: "http"; url: string } // http(s):// URL, fetched
| { kind: "port"; host: string; port: number } // host:port or [ipv6]:port, TCP connect
| { kind: "ping"; host: string }; // bare hostname or IP address, ICMP
// "[ipv6]:port" or "host:port" where host has no colons, so a bare IPv6 address is never read as host:port.
const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s[\]]+)):(\d{1,5})$/;
const BRACKETED_PATTERN = /^\[([^\]]+)\]$/;
const isIpv6 = (value: string): boolean => z.ipv6().safeParse(value).success;
const isHostOrIpv4 = (value: string): boolean => value === "localhost" || z.ipv4().safeParse(value).success || dnsHostnameRegex.test(value);
const parseHttp = (target: string): EgressTarget | null => {
try {
const { protocol, hostname } = new URL(target);
return (protocol === "http:" || protocol === "https:") && hostname.length > 0 ? { kind: "http", url: target } : null;
} catch {
return null;
}
};
export const parseEgressTarget = (raw: string): EgressTarget | null => {
const target = raw.trim();
if (target.length === 0) return null;
if (/^https?:\/\//i.test(target)) return parseHttp(target);
const hostPort = HOST_PORT_PATTERN.exec(target);
if (hostPort) {
const port = Number(hostPort[3]);
if (port < 1 || port > 65535) return null;
const bracketedHost = hostPort[1];
if (bracketedHost !== undefined) return isIpv6(bracketedHost) ? { kind: "port", host: bracketedHost, port } : null;
const host = hostPort[2]!;
return isHostOrIpv4(host) ? { kind: "port", host, port } : null;
}
const bracketed = BRACKETED_PATTERN.exec(target);
if (bracketed) return isIpv6(bracketed[1]!) ? { kind: "ping", host: bracketed[1]! } : null;
if (isIpv6(target) || isHostOrIpv4(target)) return { kind: "ping", host: target };
return null;
};
+4 -31
View File
@@ -28,7 +28,6 @@ const CONCURRENCY: Record<JobType, number> = {
evaluate: 20,
"cleanup-orphaned": 1,
"cleanup-retention": 1,
egress: 1,
};
export interface DBQueueWorkerDependencies {
@@ -62,7 +61,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
evaluate: 0,
"cleanup-orphaned": 0,
"cleanup-retention": 0,
egress: 0,
};
private checksRepository: IChecksRepository;
@@ -120,41 +118,19 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
// ********************
private runEvaluate = async (job: Job) => {
if (!job.refId || job.pendingChecks.length === 0) return;
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
if (!job.refId) return;
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
if (!monitor) {
await this.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
return;
}
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
if (!monitor) return;
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
let current = monitor;
for (const check of checks) {
const status = this.checkService.toStatusResponse(check);
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]);
if (!leaseHeld) {
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" });
return;
}
await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
}
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
const found = new Set(checks.map((check) => check.id));
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
if (missing.length > 0) {
this.logger.warn({
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
service: SERVICE_NAME,
method: "runEvaluate",
});
await this.jobsRepository.pullEvaluated(job.id, missing);
}
};
private runGeoCheck = async (job: Job) => {
@@ -199,9 +175,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
case "cleanup-retention":
await this.helper.getCleanupRetentionJob()(); // Get job and execute
break;
case "egress":
await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery
break;
}
if (job.intervalMs === null) {
+1 -15
View File
@@ -11,13 +11,10 @@ import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
import { ILogger } from "@/utils/logger.js";
import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js";
import { IEgressService } from "@/domain/egress/egress.service.js";
import type { Job } from "@/domain/jobs/job.type.js";
export interface IWorkerHelper {
getCleanupOrphanedJob(): () => Promise<void>;
getCleanupRetentionJob(): () => Promise<void>;
getEgressRecoveryJob(): (job: Job) => Promise<void>;
}
export interface MonitorActionDecision {
@@ -48,7 +45,6 @@ export class WorkerHelper implements IWorkerHelper {
private incidentsRepository: IIncidentsRepository;
private geoChecksRepository: IGeoChecksRepository;
private dockerLogsRepository: IDockerLogsRepository;
private egressService: IEgressService;
constructor(
logger: ILogger,
@@ -61,8 +57,7 @@ export class WorkerHelper implements IWorkerHelper {
checksRepository: IChecksRepository,
incidentsRepository: IIncidentsRepository,
geoChecksRepository: IGeoChecksRepository,
dockerLogsRepository: IDockerLogsRepository,
egressService: IEgressService
dockerLogsRepository: IDockerLogsRepository
) {
this.logger = logger;
this.checkService = checkService;
@@ -75,17 +70,8 @@ export class WorkerHelper implements IWorkerHelper {
this.incidentsRepository = incidentsRepository;
this.geoChecksRepository = geoChecksRepository;
this.dockerLogsRepository = dockerLogsRepository;
this.egressService = egressService;
}
// Runs while instance egress is degraded; the service removes the row once egress is back.
// Errors propagate so the queue records the failure and retries with its usual backoff.
getEgressRecoveryJob = () => {
return async (job: Job) => {
await this.egressService.checkRecovery(job);
};
};
getCleanupOrphanedJob = () => {
return async () => {
try {
@@ -41,7 +41,6 @@ export class JobScheduler implements IJobScheduler {
evaluate: POLL_MS,
"cleanup-orphaned": POLL_MS,
"cleanup-retention": POLL_MS,
egress: POLL_MS,
};
// Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd.
protected toCheckJob = (monitor: MonitorScheduleFields, now: number, immediate = false): JobSeed => ({
@@ -1,130 +0,0 @@
import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals";
import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoEgressStateRepository from "../../src/domain/egress/egress-state.repository.mongo.ts";
import { EgressStateModel } from "../../src/domain/egress/egress-state.model.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The transitions here are the guarantees several worker processes rely on: exactly one
// caller flips ok -> degraded (and degraded -> ok), and a plain read never writes. Both are
// MongoDB filter/update semantics, so they need a live engine.
let mongod: MongoMemoryServer;
beforeAll(async () => {
mongod = await MongoMemoryServer.create();
await mongoose.connect(mongod.getUri());
await EgressStateModel.init(); // unique index on singleton
}, 120_000);
afterAll(async () => {
await mongoose.disconnect();
await mongod.stop();
});
beforeEach(async () => {
await EgressStateModel.deleteMany({});
});
const probe = (target: string, reachable: boolean) => ({ target, reachable, responseTime: 5, message: reachable ? "Success" : "timeout" });
const NOW = new Date("2026-01-01T10:00:00.000Z");
const LATER = new Date("2026-01-01T10:05:00.000Z");
describe("MongoEgressStateRepository", () => {
const repo = new MongoEgressStateRepository();
describe("findSingleton", () => {
it("creates the ok singleton on first call and returns the same document afterwards", async () => {
const first = await repo.findSingleton();
const second = await repo.findSingleton();
expect(first.status).toBe("ok");
expect(second.id).toBe(first.id);
expect(await EgressStateModel.countDocuments()).toBe(1);
});
it("does not bump updatedAt on a read", async () => {
const created = await repo.findSingleton();
await new Promise((resolve) => setTimeout(resolve, 20));
const read = await repo.findSingleton();
expect(read.updatedAt).toBe(created.updatedAt);
});
it("survives two processes racing the first insert", async () => {
const [a, b] = await Promise.all([repo.findSingleton(), repo.findSingleton()]);
expect(a.id).toBe(b.id);
expect(await EgressStateModel.countDocuments()).toBe(1);
});
});
describe("markDegraded / markRecovered", () => {
it("markDegraded does nothing when the singleton has never been created (callers read it first)", async () => {
expect(await repo.markDegraded([probe("1.1.1.1", false)], NOW)).toBeNull();
expect(await EgressStateModel.countDocuments()).toBe(0);
});
it("lets exactly one caller perform each transition", async () => {
await repo.findSingleton();
const [d1, d2] = await Promise.all([repo.markDegraded([probe("1.1.1.1", false)], NOW), repo.markDegraded([probe("1.1.1.1", false)], NOW)]);
expect([d1, d2].filter((result) => result !== null)).toHaveLength(1);
expect((await repo.findSingleton()).status).toBe("degraded");
const [r1, r2] = await Promise.all([repo.markRecovered([probe("1.1.1.1", true)], LATER), repo.markRecovered([probe("1.1.1.1", true)], LATER)]);
expect([r1, r2].filter((result) => result !== null)).toHaveLength(1);
const state = await repo.findSingleton();
expect(state.status).toBe("ok");
expect(state.degradedSince).toBe(NOW.toISOString());
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
});
it("markRecovered is a no-op while ok and markDegraded is a no-op while degraded", async () => {
await repo.findSingleton();
expect(await repo.markRecovered([probe("1.1.1.1", true)], NOW)).toBeNull();
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
expect(await repo.markDegraded([probe("1.1.1.1", false)], LATER)).toBeNull();
expect((await repo.findSingleton()).degradedSince).toBe(NOW.toISOString());
});
});
describe("recordProbe", () => {
it("stores the probe without changing the status", async () => {
await repo.findSingleton();
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
const state = await repo.recordProbe([probe("1.1.1.1", false), probe("8.8.8.8", false)], LATER);
expect(state.status).toBe("degraded");
expect(state.lastProbeAt).toBe(LATER.toISOString());
expect(state.lastProbeResults.map((result) => result.target)).toEqual(["1.1.1.1", "8.8.8.8"]);
});
});
describe("reset", () => {
it("returns a degraded singleton to ok with no episode and keeps the recovery history", async () => {
await repo.findSingleton();
await repo.markDegraded([probe("1.1.1.1", false)], NOW);
await repo.markRecovered([probe("1.1.1.1", true)], LATER);
await repo.markDegraded([probe("1.1.1.1", false)], LATER);
const state = await repo.reset();
expect(state.status).toBe("ok");
expect(state.degradedSince).toBeNull();
expect(state.lastProbeAt).toBeNull();
expect(state.lastProbeResults).toEqual([]);
expect(state.lastRecoveredAt).toBe(LATER.toISOString());
});
it("creates the singleton when none exists", async () => {
const state = await repo.reset();
expect(state.status).toBe("ok");
expect(await EgressStateModel.countDocuments()).toBe(1);
});
});
});
@@ -1,200 +0,0 @@
import { describe, expect, it, jest, beforeAll, afterAll, beforeEach, afterEach } from "@jest/globals";
import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
import { CheckModel } from "../../src/domain/checks/check.model.ts";
import JobModel from "../../src/domain/jobs/job.model.ts";
import { CheckService } from "../../src/domain/checks/check.service.ts";
import { BufferService } from "../../src/service/bufferService.ts";
import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts";
import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts";
import { createMockLogger } from "../helpers/createMockLogger.ts";
import type { Job } from "../../src/domain/jobs/job.type.ts";
import type { Check } from "../../src/domain/checks/check.type.ts";
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
import type { MonitorStatusResponse } from "../../src/types/network.ts";
import type { ILogger } from "../../src/utils/logger.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// Regression test for the dropped-check bug in the check-to-evaluate handoff.
//
// The evaluate step used to be driven by a timestamp cursor: the evaluator read checks with
// createdAt strictly after the monitor's lastEvaluatedAt and advanced that cursor to each
// check's createdAt. Each worker's BufferService flushes its own in-memory batch on its own
// timer, so with two processing workers a later check produced on worker B could be flushed and
// evaluated before an earlier check produced on worker A had been flushed. When A's batch landed,
// its check was older than the cursor and the evaluate query never returned it. The check existed
// in the collection but never reached the status window, stats, incidents or notifications.
//
// The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each
// is applied, so a late batch is always evaluated. This drives the real time-series query, the
// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an
// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's.
let mongod: MongoMemoryServer;
beforeAll(async () => {
mongod = await MongoMemoryServer.create();
await mongoose.connect(mongod.getUri());
await CheckModel.createCollection(); // timeseries collections must exist before insert
await JobModel.init();
}, 120_000);
afterAll(async () => {
await mongoose.disconnect();
await mongod.stop();
});
beforeEach(async () => {
await CheckModel.deleteMany({});
await JobModel.deleteMany({});
});
// ── Helpers ──────────────────────────────────────────────────────────────────
const MONITOR_ID = new mongoose.Types.ObjectId().toString();
const TEAM_ID = new mongoose.Types.ObjectId().toString();
const T0 = Date.parse("2026-01-01T00:00:00.000Z");
const INTERVAL_MS = 30_000;
const makeMonitor = (): Monitor =>
({
id: MONITOR_ID,
teamId: TEAM_ID,
type: "http",
interval: INTERVAL_MS,
status: "up",
isActive: true,
statusWindow: [],
statusWindowSize: 5,
statusWindowThreshold: 60,
recentChecks: [],
lastEvaluatedAt: 0,
}) as unknown as Monitor;
const makeStatusResponse = (): MonitorStatusResponse => ({
monitorId: MONITOR_ID,
teamId: TEAM_ID,
type: "http",
status: true,
code: 200,
message: "OK",
responseTime: 100,
});
// Production stamps createdAt at the moment the probe ran, on the producing worker; pin it so
// the two workers' checks have a known order regardless of wall clock.
const producedAt = (check: Check, at: number): Check => {
const createdAt = new Date(at).toISOString();
return { ...check, createdAt, updatedAt: createdAt };
};
const stubSettingsService = { getSettings: () => ({ nodeEnv: "production" }) };
const stubGeoChecksService = { createGeoChecks: jest.fn<any>().mockResolvedValue([]) };
const stubDockerLogsService = { createDockerLogs: jest.fn<any>().mockResolvedValue(0) };
describe("Evaluate cursor with more than one processing worker", () => {
let logger: ILogger;
let monitorsRepository: InMemoryMonitorsRepository;
let checksRepository: MongoChecksRepository;
let checkService: CheckService;
let bufferA: BufferService;
let bufferB: BufferService;
let evaluatedCheckIds: string[];
let evaluator: DBQueueWorker;
let evaluatorJobsRepository: MongoJobsRepository;
// One BufferService per worker process, each flushing into the same collection and jobs table.
const createWorkerBuffer = (workerId: string) =>
new BufferService(
logger,
checkService,
stubGeoChecksService as any,
stubDockerLogsService as any,
stubSettingsService as any,
new MongoJobsRepository(workerId)
);
// Claim every due evaluate job and run it through the worker's real per-job path.
const runDueEvaluateJobs = async () => {
const jobs = await evaluatorJobsRepository.claimDueBatch("evaluate", 20, Date.now());
const runJob = (evaluator as unknown as { runJob: (job: Job) => Promise<void> }).runJob;
for (const job of jobs) await runJob(job);
return jobs.length;
};
beforeEach(() => {
logger = createMockLogger() as unknown as ILogger;
monitorsRepository = new InMemoryMonitorsRepository();
monitorsRepository.seed(makeMonitor());
checksRepository = new MongoChecksRepository(logger);
checkService = new CheckService(monitorsRepository, logger, checksRepository);
bufferA = createWorkerBuffer("worker-a");
bufferB = createWorkerBuffer("worker-b");
evaluatedCheckIds = [];
evaluatorJobsRepository = new MongoJobsRepository("worker-b");
evaluator = new DBQueueWorker({
logger,
isDbConnected: () => true,
jobsRepository: evaluatorJobsRepository,
monitorsRepository,
checksRepository,
checkService,
bufferService: bufferB,
checkProducer: { produce: jest.fn<any>() },
checkEvaluator: {
evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
evaluatedCheckIds.push(check.id);
return {
monitor,
status,
check,
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 },
decision: {},
};
},
} as any,
geoCheckPipeline: { run: jest.fn<any>() },
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
helper: {} as any,
queueWorkersRepository: {} as any,
queueMode: "worker",
queuePrimaryProcesses: true,
workerId: "worker-b",
});
});
afterEach(async () => {
await bufferA.shutdown();
await bufferB.shutdown();
});
it("evaluates every persisted check even when a later worker's batch flushes first", async () => {
// t=0: worker A holds the check job, probes, and buffers the result. Its flush timer has not fired.
const c1 = producedAt(checkService.toCheck(makeStatusResponse())!, T0);
bufferA.addToBuffer(c1);
// t=30s: worker B claims the next run of the same job, probes, and buffers the result.
const c2 = producedAt(checkService.toCheck(makeStatusResponse())!, T0 + INTERVAL_MS);
bufferB.addToBuffer(c2);
// t=45s: worker B's flush timer fires first. c2 is inserted and an evaluate job is queued.
await bufferB.flushBuffer();
expect(await runDueEvaluateJobs()).toBe(1);
expect(evaluatedCheckIds).toEqual([c2.id]);
// t=60s: worker A's flush timer fires. c1 is inserted and the evaluate job is pulled due again.
await bufferA.flushBuffer();
expect(await runDueEvaluateJobs()).toBe(1);
// Both checks are durably stored and visible in history.
const stored = await CheckModel.find({ "metadata.monitorId": new mongoose.Types.ObjectId(MONITOR_ID) }).lean();
expect(stored.map((doc) => String(doc._id)).sort()).toEqual([c1.id, c2.id].sort());
// Every stored check must reach the evaluator exactly once.
expect(evaluatedCheckIds).toHaveLength(2);
expect(evaluatedCheckIds).toContain(c1.id);
});
});
+14 -227
View File
@@ -3,7 +3,7 @@ import mongoose from "mongoose";
import { MongoMemoryServer } from "mongodb-memory-server";
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
import { BACKOFF_MS, LOCK_MS, PARKED, type Job, type JobType, type PendingCheck } from "../../src/domain/jobs/job.type.ts";
import { BACKOFF_MS, LOCK_MS, type Job, type JobType } from "../../src/domain/jobs/job.type.ts";
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
@@ -58,9 +58,6 @@ const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string
const WORKER_ID = "worker-under-test";
// Pending entries as ingest pushes them: one per inserted check, stamped with the check's createdAt.
const pending = (...checkIds: string[]): PendingCheck[] => checkIds.map((checkId, i) => ({ checkId, createdAt: NOW + i }));
describe("MongoJobsRepository", () => {
let repo: MongoJobsRepository;
@@ -183,7 +180,7 @@ describe("MongoJobsRepository", () => {
it("does not release a lock owned by another worker (fenced no-op)", async () => {
// A worker whose lease expired and was reclaimed must not clobber the new owner's lock.
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7, failCount: 2, lastFailReason: "timeout" });
await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7 });
const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000);
@@ -192,72 +189,6 @@ describe("MongoJobsRepository", () => {
expect(row?.lockedBy).toBe("other-worker"); // lock untouched
expect(row?.lockedUntil).toBe(NOW + LOCK_MS);
expect(row?.runCount).toBe(7); // not double-counted
expect(row?.failCount).toBe(2); // failure state untouched
expect(row?.lastFailReason).toBe("timeout");
});
it("clears the failure state so failCount counts consecutive failures", async () => {
await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, failCount: 3, lastFailReason: "connection refused" });
await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000);
const row = await readRow("check:mon-1");
expect(row?.failCount).toBe(0);
expect(row?.lastFailReason).toBeNull();
});
});
// ── recordOneShot (park after a single run) ─────────────────────────────────
describe("recordOneShot", () => {
it("parks the job, releases the lease, counts the run and clears the failure state", async () => {
await seedJob({
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
runCount: 1,
failCount: 2,
lastFailReason: "connection refused",
});
const ok = await repo.recordOneShot("check:mon-1", NOW + 1_000);
expect(ok).toBe(true);
const row = await readRow("check:mon-1");
expect(row?.nextScheduledAt).toBe(PARKED);
expect(row?.lockedBy).toBeNull();
expect(row?.lockedUntil).toBeNull();
expect(row?.lastFinishedAt).toBe(NOW + 1_000);
expect(row?.runCount).toBe(2);
expect(row?.failCount).toBe(0);
expect(row?.lastFailReason).toBeNull();
});
it("does not touch a lock owned by another worker (fenced no-op)", async () => {
await seedJob({ intervalMs: null, lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, failCount: 2, lastFailReason: "timeout" });
expect(await repo.recordOneShot("check:mon-1", NOW + 1_000)).toBe(false);
const row = await readRow("check:mon-1");
expect(row?.nextScheduledAt).toBe(NOW); // not parked
expect(row?.lockedBy).toBe("other-worker");
expect(row?.failCount).toBe(2);
expect(row?.lastFailReason).toBe("timeout");
});
it("re-arms instead of parking when checks were pushed while the run held the lease", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c-late"), // arrived mid-run, not yet pulled
});
expect(await repo.recordOneShot("evaluate:mon-1", NOW + 1_000)).toBe(true);
const row = await readRow("evaluate:mon-1");
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // due now, not PARKED
expect(row?.lockedBy).toBeNull();
expect(row?.runCount).toBe(1);
});
});
@@ -345,10 +276,10 @@ describe("MongoJobsRepository", () => {
});
});
// ── upsertEvaluate (handoff: $push ids + $min due time) ──────────────────────
// ── upsertEvaluate (handoff + $min coalescing) ───────────────────────────────
describe("upsertEvaluate", () => {
it("creates a single evaluate row keyed by monitor carrying the pushed ids", async () => {
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
it("creates a single evaluate row keyed by monitor", async () => {
await repo.upsertEvaluate("mon-1", NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.type).toBe("evaluate");
@@ -356,12 +287,11 @@ describe("MongoJobsRepository", () => {
expect(row?.nextScheduledAt).toBe(NOW);
expect(row?.intervalMs).toBeNull();
expect(row?.isActive).toBe(true);
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
});
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
await repo.upsertEvaluate("mon-1", pending("c2"), NOW + 9_000);
await repo.upsertEvaluate("mon-1", NOW);
await repo.upsertEvaluate("mon-1", NOW + 9_000);
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
expect(rows).toHaveLength(1);
@@ -369,117 +299,19 @@ describe("MongoJobsRepository", () => {
});
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 9_000);
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
await repo.upsertEvaluate("mon-1", NOW + 9_000);
await repo.upsertEvaluate("mon-1", NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.nextScheduledAt).toBe(NOW);
});
it("never duplicates an entry when the same batch is armed twice (retry after a failed arm)", async () => {
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW + 1_000);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
});
it("accumulates ids across flushes so a late batch is never lost", async () => {
await repo.upsertEvaluate("mon-1", pending("c2"), NOW); // later check, flushed first
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 15_000); // earlier check, flushed later
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2", "c1"]);
});
it("keeps pushing onto a row that is currently leased by a worker", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: "other-worker",
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1"),
});
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
const row = await readRow("evaluate:mon-1");
expect(row?.lockedBy).toBe("other-worker"); // lease untouched
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1", "c2"]);
});
it("concurrent upserts for one monitor produce exactly one row holding every id", async () => {
it("concurrent upserts for one monitor produce exactly one row", async () => {
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
await Promise.all(workers.map((w, i) => w.upsertEvaluate("mon-1", pending(`c${i}`), NOW)));
await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW)));
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
expect(rows).toHaveLength(1);
expect(rows[0].pendingChecks).toHaveLength(20);
});
});
// ── pullEvaluated (evaluator removes applied ids) ───────────────────────────
describe("pullEvaluated", () => {
it("removes only the listed ids and leaves the rest", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1", "c2", "c3"),
});
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1", "c3"])).toBe(true);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2"]);
});
it("is fenced on the lease: another worker's row is untouched", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: "other-worker",
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1"),
});
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(false);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1"]);
});
it("reports the lease as held even when the ids were already pulled", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: [],
});
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(true);
});
it("is a no-op for an empty id list", async () => {
await seedJob({
_id: "evaluate:mon-1",
type: "evaluate",
intervalMs: null,
lockedBy: ownedBy(repo),
lockedUntil: NOW + LOCK_MS,
pendingChecks: pending("c1"),
});
expect(await repo.pullEvaluated("evaluate:mon-1", [])).toBe(true);
const row = await readRow("evaluate:mon-1");
expect(row?.pendingChecks).toHaveLength(1);
const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" });
expect(count).toBe(1);
});
});
@@ -495,7 +327,6 @@ describe("MongoJobsRepository", () => {
intervalMs: 60_000,
lockedBy: null,
lockedUntil: null,
pendingChecks: [],
runCount: 0,
failCount: 0,
lastFinishedAt: null,
@@ -527,7 +358,6 @@ describe("MongoJobsRepository", () => {
intervalMs: 86_400_000,
lockedBy: null,
lockedUntil: null,
pendingChecks: [],
runCount: 0,
failCount: 0,
lastFinishedAt: null,
@@ -608,7 +438,7 @@ describe("MongoJobsRepository", () => {
it("deleteById drops every row for the monitor", async () => {
await seedJob({ _id: "check:mon-1", type: "check" });
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
await repo.upsertEvaluate("mon-1", NOW);
await repo.deleteById("mon-1");
@@ -703,47 +533,4 @@ describe("MongoJobsRepository", () => {
expect(await repo.countDueBacklog(NOW)).toBe(0);
});
});
// ── global rows: egress recovery job ──────────────────────────────────────
describe("global rows", () => {
const repo = new MongoJobsRepository(WORKER_ID);
const egressRow = (nextScheduledAt: number) =>
seedJob({ _id: "egress", type: "egress" as JobType, refId: null, nextScheduledAt, intervalMs: 30_000 });
it("deleteGlobalJob removes the row by its canonical _id and leaves other global rows alone", async () => {
await egressRow(NOW);
await seedJob({ _id: "cleanup-orphaned", type: "cleanup-orphaned" as JobType, refId: null });
expect(await repo.deleteGlobalJob("egress")).toBe(true);
expect(await readRow("egress")).toBeNull();
expect(await readRow("cleanup-orphaned")).not.toBeNull();
expect(await repo.deleteGlobalJob("egress")).toBe(false);
});
it("deleteGlobalJobIfUnchanged removes the row only while nextScheduledAt matches the claimed value", async () => {
await egressRow(NOW);
// Another worker started a new episode and rescheduled the row after this run claimed it
await JobModel.updateOne({ _id: "egress" }, { $set: { nextScheduledAt: NOW + 30_000 } });
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW)).toBe(false);
expect(await readRow("egress")).not.toBeNull();
// Nobody touched it: the row goes
expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW + 30_000)).toBe(true);
expect(await readRow("egress")).toBeNull();
});
it("recordSuccess after a refused conditional delete is a no-op once the lock has moved on", async () => {
await egressRow(NOW);
const [claimed] = await repo.claimDueBatch("egress", 1, NOW);
expect(claimed?.id).toBe("egress");
// Row deleted by the handler (recovered), then recordSuccess runs from the queue: nothing to update
expect(await repo.deleteGlobalJobIfUnchanged("egress", claimed!.nextScheduledAt)).toBe(true);
expect(await repo.recordSuccess("egress", claimed!.nextScheduledAt, 30_000, NOW + 1_000)).toBe(false);
expect(await readRow("egress")).toBeNull();
});
});
});
@@ -33,7 +33,7 @@ const createMockHttpProvider = () =>
const createMockSettingsService = (apiKey?: string) =>
({
getCachedDBSettings: jest.fn().mockResolvedValue({ pagespeedApiKey: apiKey }),
getDBSettings: jest.fn().mockResolvedValue({ pagespeedApiKey: apiKey }),
}) as unknown as jest.Mocked<ISettingsService>;
const createProvider = (opts?: { apiKey?: string }) => {
@@ -154,7 +154,7 @@ describe("PageSpeedProvider", () => {
const logger = createMockLogger();
const httpProvider = createMockHttpProvider();
const settingsService = {
getCachedDBSettings: jest.fn().mockResolvedValue(null),
getDBSettings: jest.fn().mockResolvedValue(null),
} as unknown as jest.Mocked<ISettingsService>;
const provider = new PageSpeedProvider(httpProvider, settingsService, logger as any);
@@ -2,7 +2,6 @@ import { describe, expect, it, jest } from "@jest/globals";
import { PingProvider } from "../../../../src/service/network/PingProvider.ts";
import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts";
import type { Monitor } from "../../../../src/domain/monitors/monitor.type.ts";
import * as net from "net";
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -28,7 +27,7 @@ const createMockPing = (result?: Partial<{ alive: boolean; time: number | string
// ── Contract ─────────────────────────────────────────────────────────────────
testStatusProviderContract("PingProvider", {
create: () => new PingProvider(createMockPing() as any, net),
create: () => new PingProvider(createMockPing() as any),
supportedType: "ping",
unsupportedType: "http",
makeMonitor: () => makeMonitor(),
@@ -38,7 +37,7 @@ testStatusProviderContract("PingProvider", {
describe("PingProvider", () => {
it("returns success when host is alive", async () => {
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any, net);
const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any);
const result = await provider.handle(makeMonitor());
@@ -56,7 +55,7 @@ describe("PingProvider", () => {
});
it("returns failure when host is not alive", async () => {
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any, net);
const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any);
const result = await provider.handle(makeMonitor());
@@ -70,7 +69,7 @@ describe("PingProvider", () => {
});
it("parses string time values", async () => {
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any, net);
const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any);
const result = await provider.handle(makeMonitor());
@@ -78,7 +77,7 @@ describe("PingProvider", () => {
});
it("defaults responseTime to 0 for non-numeric time", async () => {
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any, net);
const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any);
const result = await provider.handle(makeMonitor());
@@ -86,7 +85,7 @@ describe("PingProvider", () => {
});
it("defaults status to false when alive is undefined", async () => {
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any, net);
const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any);
const result = await provider.handle(makeMonitor());
@@ -95,7 +94,7 @@ describe("PingProvider", () => {
it("sanitizes URL by stripping protocol and path", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any, net);
const provider = new PingProvider(mockPing as any);
await provider.handle(makeMonitor({ url: "https://example.com/path?query=1" }));
@@ -104,54 +103,36 @@ describe("PingProvider", () => {
it("sanitizes URL by stripping port", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any, net);
const provider = new PingProvider(mockPing as any);
await provider.handle(makeMonitor({ url: "example.com:8080" }));
expect(mockPing.promise.probe).toHaveBeenCalledWith("example.com");
});
it("keeps a bare IPv6 address intact", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any, net);
await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" }));
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
});
it("unwraps a bracketed IPv6 address and drops its port", async () => {
const mockPing = createMockPing();
const provider = new PingProvider(mockPing as any, net);
await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" }));
expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111");
});
it("throws AppError when url is missing", async () => {
const provider = new PingProvider(createMockPing() as any, net);
const provider = new PingProvider(createMockPing() as any);
await expect(provider.handle(makeMonitor({ url: "" }))).rejects.toThrow("URL is required for ping monitor");
});
it("throws AppError when ping.probe rejects", async () => {
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(new Error("Host unreachable")) } };
const provider = new PingProvider(mockPing as any, net);
const provider = new PingProvider(mockPing as any);
await expect(provider.handle(makeMonitor())).rejects.toThrow("Host unreachable");
});
it("throws AppError when probe returns no response", async () => {
const mockPing = { promise: { probe: jest.fn().mockResolvedValue(null) } };
const provider = new PingProvider(mockPing as any, net);
const provider = new PingProvider(mockPing as any);
await expect(provider.handle(makeMonitor())).rejects.toThrow("No response from ping");
});
it("throws AppError with stringified message for non-Error throws", async () => {
const mockPing = { promise: { probe: jest.fn().mockRejectedValue(42) } };
const provider = new PingProvider(mockPing as any, net);
const provider = new PingProvider(mockPing as any);
await expect(provider.handle(makeMonitor())).rejects.toThrow("42");
});
@@ -45,8 +45,6 @@ const makeCheck = (overrides?: Partial<Check>): Check =>
statusCode: 200,
responseTime: 100,
message: "OK",
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
...overrides,
}) as Check;
@@ -394,44 +392,9 @@ describe("BufferService", () => {
await service.flushBuffer();
const createdAt = Date.parse("2026-01-01T00:00:00.000Z");
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith(
"mon-1",
[
{ checkId: "c1", createdAt },
{ checkId: "c2", createdAt },
],
expect.any(Number)
);
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
});
it("retries a failed arm on the next flush without re-storing the checks", async () => {
const { service, checkService, jobsRepository, logger } = createService();
(jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed"));
service.addToBuffer(makeCheck({ id: "c1" }));
await service.flushBuffer();
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
await service.flushBuffer(); // buffer is empty, only the retry runs
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number));
await service.flushBuffer(); // nothing left to retry
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
});
it("ingestChecks with an empty batch neither writes nor arms", async () => {
const { service, checkService, jobsRepository } = createService();
await service.ingestChecks([]);
expect(checkService.createChecks).not.toHaveBeenCalled();
expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-1", expect.any(Number));
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", expect.any(Number));
});
it("does not arm the evaluate stage when the check write fails", async () => {
@@ -9,7 +9,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts";
// ── Notes ──────────────────────────────────────────────────────────────────────
//
// DBQueueWorker is the orchestrator: it drives the polling loops, claims batches
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and
// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and
// renews their locks, and registers itself in the worker heartbeat. The repository
// claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we
// test only the orchestration around it, with the repository fully mocked.
@@ -56,7 +56,6 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
intervalMs: 60000,
lockedBy: null,
lockedUntil: null,
pendingChecks: [],
runCount: 0,
failCount: 0,
lastFinishedAt: null,
@@ -64,22 +63,12 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
...overrides,
});
// An evaluate row as ingest leaves it: one pending entry per stored check
const makeEvaluateJob = (...checkIds: string[]): Job =>
makeJob({
id: "evaluate:m1",
type: "evaluate",
intervalMs: null,
pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })),
});
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
const jobsRepository = {
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
upsertJob: jest.fn<any>().mockResolvedValue(true),
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
pullEvaluated: jest.fn<any>().mockResolvedValue(true),
recordSuccess: jest.fn<any>().mockResolvedValue(true),
recordFailure: jest.fn<any>().mockResolvedValue(true),
recordOneShot: jest.fn<any>().mockResolvedValue(true),
@@ -103,6 +92,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
};
const checkService = {
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
toLastEvaluatedAt: jest.fn<any>().mockReturnValue(12345),
};
const bufferService = {
addToBuffer: jest.fn<any>(),
@@ -123,7 +113,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
};
const queueWorkersRepository = {
upsert: jest.fn<any>().mockResolvedValue(undefined),
@@ -237,7 +226,7 @@ describe("DBQueueWorker", () => {
const { mocks } = await start({ queueMode: "worker" });
const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]);
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"]));
expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"]));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number));
expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number));
@@ -379,78 +368,27 @@ describe("DBQueueWorker", () => {
});
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
const job = makeEvaluateJob("c1");
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
});
it("evaluate job loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
const job = makeEvaluateJob("c1");
it("evaluate job dispatches each unevaluated check and advances lastEvaluatedAt", async () => {
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
});
expect(checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks);
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
// The monitor cursor is gone: nothing writes lastEvaluatedAt
expect(mocks.monitorsRepository.updateById).not.toHaveBeenCalled();
});
it("evaluate job with no pending checks does nothing", async () => {
const job = makeEvaluateJob();
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
expect(mocks.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
expect(mocks.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled();
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
});
it("evaluate job for a deleted monitor drops every pending id", async () => {
const job = makeEvaluateJob("c1", "c2");
const monitorsRepository = { findByIdLean: jest.fn<any>().mockResolvedValue(null), updateById: jest.fn<any>() };
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, monitorsRepository },
});
expect(mocks.checkEvaluator.evaluate).not.toHaveBeenCalled();
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
});
it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
const job = makeEvaluateJob("c1", "c2", "c3");
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) };
const jobsRepository = {
...createWorker().mocks.jobsRepository,
claimDueBatch: claimOnce(job),
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
};
const { mocks } = await start({ mocks: { jobsRepository, checksRepository } });
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2);
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" }));
});
it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
const job = makeEvaluateJob("c1", "c-gone");
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up
const { mocks } = await start({
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
});
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(1);
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]);
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
});
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
const job = makeEvaluateJob("c1", "c2");
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
const initialMonitor = makeMonitor();
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
const monitorsRepository = {
@@ -502,20 +440,6 @@ describe("DBQueueWorker", () => {
expect(retentionFn).toHaveBeenCalled();
});
it("egress job runs the helper's egress recovery function with the claimed job", async () => {
const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 });
const egressFn = jest.fn<any>().mockResolvedValue(undefined);
const helper = {
getCleanupOrphanedJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getCleanupRetentionJob: jest.fn<any>().mockReturnValue(jest.fn<any>().mockResolvedValue(undefined)),
getEgressRecoveryJob: jest.fn<any>().mockReturnValue(egressFn),
};
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } });
expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" }));
expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number));
});
it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => {
const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 });
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
@@ -1,472 +0,0 @@
import { afterEach, describe, expect, it, jest } from "@jest/globals";
import { EgressService } from "../../../src/domain/egress/egress.service.ts";
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
import type { Job } from "../../../src/domain/jobs/job.type.ts";
import { createMockLogger } from "../../helpers/createMockLogger.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
id: "egress-1",
status: "ok",
degradedSince: null,
lastRecoveredAt: null,
lastProbeAt: null,
lastProbeResults: [],
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
...overrides,
});
const makeSettings = (overrides?: Record<string, unknown>) => ({
egressCheckEnabled: true,
egressCheckTargets: ["1.1.1.1", "8.8.8.8"],
egressPollIntervalSeconds: 30,
egressNotifications: ["notif-1"],
...overrides,
});
// The claimed "egress" job row as the queue hands it to the handler
const makeJob = (overrides?: Partial<Job>): Job => ({
id: "egress",
type: "egress",
refId: null,
isActive: true,
nextScheduledAt: 1_000,
intervalMs: 30_000,
lockedBy: "worker-1",
lockedUntil: 61_000,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
...overrides,
});
// requestStatus mock that answers per target url
const statusFor = (reachableTargets: string[]) =>
jest.fn().mockImplementation(async (monitor: any) => ({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
status: reachableTargets.includes(monitor.url),
code: reachableTargets.includes(monitor.url) ? 200 : 5000,
message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed",
responseTime: 12,
}));
const degradedRepository = (overrides?: Record<string, any>) => ({
findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })),
markDegraded: jest.fn().mockResolvedValue(null),
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })),
reset: jest.fn(),
...overrides,
});
const createService = (overrides?: Record<string, any>) => {
const defaults = {
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings()) },
egressStateRepository: {
findSingleton: jest.fn().mockResolvedValue(makeState()),
recordProbe: jest.fn().mockResolvedValue(makeState()),
markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })),
markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })),
reset: jest.fn(),
},
jobsRepository: {
upsertJob: jest.fn().mockResolvedValue(true),
upsertCleanupJob: jest.fn().mockResolvedValue(true),
deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true),
},
networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) },
proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) },
logger: createMockLogger(),
...overrides,
};
const service = new EgressService(
defaults.settingsService as any,
defaults.egressStateRepository as any,
defaults.jobsRepository as any,
defaults.networkService as any,
defaults.proxyResolver as any,
defaults.logger as any
);
return { service, defaults };
};
const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) =>
(defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url);
// ── Tests ────────────────────────────────────────────────────────────────────
describe("EgressService", () => {
afterEach(() => {
jest.useRealTimers();
});
// ── probeTargets ──────────────────────────────────────────────────────────
describe("probeTargets", () => {
it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => {
const { service, defaults } = createService();
await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]);
const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any);
expect(monitors).toEqual([
expect.objectContaining({ type: "ping", url: "1.1.1.1" }),
expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }),
expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }),
expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }),
expect.objectContaining({ type: "ping", url: "2606:4700::1111" }),
]);
});
it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => {
const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128");
const { service, defaults } = createService({ proxyResolver: { resolve } });
await service.probeTargets(["https://example.com/health"]);
expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" }));
expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), {
proxyUrl: "http://proxy.internal:3128",
});
});
it("reports reachable only when the provider says status is true", async () => {
const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } });
const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]);
expect(results).toEqual([
expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }),
expect.objectContaining({ target: "8.8.8.8", reachable: false }),
]);
});
it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => {
const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({
monitorId: monitor.id,
teamId: monitor.teamId,
type: monitor.type,
status: false,
code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR
message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED",
responseTime: 3,
}));
const { service } = createService({ networkService: { requestStatus } });
const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]);
expect(results).toEqual([
expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }),
expect.objectContaining({ target: "https://example.com/refused", reachable: false }),
expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts
]);
});
it("counts an unparseable stored target as unreachable without probing", async () => {
const { service, defaults } = createService();
const results = await service.probeTargets(["[1.1.1.1"]);
expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]);
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
});
it("counts a thrown provider error as unreachable", async () => {
const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } });
const results = await service.probeTargets(["1.1.1.1"]);
expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]);
});
it("counts a hung provider as unreachable once the probe timeout elapses", async () => {
jest.useFakeTimers();
const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
const pending = service.probeTargets(["1.1.1.1"]);
await jest.advanceTimersByTimeAsync(5000);
const results = await pending;
expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]);
});
});
// ── assessAfterFailure ────────────────────────────────────────────────────
describe("assessAfterFailure", () => {
it("returns null and does not probe or record when the feature is disabled", async () => {
const { service, defaults } = createService({
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
});
const result = await service.assessAfterFailure();
expect(result).toBeNull();
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
});
it("returns ok and records the probe when any target is reachable", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
const result = await service.assessAfterFailure();
expect(result).toBe("ok");
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith(
[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })],
expect.any(Date)
);
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } });
const result = await service.assessAfterFailure();
expect(result).toBe("degraded");
expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith(
[expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
expect.any(Date)
);
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") }));
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({
id: "egress",
type: "egress",
refId: null,
isActive: true,
nextScheduledAt: expect.any(Number),
intervalMs: 30_000,
});
expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled();
// Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state
const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!;
const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!;
expect(scheduleOrder).toBeLessThan(transitionOrder);
});
it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => {
const { service, defaults } = createService({
networkService: { requestStatus: statusFor([]) },
egressStateRepository: degradedRepository({
findSingleton: jest.fn().mockResolvedValue(makeState()),
markDegraded: jest.fn().mockResolvedValue(null),
}),
});
const result = await service.assessAfterFailure();
expect(result).toBe("degraded");
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") }));
});
it("does not mark degraded when the recovery job cannot be scheduled", async () => {
const { service, defaults } = createService({
networkService: { requestStatus: statusFor([]) },
jobsRepository: {
upsertJob: jest.fn(),
upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")),
deleteGlobalJobIfUnchanged: jest.fn(),
},
});
expect(await service.assessAfterFailure()).toBeNull();
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
});
it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => {
const { service, defaults } = createService({ egressStateRepository: degradedRepository() });
const result = await service.assessAfterFailure();
expect(result).toBe("degraded");
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 }));
expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled();
});
it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => {
const { service, defaults } = createService({
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) },
networkService: { requestStatus: statusFor([]) },
});
await service.assessAfterFailure();
expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 }));
});
it("falls back to the default targets when the configured list is empty", async () => {
const { service, defaults } = createService({
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) },
});
await service.assessAfterFailure();
expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]);
});
it("returns null and logs when the assessment itself fails", async () => {
const { service, defaults } = createService({
settingsService: { getCachedDBSettings: jest.fn().mockRejectedValue(new Error("db down")) },
});
const result = await service.assessAfterFailure();
expect(result).toBeNull();
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" }));
});
});
// ── shared assessment ─────────────────────────────────────────────────────
describe("shared assessment", () => {
it("shares one probe between concurrent failures", async () => {
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]);
expect(results).toEqual(["ok", "ok", "ok"]);
expect(defaults.settingsService.getCachedDBSettings).toHaveBeenCalledTimes(1);
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
});
it("keeps sharing an in-flight assessment that outlives the reuse window", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } });
const first = service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call
const second = service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle
expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]);
expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds
});
it("reuses a settled assessment within the reuse window and probes again after it", async () => {
jest.useFakeTimers();
const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } });
await service.assessAfterFailure();
await jest.advanceTimersByTimeAsync(1_000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
await jest.advanceTimersByTimeAsync(5_000);
await service.assessAfterFailure();
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2);
});
it("reuses a disabled result instead of re-reading settings for every failure", async () => {
const getCachedDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false }));
const { service } = createService({ settingsService: { getCachedDBSettings } });
expect(await service.assessAfterFailure()).toBeNull();
expect(await service.assessAfterFailure()).toBeNull();
expect(getCachedDBSettings).toHaveBeenCalledTimes(1);
});
it("does not cache an internal failure", async () => {
const getCachedDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings());
const { service, defaults } = createService({ settingsService: { getCachedDBSettings } });
expect(await service.assessAfterFailure()).toBeNull();
expect(await service.assessAfterFailure()).toBe("ok");
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
});
});
// ── checkRecovery (the "egress" job) ──────────────────────────────────────
describe("checkRecovery", () => {
it("marks recovered, releases the job and logs when a target becomes reachable", async () => {
const { service, defaults } = createService({
egressStateRepository: degradedRepository(),
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
});
const job = makeJob({ nextScheduledAt: 123_456 });
await service.checkRecovery(job);
expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith(
[expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })],
expect.any(Date)
);
// Conditional on the schedule this run claimed, so a concurrent new episode keeps its job
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456);
expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }));
});
it("releases the job but does not log a recovery when another process already recorded it", async () => {
const { service, defaults } = createService({
egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }),
networkService: { requestStatus: statusFor(["1.1.1.1"]) },
});
await service.checkRecovery(makeJob());
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
expect(defaults.logger.info).not.toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })
);
});
it("records the probe and keeps the job while every target stays unreachable", async () => {
const { service, defaults } = createService({
egressStateRepository: degradedRepository(),
networkService: { requestStatus: statusFor([]) },
});
await service.checkRecovery(makeJob());
expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1);
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled();
});
it("releases the job without probing when the persisted state is no longer degraded", async () => {
const { service, defaults } = createService();
await service.checkRecovery(makeJob());
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
});
it("releases the job without probing when the feature has been disabled mid-episode", async () => {
const { service, defaults } = createService({
settingsService: { getCachedDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) },
egressStateRepository: degradedRepository(),
});
await service.checkRecovery(makeJob());
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000);
});
it("lets errors propagate so the queue records the failure and retries", async () => {
const { service } = createService({
egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }),
});
await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down");
});
});
});
@@ -1,100 +0,0 @@
import { describe, expect, it, jest } from "@jest/globals";
import { EgressStateService } from "../../../src/domain/egress/egress-state.service.ts";
import type { EgressState } from "../../../src/domain/egress/egress.type.ts";
// ── Helpers ──────────────────────────────────────────────────────────────────
const createEgressStateRepo = () => ({
findSingleton: jest.fn(),
recordProbe: jest.fn(),
markDegraded: jest.fn(),
markRecovered: jest.fn(),
reset: jest.fn(),
});
const createJobsRepo = () => ({
deleteGlobalJob: jest.fn().mockResolvedValue(true),
});
const createService = () => {
const egressStateRepository = createEgressStateRepo();
const jobsRepository = createJobsRepo();
const service = new EgressStateService(egressStateRepository as any, jobsRepository as any);
return { service, egressStateRepository, jobsRepository };
};
const makeState = (overrides?: Partial<EgressState>): EgressState => ({
id: "egress-1",
status: "ok",
degradedSince: null,
lastRecoveredAt: null,
lastProbeAt: null,
lastProbeResults: [],
createdAt: "2026-01-01T00:00:00Z",
updatedAt: "2026-01-01T00:00:00Z",
...overrides,
});
// ── Tests ────────────────────────────────────────────────────────────────────
describe("EgressStateService", () => {
describe("getState", () => {
it("returns the singleton state from the repository", async () => {
const { service, egressStateRepository } = createService();
const state = makeState({
status: "degraded",
degradedSince: "2026-01-01T00:05:00Z",
lastProbeAt: "2026-01-01T00:05:00Z",
lastProbeResults: [{ target: "1.1.1.1", reachable: false, responseTime: 0, message: "timeout" }],
});
(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(state);
const result = await service.getState();
expect(egressStateRepository.findSingleton).toHaveBeenCalledTimes(1);
expect(result).toEqual(state);
});
it("does not call any transition method", async () => {
const { service, egressStateRepository } = createService();
(egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(makeState());
await service.getState();
expect(egressStateRepository.recordProbe).not.toHaveBeenCalled();
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
});
it("propagates repository errors", async () => {
const { service, egressStateRepository } = createService();
(egressStateRepository.findSingleton as jest.Mock).mockRejectedValue(new Error("db down"));
await expect(service.getState()).rejects.toThrow("db down");
});
});
describe("reset", () => {
it("removes any pending recovery job and returns the state to ok", async () => {
const { service, egressStateRepository, jobsRepository } = createService();
const reset = makeState();
(egressStateRepository.reset as jest.Mock).mockResolvedValue(reset);
const result = await service.reset();
expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress");
expect(egressStateRepository.reset).toHaveBeenCalledTimes(1);
expect(result).toEqual(reset);
});
it("does not touch the probe-driven transitions", async () => {
const { service, egressStateRepository } = createService();
(egressStateRepository.reset as jest.Mock).mockResolvedValue(makeState());
await service.reset();
expect(egressStateRepository.markDegraded).not.toHaveBeenCalled();
expect(egressStateRepository.markRecovered).not.toHaveBeenCalled();
});
});
});
@@ -143,21 +143,6 @@ describe("JobScheduler", () => {
});
});
describe("addJob", () => {
it("preserves an arbitrary monitor interval in the scheduled job", async () => {
const { scheduler, mocks } = createScheduler();
await scheduler.addJob("m1", makeMonitor({ interval: 75000 }));
expect(mocks.jobsRepository.upsertJob).toHaveBeenCalledWith(
expect.objectContaining({
id: "check:m1",
intervalMs: 75000,
})
);
});
});
// Regression: a heartbeat upsert racing shutdown's deleteById could resurrect the
// registry row, leaving a ghost worker that inflates the alive count until TTL.
// shutdown() must await the in-flight beat and issue the delete as the LAST write.
@@ -685,22 +685,6 @@ describe("NotificationMessageBuilder", () => {
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
// A temperature exactly AT the threshold must not report a breach here, matching
// StatusService.computeHardwareStatus's own strict `>` comparison (cpu/memory/disk
// already agree with this file on `>`; temp alone used to diverge with `>=`, so an
// incident-driving check at exactly the threshold never fired but the notification
// content for a breach some OTHER metric triggered could still list temp as breaching).
it("does not report temperature breach when exactly at threshold", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 });
const response = makeStatusResponse({
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [80] } as any }),
} as any);
const breaches = builder.extractThresholdBreaches(monitor, response);
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
});
it("skips temperature check when tempAlertThreshold is undefined", () => {
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined });
const response = makeStatusResponse({
@@ -28,7 +28,6 @@ const createMonitorsRepo = () => ({
const createSettingsService = (clientHost = "https://app.example.com") => ({
getSettings: jest.fn().mockReturnValue({ clientHost }),
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
});
const createMessageBuilder = () => ({
@@ -372,26 +371,5 @@ describe("NotificationsService", () => {
expect(result).toBe(deleted);
expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1");
});
it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => {
const { service, notificationsRepository, settingsService } = createService();
(notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification());
await service.deleteById("notif-1", "team-1");
expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1");
const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!;
const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!;
expect(deleteOrder).toBeLessThan(cascadeOrder);
});
it("does not touch app settings when the delete is refused", async () => {
const { service, notificationsRepository, settingsService } = createService();
(notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found"));
await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found");
expect(settingsService.removeEgressNotification).not.toHaveBeenCalled();
});
});
});
@@ -27,7 +27,7 @@ const makeProxy = (overrides?: Partial<Proxy>): Proxy =>
const createResolver = (overrides?: Record<string, any>) => {
const defaults = {
proxiesRepository: { findByIdOrNull: jest.fn<() => Promise<Proxy | null>>().mockResolvedValue(makeProxy()) },
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: "p1" }) },
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: "p1" }) },
logger: createMockLogger(),
ttlMs: 60_000,
...overrides,
@@ -46,7 +46,7 @@ describe("ProxyResolver", () => {
expect(result).toBeUndefined();
expect(defaults.proxiesRepository.findByIdOrNull).not.toHaveBeenCalled();
expect(defaults.settingsService.getCachedDBSettings).not.toHaveBeenCalled();
expect(defaults.settingsService.getDBSettings).not.toHaveBeenCalled();
});
it("returns undefined for proxyMode 'none' without calling settings", async () => {
@@ -55,7 +55,7 @@ describe("ProxyResolver", () => {
const result = await resolver.resolve(makeMonitor({ proxyMode: "none" }));
expect(result).toBeUndefined();
expect(defaults.settingsService.getCachedDBSettings).not.toHaveBeenCalled();
expect(defaults.settingsService.getDBSettings).not.toHaveBeenCalled();
});
// ── custom mode ───────────────────────────────────────────────────────────
@@ -94,7 +94,7 @@ describe("ProxyResolver", () => {
it("returns undefined when the global proxy is disabled, without a repo call", async () => {
const { resolver, defaults } = createResolver({
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: false, globalProxyId: "p1" }) },
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: false, globalProxyId: "p1" }) },
});
const result = await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
@@ -114,7 +114,7 @@ describe("ProxyResolver", () => {
it("returns undefined when the global proxy is enabled but globalProxyId is null, without a repo call", async () => {
const { resolver, defaults } = createResolver({
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: null }) },
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockResolvedValue({ globalProxyEnabled: true, globalProxyId: null }) },
});
const result = await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
@@ -140,7 +140,7 @@ describe("ProxyResolver", () => {
const result = await resolver.resolve(makeMonitor({ proxyMode: "garbage" as Monitor["proxyMode"] }));
expect(result).toBe("http://proxy.example.com:8080");
expect(defaults.settingsService.getCachedDBSettings).toHaveBeenCalled();
expect(defaults.settingsService.getDBSettings).toHaveBeenCalled();
});
// ── never throws ──────────────────────────────────────────────────────────
@@ -158,7 +158,7 @@ describe("ProxyResolver", () => {
it("returns undefined and warns when the settings service throws", async () => {
const { resolver, defaults } = createResolver({
settingsService: { getCachedDBSettings: jest.fn<() => Promise<any>>().mockRejectedValue(new Error("db down")) },
settingsService: { getDBSettings: jest.fn<() => Promise<any>>().mockRejectedValue(new Error("db down")) },
});
const result = await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
@@ -175,6 +175,7 @@ describe("ProxyResolver", () => {
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
expect(defaults.proxiesRepository.findByIdOrNull).toHaveBeenCalledTimes(1);
});
@@ -184,6 +185,7 @@ describe("ProxyResolver", () => {
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
await resolver.resolve(makeMonitor({ proxyMode: "inherit" }));
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(2);
expect(defaults.proxiesRepository.findByIdOrNull).toHaveBeenCalledTimes(2);
});
@@ -207,6 +209,7 @@ describe("ProxyResolver", () => {
resolver.resolve(makeMonitor({ proxyMode: "inherit" })),
]);
expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1);
expect(defaults.proxiesRepository.findByIdOrNull).toHaveBeenCalledTimes(1);
});
@@ -43,13 +43,12 @@ const createSettingsRepo = () =>
create: jest.fn().mockResolvedValue(makeSettings()),
findSingleton: jest.fn().mockResolvedValue(makeSettings()),
update: jest.fn().mockResolvedValue(makeSettings()),
removeEgressNotification: jest.fn().mockResolvedValue(undefined),
deleteLegacy: jest.fn().mockResolvedValue(true),
}) as unknown as jest.Mocked<ISettingsRepository>;
const createService = (envOverrides?: Partial<ValidatedEnv>, dbSettingsTTLMs?: number) => {
const createService = (envOverrides?: Partial<ValidatedEnv>) => {
const env = makeEnv(envOverrides);
const service = new SettingsService(env, dbSettingsTTLMs);
const service = new SettingsService(env);
const settingsRepository = createSettingsRepo();
service.setRepository(settingsRepository);
return { service, settingsRepository, env };
@@ -172,66 +171,6 @@ describe("SettingsService", () => {
});
});
// ── getCachedDBSettings ─────────────────────────────────────────────────
describe("getCachedDBSettings", () => {
it("serves a second call within the TTL from cache", async () => {
const { service, settingsRepository } = createService();
const first = await service.getCachedDBSettings();
const second = await service.getCachedDBSettings();
expect(second).toBe(first);
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(1);
});
it("refetches after the TTL expires", async () => {
const { service, settingsRepository } = createService(undefined, 0);
await service.getCachedDBSettings();
await service.getCachedDBSettings();
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
});
it("shares one in-flight fetch between concurrent calls", async () => {
const { service, settingsRepository } = createService();
await Promise.all([service.getCachedDBSettings(), service.getCachedDBSettings(), service.getCachedDBSettings()]);
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(1);
});
it("does not cache a rejected fetch, the next call retries", async () => {
const { service, settingsRepository } = createService();
(settingsRepository.findSingleton as jest.Mock).mockRejectedValueOnce(new Error("db down")).mockResolvedValueOnce(makeSettings());
await expect(service.getCachedDBSettings()).rejects.toThrow("db down");
await expect(service.getCachedDBSettings()).resolves.toEqual(makeSettings());
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
});
it("refetches after updateDbSettings so the write is visible immediately", async () => {
const { service, settingsRepository } = createService();
await service.getCachedDBSettings();
await service.updateDbSettings({ checkTTL: 60 });
await service.getCachedDBSettings();
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
});
it("refetches after removeEgressNotification", async () => {
const { service, settingsRepository } = createService();
await service.getCachedDBSettings();
await service.removeEgressNotification("notif-1");
await service.getCachedDBSettings();
expect(settingsRepository.findSingleton).toHaveBeenCalledTimes(2);
});
});
// ── updateDbSettings ────────────────────────────────────────────────────
describe("updateDbSettings", () => {
@@ -253,20 +192,4 @@ describe("SettingsService", () => {
await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized");
});
});
describe("removeEgressNotification", () => {
it("delegates the pull to the repository", async () => {
const { service, settingsRepository } = createService();
await service.removeEgressNotification("notif-1");
expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1");
});
it("throws when repository is not set", async () => {
const service = new SettingsService(makeEnv());
await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized");
});
});
});
+1 -24
View File
@@ -41,9 +41,6 @@ const createHelper = (overrides?: Record<string, unknown>) => {
const checkService = {
deleteOlderThan: jest.fn().mockResolvedValue(0),
};
const egressService = {
checkRecovery: jest.fn().mockResolvedValue(undefined),
};
const defaults = {
logger: createMockLogger(),
@@ -57,7 +54,6 @@ const createHelper = (overrides?: Record<string, unknown>) => {
incidentsRepository,
geoChecksRepository,
dockerLogsRepository,
egressService,
...overrides,
};
@@ -72,8 +68,7 @@ const createHelper = (overrides?: Record<string, unknown>) => {
defaults.checksRepository as any,
defaults.incidentsRepository as any,
defaults.geoChecksRepository as any,
defaults.dockerLogsRepository as any,
defaults.egressService as any
defaults.dockerLogsRepository as any
);
return { helper, defaults };
};
@@ -189,22 +184,4 @@ describe("WorkerHelper", () => {
expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined }));
});
});
describe("getEgressRecoveryJob", () => {
it("returns a function that hands the claimed job to the egress service", async () => {
const { helper, defaults } = createHelper();
const job = { id: "egress", type: "egress", refId: null, nextScheduledAt: 1000 } as any;
await helper.getEgressRecoveryJob()(job);
expect(defaults.egressService.checkRecovery).toHaveBeenCalledWith(job);
});
it("lets the egress service's error propagate so the queue records the failure", async () => {
const { helper, defaults } = createHelper();
(defaults.egressService.checkRecovery as jest.Mock).mockRejectedValue(new Error("db down"));
await expect(helper.getEgressRecoveryJob()({ id: "egress" } as any)).rejects.toThrow("db down");
});
});
});
@@ -1,49 +0,0 @@
import { describe, expect, it } from "@jest/globals";
import { parseEgressTarget } from "../../../src/utils/egressTarget.ts";
describe("parseEgressTarget", () => {
it("parses http(s) URLs", () => {
expect(parseEgressTarget("https://example.com/health")).toEqual({ kind: "http", url: "https://example.com/health" });
expect(parseEgressTarget("HTTP://example.com")).toEqual({ kind: "http", url: "HTTP://example.com" });
expect(parseEgressTarget("http://[2606:4700::1111]:8080/")).toEqual({ kind: "http", url: "http://[2606:4700::1111]:8080/" });
});
it("parses host:port with a hostname, IPv4 address, localhost or bracketed IPv6 address", () => {
expect(parseEgressTarget("dns.google:53")).toEqual({ kind: "port", host: "dns.google", port: 53 });
expect(parseEgressTarget("8.8.8.8:53")).toEqual({ kind: "port", host: "8.8.8.8", port: 53 });
expect(parseEgressTarget("localhost:8080")).toEqual({ kind: "port", host: "localhost", port: 8080 });
expect(parseEgressTarget("[2606:4700::1111]:443")).toEqual({ kind: "port", host: "2606:4700::1111", port: 443 });
});
it("parses bare hosts and addresses as ping targets", () => {
expect(parseEgressTarget("1.1.1.1")).toEqual({ kind: "ping", host: "1.1.1.1" });
expect(parseEgressTarget("example.com")).toEqual({ kind: "ping", host: "example.com" });
expect(parseEgressTarget("2606:4700:4700::1111")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
expect(parseEgressTarget("[2606:4700:4700::1111]")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" });
expect(parseEgressTarget(" 1.1.1.1 ")).toEqual({ kind: "ping", host: "1.1.1.1" });
});
it("rejects anything the probe could not reach", () => {
for (const target of [
"",
"ftp://example.com",
"http://",
"bad host",
"1.1.1.1;rm",
"[1.1.1.1",
"1.1.1.1]",
"[::1",
"::1]",
"[[::1]]",
"[not-an-address]:53",
"[1.1.1.1]:53",
"999.1.1.1",
"1.1.1.1:0",
"1.1.1.1:70000",
"1.1.1.1:",
"host-without-tld",
]) {
expect(parseEgressTarget(target)).toBeNull();
}
});
});
@@ -1,99 +0,0 @@
import { describe, expect, it } from "@jest/globals";
import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts";
import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts";
const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8";
describe("settingsValidation", () => {
describe("updateAppSettingsBodyValidation egress fields", () => {
it("accepts a full set of valid egress settings", () => {
const parsed = updateAppSettingsBodyValidation.parse({
egressCheckEnabled: true,
egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"],
egressPollIntervalSeconds: 30,
egressNotifications: [validNotificationId],
});
expect(parsed.egressCheckEnabled).toBe(true);
expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]);
expect(parsed.egressPollIntervalSeconds).toBe(30);
expect(parsed.egressNotifications).toEqual([validNotificationId]);
});
it("accepts a body with no egress fields", () => {
expect(() => updateAppSettingsBodyValidation.parse({ checkTTL: 30 })).not.toThrow();
});
it("accepts an empty target list (the worker falls back to the default targets)", () => {
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: [] }).egressCheckTargets).toEqual([]);
});
it("rejects more targets than the maximum", () => {
const targets = Array.from({ length: MAX_EGRESS_TARGETS + 1 }, (_, i) => `10.0.0.${i}`);
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow();
});
it("accepts IPv6 targets, bare or bracketed with a port", () => {
const targets = ["2606:4700:4700::1111", "[2606:4700:4700::1111]", "[2606:4700:4700::1111]:53", "localhost", "localhost:8080"];
expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets }).egressCheckTargets).toEqual(targets);
});
it("rejects targets that are not a host, host:port or http(s) URL", () => {
for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm", "http://"]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
}
});
it("rejects malformed addresses that the probe could never reach", () => {
for (const target of [
"[1.1.1.1",
"1.1.1.1]",
"[::1",
"::1]",
"[[::1]]",
"[not-an-address]:53",
"999.1.1.1",
"1.1.1.1:0",
"1.1.1.1:70000",
"[::1]:",
]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow();
}
});
it("accepts the poll interval boundaries", () => {
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
MIN_EGRESS_POLL_INTERVAL_SECONDS
);
expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe(
MAX_EGRESS_POLL_INTERVAL_SECONDS
);
});
it("rejects out-of-range and non-integer poll intervals", () => {
for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow();
}
});
it("accepts an empty notification list", () => {
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]);
});
it("accepts upper-case hex notification ids", () => {
const id = "64B7F0C2A1D2E3F4A5B6C7D8";
expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [id] }).egressNotifications).toEqual([id]);
});
it("rejects notification ids that are not 24-character hex", () => {
for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) {
expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow();
}
});
it("rejects a non-boolean enabled flag", () => {
expect(() => updateAppSettingsBodyValidation.parse({ egressCheckEnabled: "yes" })).toThrow();
});
});
});