Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a277a89bef | ||
|
|
9d0dc3f330 | ||
|
|
da8b230cec | ||
|
|
f4864b5455 | ||
|
|
f299098548 | ||
|
|
87eab6f996 | ||
|
|
31400a2502 | ||
|
|
fa0c0abcac | ||
|
|
9102335d3e | ||
|
|
b2ace1bec5 |
No files matched your search
@@ -2,6 +2,10 @@ import { createSlice, type PayloadAction } from "@reduxjs/toolkit";
|
|||||||
|
|
||||||
type ThemeMode = "light" | "dark";
|
type ThemeMode = "light" | "dark";
|
||||||
type ChartType = "histogram" | "line";
|
type ChartType = "histogram" | "line";
|
||||||
|
|
||||||
|
export const dashboardCardKeys = ["monitorStatus", "monitorsByType"] as const;
|
||||||
|
export type DashboardCardKey = (typeof dashboardCardKeys)[number];
|
||||||
|
|
||||||
type TableName =
|
type TableName =
|
||||||
| "monitors"
|
| "monitors"
|
||||||
| "team"
|
| "team"
|
||||||
@@ -35,6 +39,7 @@ interface UIState {
|
|||||||
language: string;
|
language: string;
|
||||||
starPromptOpen: boolean;
|
starPromptOpen: boolean;
|
||||||
chartType: ChartType;
|
chartType: ChartType;
|
||||||
|
dashboardVisibleCards: DashboardCardKey[];
|
||||||
}
|
}
|
||||||
|
|
||||||
const initialMode: ThemeMode = window?.matchMedia?.("(prefers-color-scheme: dark)")
|
const initialMode: ThemeMode = window?.matchMedia?.("(prefers-color-scheme: dark)")
|
||||||
@@ -74,6 +79,7 @@ const initialState: UIState = {
|
|||||||
language: "en",
|
language: "en",
|
||||||
starPromptOpen: true,
|
starPromptOpen: true,
|
||||||
chartType: "histogram",
|
chartType: "histogram",
|
||||||
|
dashboardVisibleCards: [...dashboardCardKeys],
|
||||||
};
|
};
|
||||||
|
|
||||||
const uiSlice = createSlice({
|
const uiSlice = createSlice({
|
||||||
@@ -117,6 +123,9 @@ const uiSlice = createSlice({
|
|||||||
setChartType: (state, action: PayloadAction<ChartType>) => {
|
setChartType: (state, action: PayloadAction<ChartType>) => {
|
||||||
state.chartType = action.payload;
|
state.chartType = action.payload;
|
||||||
},
|
},
|
||||||
|
setDashboardVisibleCards: (state, action: PayloadAction<DashboardCardKey[]>) => {
|
||||||
|
state.dashboardVisibleCards = action.payload;
|
||||||
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -133,4 +142,5 @@ export const {
|
|||||||
setLanguage,
|
setLanguage,
|
||||||
setStarPromptOpen,
|
setStarPromptOpen,
|
||||||
setChartType,
|
setChartType,
|
||||||
|
setDashboardVisibleCards,
|
||||||
} = uiSlice.actions;
|
} = uiSlice.actions;
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
import { useState } from "react";
|
||||||
|
import Stack from "@mui/material/Stack";
|
||||||
|
import FormControlLabel from "@mui/material/FormControlLabel";
|
||||||
|
import { useTranslation } from "react-i18next";
|
||||||
|
import { Checkbox, Dialog } from "@/Components/inputs";
|
||||||
|
import { type DashboardCardKey, dashboardCardKeys } from "@/Features/UI/uiSlice";
|
||||||
|
|
||||||
|
export type { DashboardCardKey };
|
||||||
|
|
||||||
|
interface EditCardsModalProps {
|
||||||
|
open: boolean;
|
||||||
|
visibleCards: Set<DashboardCardKey>;
|
||||||
|
onClose: () => void;
|
||||||
|
onSave: (visibleCards: Set<DashboardCardKey>) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const EditCardsModal = ({
|
||||||
|
open,
|
||||||
|
visibleCards,
|
||||||
|
onClose,
|
||||||
|
onSave,
|
||||||
|
}: EditCardsModalProps) => {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
const [draft, setDraft] = useState<Set<DashboardCardKey>>(new Set(visibleCards));
|
||||||
|
|
||||||
|
const handleClose = () => {
|
||||||
|
setDraft(new Set(visibleCards));
|
||||||
|
onClose();
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleToggle = (key: DashboardCardKey) => {
|
||||||
|
setDraft((prev) => {
|
||||||
|
const next = new Set(prev);
|
||||||
|
if (next.has(key)) {
|
||||||
|
next.delete(key);
|
||||||
|
} else {
|
||||||
|
next.add(key);
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dialog
|
||||||
|
open={open}
|
||||||
|
title={t("pages.dashboard.editCardsModal.title")}
|
||||||
|
content={t("pages.dashboard.editCardsModal.description")}
|
||||||
|
onCancel={handleClose}
|
||||||
|
onConfirm={() => onSave(draft)}
|
||||||
|
confirmText={t("pages.dashboard.editCardsModal.done")}
|
||||||
|
maxWidth="xs"
|
||||||
|
fullWidth
|
||||||
|
>
|
||||||
|
<Stack>
|
||||||
|
{dashboardCardKeys.map((key) => (
|
||||||
|
<FormControlLabel
|
||||||
|
key={key}
|
||||||
|
label={t(`pages.dashboard.cards.${key}`)}
|
||||||
|
control={
|
||||||
|
<Checkbox
|
||||||
|
checked={draft.has(key)}
|
||||||
|
disabled={draft.has(key) && draft.size === 1}
|
||||||
|
onChange={() => handleToggle(key)}
|
||||||
|
/>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</Stack>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -9,41 +9,50 @@ import { useTranslation } from "react-i18next";
|
|||||||
import type { MonitorsSummary } from "@/Types/Monitor";
|
import type { MonitorsSummary } from "@/Types/Monitor";
|
||||||
|
|
||||||
interface MonitorStatusCardProps {
|
interface MonitorStatusCardProps {
|
||||||
summary: MonitorsSummary;
|
summary: MonitorsSummary | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MonitorStatusCard = ({ summary }: MonitorStatusCardProps) => {
|
export const MonitorStatusCard = ({ summary }: MonitorStatusCardProps) => {
|
||||||
const theme = useTheme();
|
const theme = useTheme();
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
|
||||||
const up = summary.upMonitors;
|
const up = summary?.upMonitors ?? 0;
|
||||||
const total = summary.totalMonitors;
|
const total = summary?.totalMonitors ?? 0;
|
||||||
const ratio = total > 0 ? up / total : 0;
|
const ratio = total > 0 ? up / total : 0;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<DashboardCard title={t("pages.dashboard.cards.monitorStatus")}>
|
<DashboardCard title={t("pages.dashboard.cards.monitorStatus")}>
|
||||||
<Stack gap={theme.spacing(SPACING.SM)}>
|
{total === 0 ? (
|
||||||
<Stack
|
<Typography
|
||||||
direction="row"
|
fontSize={typographyLevels.m}
|
||||||
alignItems="baseline"
|
color={theme.palette.text.disabled}
|
||||||
gap={theme.spacing(SPACING.XS)}
|
|
||||||
>
|
>
|
||||||
<Typography
|
{t("pages.dashboard.noData")}
|
||||||
fontSize={typographyLevels.xxl}
|
</Typography>
|
||||||
fontWeight={500}
|
) : (
|
||||||
color={theme.palette.text.primary}
|
<Stack gap={theme.spacing(SPACING.SM)}>
|
||||||
|
<Stack
|
||||||
|
direction="row"
|
||||||
|
alignItems="baseline"
|
||||||
|
gap={theme.spacing(SPACING.XS)}
|
||||||
>
|
>
|
||||||
{up}
|
<Typography
|
||||||
</Typography>
|
fontSize={typographyLevels.xxl}
|
||||||
<Typography
|
fontWeight={500}
|
||||||
fontSize={typographyLevels.m}
|
color={theme.palette.text.primary}
|
||||||
color={theme.palette.text.secondary}
|
>
|
||||||
>
|
{up}
|
||||||
{t("pages.dashboard.monitorStatus.ofUp", { total })}
|
</Typography>
|
||||||
</Typography>
|
<Typography
|
||||||
|
fontSize={typographyLevels.m}
|
||||||
|
color={theme.palette.text.secondary}
|
||||||
|
>
|
||||||
|
{t("pages.dashboard.monitorStatus.ofUp", { total })}
|
||||||
|
</Typography>
|
||||||
|
</Stack>
|
||||||
|
<ProgressBar fillRatio={ratio} />
|
||||||
</Stack>
|
</Stack>
|
||||||
<ProgressBar fillRatio={ratio} />
|
)}
|
||||||
</Stack>
|
|
||||||
</DashboardCard>
|
</DashboardCard>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
@@ -1,14 +1,16 @@
|
|||||||
import Stack from "@mui/material/Stack";
|
import Stack from "@mui/material/Stack";
|
||||||
|
import Typography from "@mui/material/Typography";
|
||||||
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
|
import { DashboardCard } from "@/Pages/Dashboard/components/DashboardCard";
|
||||||
import { BarRow } from "@/Pages/Dashboard/components/cards/BarRow";
|
import { BarRow } from "@/Pages/Dashboard/components/cards/BarRow";
|
||||||
import { useTranslation } from "react-i18next";
|
import { useTranslation } from "react-i18next";
|
||||||
import { useTheme } from "@mui/material/styles";
|
import { useTheme } from "@mui/material/styles";
|
||||||
import type { DashboardTypeCount } from "@/Pages/Dashboard/types";
|
import type { MonitorTypeCount } from "@/Types/Monitor";
|
||||||
import { getMonitorTypeLabel } from "@/Types/Monitor";
|
import { getMonitorTypeLabel } from "@/Types/Monitor";
|
||||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||||
|
import { typographyLevels } from "@/Utils/Theme/Palette";
|
||||||
|
|
||||||
interface MonitorsByTypeCardProps {
|
interface MonitorsByTypeCardProps {
|
||||||
monitorsByType: DashboardTypeCount[];
|
monitorsByType: MonitorTypeCount[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MonitorsByTypeCard = ({ monitorsByType }: MonitorsByTypeCardProps) => {
|
export const MonitorsByTypeCard = ({ monitorsByType }: MonitorsByTypeCardProps) => {
|
||||||
@@ -19,17 +21,26 @@ export const MonitorsByTypeCard = ({ monitorsByType }: MonitorsByTypeCardProps)
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<DashboardCard title={t("pages.dashboard.cards.monitorsByType")}>
|
<DashboardCard title={t("pages.dashboard.cards.monitorsByType")}>
|
||||||
<Stack gap={theme.spacing(LAYOUT.SM)}>
|
{monitorsByType.length === 0 ? (
|
||||||
{monitorsByType.map(({ type, count }) => (
|
<Typography
|
||||||
<BarRow
|
fontSize={typographyLevels.m}
|
||||||
key={type}
|
color={theme.palette.text.disabled}
|
||||||
label={getMonitorTypeLabel(type, t)}
|
>
|
||||||
value={String(count)}
|
{t("pages.dashboard.noData")}
|
||||||
fillRatio={count / maxCount}
|
</Typography>
|
||||||
valueColor={theme.palette.text.primary}
|
) : (
|
||||||
/>
|
<Stack gap={theme.spacing(LAYOUT.SM)}>
|
||||||
))}
|
{monitorsByType.map(({ type, count }) => (
|
||||||
</Stack>
|
<BarRow
|
||||||
|
key={type}
|
||||||
|
label={getMonitorTypeLabel(type, t)}
|
||||||
|
value={String(count)}
|
||||||
|
fillRatio={count / maxCount}
|
||||||
|
valueColor={theme.palette.text.primary}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</Stack>
|
||||||
|
)}
|
||||||
</DashboardCard>
|
</DashboardCard>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
@@ -1,43 +1,97 @@
|
|||||||
|
import { useMemo, useState, type ReactNode } from "react";
|
||||||
import Box from "@mui/material/Box";
|
import Box from "@mui/material/Box";
|
||||||
|
import Stack from "@mui/material/Stack";
|
||||||
import { useTheme } from "@mui/material/styles";
|
import { useTheme } from "@mui/material/styles";
|
||||||
|
import { useTranslation } from "react-i18next";
|
||||||
|
import { useSelector, useDispatch } from "react-redux";
|
||||||
|
|
||||||
import { BasePage } from "@/Components/design-elements/BasePage";
|
import { BasePage } from "@/Components/design-elements/BasePage";
|
||||||
|
import { Button } from "@/Components/inputs";
|
||||||
import { LAYOUT } from "@/Utils/Theme/constants";
|
import { LAYOUT } from "@/Utils/Theme/constants";
|
||||||
|
import { useMonitorListController } from "@/Hooks/useMonitorListController";
|
||||||
|
import { MonitorTypes, type MonitorTypeCount } from "@/Types/Monitor";
|
||||||
|
import {
|
||||||
|
setDashboardVisibleCards,
|
||||||
|
dashboardCardKeys,
|
||||||
|
type DashboardCardKey,
|
||||||
|
} from "@/Features/UI/uiSlice";
|
||||||
|
import type { RootState, AppDispatch } from "@/store";
|
||||||
|
|
||||||
import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard";
|
import { MonitorStatusCard } from "@/Pages/Dashboard/components/cards/MonitorStatusCard";
|
||||||
import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard";
|
import { MonitorsByTypeCard } from "@/Pages/Dashboard/components/cards/MonitorsByTypeCard";
|
||||||
import type { MonitorsSummary } from "@/Types/Monitor";
|
import { EditCardsModal } from "@/Pages/Dashboard/components/EditCardsModal";
|
||||||
import type { DashboardTypeCount } from "@/Pages/Dashboard/types";
|
|
||||||
|
|
||||||
const MOCK_SUMMARY: MonitorsSummary = {
|
|
||||||
totalMonitors: 12,
|
|
||||||
upMonitors: 10,
|
|
||||||
downMonitors: 1,
|
|
||||||
pausedMonitors: 1,
|
|
||||||
initializingMonitors: 0,
|
|
||||||
maintenanceMonitors: 0,
|
|
||||||
breachedMonitors: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
const MOCK_MONITORS_BY_TYPE: DashboardTypeCount[] = [
|
|
||||||
{ type: "http", count: 7 },
|
|
||||||
{ type: "ping", count: 3 },
|
|
||||||
{ type: "port", count: 2 },
|
|
||||||
];
|
|
||||||
|
|
||||||
const Dashboard = () => {
|
const Dashboard = () => {
|
||||||
const theme = useTheme();
|
const theme = useTheme();
|
||||||
|
const { t } = useTranslation();
|
||||||
|
const dispatch = useDispatch<AppDispatch>();
|
||||||
|
const [editOpen, setEditOpen] = useState(false);
|
||||||
|
const visibleCardsList = useSelector(
|
||||||
|
(state: RootState) => state.ui.dashboardVisibleCards
|
||||||
|
);
|
||||||
|
const visibleCards = useMemo(() => new Set(visibleCardsList), [visibleCardsList]);
|
||||||
|
|
||||||
|
const { monitors, summary } = useMonitorListController({
|
||||||
|
types: [...MonitorTypes],
|
||||||
|
checksLimit: 1,
|
||||||
|
refreshInterval: 30000,
|
||||||
|
rowsPerPageTable: "monitors",
|
||||||
|
rowsPerPageDefault: 100,
|
||||||
|
});
|
||||||
|
|
||||||
|
const monitorsByType = useMemo<MonitorTypeCount[]>(() => {
|
||||||
|
const countMap = new Map<MonitorTypeCount["type"], number>();
|
||||||
|
(monitors ?? []).forEach((m) =>
|
||||||
|
countMap.set(m.type, (countMap.get(m.type) ?? 0) + 1)
|
||||||
|
);
|
||||||
|
return [...countMap.entries()]
|
||||||
|
.map(([type, count]) => ({ type, count }))
|
||||||
|
.sort((a, b) => b.count - a.count);
|
||||||
|
}, [monitors]);
|
||||||
|
|
||||||
|
const cardComponents: Record<DashboardCardKey, ReactNode> = {
|
||||||
|
monitorStatus: <MonitorStatusCard summary={summary} />,
|
||||||
|
monitorsByType: <MonitorsByTypeCard monitorsByType={monitorsByType} />,
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<BasePage headerKey="dashboard">
|
<BasePage headerKey="dashboard">
|
||||||
<Box
|
<Stack gap={theme.spacing(LAYOUT.SM)}>
|
||||||
display="grid"
|
<Stack
|
||||||
gridTemplateColumns={{ xs: "1fr", md: "1fr 1fr" }}
|
direction="row"
|
||||||
gap={theme.spacing(LAYOUT.MD)}
|
justifyContent="flex-end"
|
||||||
>
|
>
|
||||||
<MonitorStatusCard summary={MOCK_SUMMARY} />
|
<Button
|
||||||
<MonitorsByTypeCard monitorsByType={MOCK_MONITORS_BY_TYPE} />
|
variant="outlined"
|
||||||
</Box>
|
size="small"
|
||||||
|
onClick={() => setEditOpen(true)}
|
||||||
|
>
|
||||||
|
{t("pages.dashboard.editCards")}
|
||||||
|
</Button>
|
||||||
|
</Stack>
|
||||||
|
|
||||||
|
<Box
|
||||||
|
display="grid"
|
||||||
|
gridTemplateColumns={{ xs: "1fr", md: "1fr 1fr" }}
|
||||||
|
gap={theme.spacing(LAYOUT.MD)}
|
||||||
|
>
|
||||||
|
{dashboardCardKeys
|
||||||
|
.filter((key) => visibleCards.has(key))
|
||||||
|
.map((key) => (
|
||||||
|
<div key={key}>{cardComponents[key]}</div>
|
||||||
|
))}
|
||||||
|
</Box>
|
||||||
|
</Stack>
|
||||||
|
|
||||||
|
<EditCardsModal
|
||||||
|
open={editOpen}
|
||||||
|
visibleCards={visibleCards}
|
||||||
|
onClose={() => setEditOpen(false)}
|
||||||
|
onSave={(next) => {
|
||||||
|
dispatch(setDashboardVisibleCards([...next]));
|
||||||
|
setEditOpen(false);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
</BasePage>
|
</BasePage>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,6 +0,0 @@
|
|||||||
import type { MonitorType } from "@/Types/Monitor";
|
|
||||||
|
|
||||||
export interface DashboardTypeCount {
|
|
||||||
type: MonitorType;
|
|
||||||
count: number;
|
|
||||||
}
|
|
||||||
@@ -192,6 +192,11 @@ export interface MonitorsSummary {
|
|||||||
breachedMonitors: number;
|
breachedMonitors: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface MonitorTypeCount {
|
||||||
|
type: MonitorType;
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface MonitorsWithChecksResponse {
|
export interface MonitorsWithChecksResponse {
|
||||||
count: number;
|
count: number;
|
||||||
monitors: MonitorWithChecks[];
|
monitors: MonitorWithChecks[];
|
||||||
|
|||||||
@@ -1911,6 +1911,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"dashboard": {
|
"dashboard": {
|
||||||
|
"noData": "No data available at this moment.",
|
||||||
"monitorStatus": {
|
"monitorStatus": {
|
||||||
"ofUp": "of {{total}} up"
|
"ofUp": "of {{total}} up"
|
||||||
},
|
},
|
||||||
@@ -1921,6 +1922,12 @@
|
|||||||
"header": {
|
"header": {
|
||||||
"title": "Dashboard",
|
"title": "Dashboard",
|
||||||
"description": "See every monitor type in one place and spot what needs attention."
|
"description": "See every monitor type in one place and spot what needs attention."
|
||||||
|
},
|
||||||
|
"editCards": "Edit cards",
|
||||||
|
"editCardsModal": {
|
||||||
|
"title": "Edit cards",
|
||||||
|
"description": "Select the cards to display on your dashboard.",
|
||||||
|
"done": "Done"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import type {
|
|||||||
PageSpeedChecksResult,
|
PageSpeedChecksResult,
|
||||||
UptimeChecksResult,
|
UptimeChecksResult,
|
||||||
} from "@/domain/checks/check.type.js";
|
} from "@/domain/checks/check.type.js";
|
||||||
|
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||||
import { CheckFilter, DateRange } from "@/types/query.js";
|
import { CheckFilter, DateRange } from "@/types/query.js";
|
||||||
|
|
||||||
@@ -43,7 +44,7 @@ export interface IChecksRepository {
|
|||||||
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult | DockerChecksResult>;
|
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult | DockerChecksResult>;
|
||||||
findDockerContainerChecks(monitorId: string, containerName: string, dateRange: DateRange): Promise<Omit<DockerContainerStats, "restartsInRange">>;
|
findDockerContainerChecks(monitorId: string, containerName: string, dateRange: DateRange): Promise<Omit<DockerContainerStats, "restartsInRange">>;
|
||||||
findSummaryByTeamId(teamId: string, dateRange: DateRange): Promise<ChecksSummary>;
|
findSummaryByTeamId(teamId: string, dateRange: DateRange): Promise<ChecksSummary>;
|
||||||
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
|
findUnevaluatedByMonitorId(monitorId: string, pending: PendingCheck[]): Promise<Check[]>;
|
||||||
getDailyStatusBuckets(monitorIds: string[], days: number, timezone: string): Promise<DailyCheckBucket[]>;
|
getDailyStatusBuckets(monitorIds: string[], days: number, timezone: string): Promise<DailyCheckBucket[]>;
|
||||||
|
|
||||||
// update
|
// update
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ import {
|
|||||||
getDockerTotalChecks,
|
getDockerTotalChecks,
|
||||||
getDockerUpChecks,
|
getDockerUpChecks,
|
||||||
} from "@/domain/checks/check.docker.aggregation.js";
|
} from "@/domain/checks/check.docker.aggregation.js";
|
||||||
|
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||||
|
|
||||||
const SERVICE_NAME = "ChecksRepository";
|
const SERVICE_NAME = "ChecksRepository";
|
||||||
|
|
||||||
@@ -360,10 +361,23 @@ class MongoChecksRepository implements IChecksRepository {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
|
findUnevaluatedByMonitorId = async (monitorId: string, pending: PendingCheck[]) => {
|
||||||
|
if (pending.length === 0) return [];
|
||||||
|
|
||||||
|
const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
|
||||||
|
|
||||||
|
// Bound the query to the pending time range so the monitorId + createdAt index is used
|
||||||
|
let from = Number.POSITIVE_INFINITY;
|
||||||
|
let to = Number.NEGATIVE_INFINITY;
|
||||||
|
for (const entry of pending) {
|
||||||
|
if (entry.createdAt < from) from = entry.createdAt;
|
||||||
|
if (entry.createdAt > to) to = entry.createdAt;
|
||||||
|
}
|
||||||
|
|
||||||
const docs = await CheckModel.find({
|
const docs = await CheckModel.find({
|
||||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||||
createdAt: { $gt: new Date(since) },
|
createdAt: { $gte: new Date(from), $lte: new Date(to) },
|
||||||
|
_id: { $in: ids },
|
||||||
})
|
})
|
||||||
.sort({ createdAt: 1 })
|
.sort({ createdAt: 1 })
|
||||||
.lean<CheckDocument[]>();
|
.lean<CheckDocument[]>();
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ const SERVICE_NAME = "checkService";
|
|||||||
|
|
||||||
export interface ICheckService {
|
export interface ICheckService {
|
||||||
createChecks(checks: Check[]): Promise<Check[]>;
|
createChecks(checks: Check[]): Promise<Check[]>;
|
||||||
toLastEvaluatedAt(check: Check): number;
|
|
||||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||||
getChecksByMonitor(params: {
|
getChecksByMonitor(params: {
|
||||||
@@ -55,10 +54,6 @@ export class CheckService implements ICheckService {
|
|||||||
return this.checksRepository.createChecks(checks);
|
return this.checksRepository.createChecks(checks);
|
||||||
};
|
};
|
||||||
|
|
||||||
toLastEvaluatedAt = (check: Check) => {
|
|
||||||
return new Date(check.createdAt).getTime();
|
|
||||||
};
|
|
||||||
|
|
||||||
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
||||||
|
|
||||||
|
|||||||
@@ -47,6 +47,18 @@ const JobSchema = new Schema<JobDocument>(
|
|||||||
default: null,
|
default: null,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// Pending checks, inserted but not yet evaluated
|
||||||
|
pendingChecks: {
|
||||||
|
type: [
|
||||||
|
{
|
||||||
|
checkId: { type: String, required: true },
|
||||||
|
createdAt: { type: Number, required: true },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
default: [],
|
||||||
|
_id: false,
|
||||||
|
},
|
||||||
|
|
||||||
// Observability
|
// Observability
|
||||||
runCount: {
|
runCount: {
|
||||||
type: Number,
|
type: Number,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
|
import { Job, JobSeed, JobType, PendingCheck } from "@/domain/jobs/job.type.js";
|
||||||
|
|
||||||
export type JobPageQuery = {
|
export type JobPageQuery = {
|
||||||
page?: number;
|
page?: number;
|
||||||
@@ -32,9 +32,12 @@ export interface IJobsRepository {
|
|||||||
recordOneShot(id: string, now: number): Promise<boolean>;
|
recordOneShot(id: string, now: number): Promise<boolean>;
|
||||||
|
|
||||||
// ********************
|
// ********************
|
||||||
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
|
// Hand off to evaluator // This is accomplished pushing newly inserted Check IDs onto the evaluation row
|
||||||
// ********************
|
// ********************
|
||||||
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
|
upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
|
||||||
|
|
||||||
|
// Removes processed checks from the evaluate row. Returns false when this worker no longer holds the lease.
|
||||||
|
pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
|
||||||
|
|
||||||
// ********************
|
// ********************
|
||||||
// Job CRUD
|
// Job CRUD
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
|
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
|
||||||
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
||||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
|
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId, PendingCheck } from "@/domain/jobs/job.type.js";
|
||||||
|
|
||||||
// Cap for rearm interval, never want to wait more than 15s.
|
// Cap for rearm interval, never want to wait more than 15s.
|
||||||
const REARM_JITTER_MAX_MS = 15_000;
|
const REARM_JITTER_MAX_MS = 15_000;
|
||||||
@@ -18,6 +18,7 @@ class MongoJobsRepository implements IJobsRepository {
|
|||||||
intervalMs: doc.intervalMs,
|
intervalMs: doc.intervalMs,
|
||||||
lockedBy: doc.lockedBy,
|
lockedBy: doc.lockedBy,
|
||||||
lockedUntil: doc.lockedUntil,
|
lockedUntil: doc.lockedUntil,
|
||||||
|
pendingChecks: doc.pendingChecks ?? [],
|
||||||
runCount: doc.runCount,
|
runCount: doc.runCount,
|
||||||
failCount: doc.failCount,
|
failCount: doc.failCount,
|
||||||
lastFinishedAt: doc.lastFinishedAt,
|
lastFinishedAt: doc.lastFinishedAt,
|
||||||
@@ -146,24 +147,27 @@ class MongoJobsRepository implements IJobsRepository {
|
|||||||
_id: id,
|
_id: id,
|
||||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||||
},
|
},
|
||||||
{
|
[
|
||||||
$set: {
|
{
|
||||||
nextScheduledAt: PARKED,
|
$set: {
|
||||||
lockedBy: null, // Remove lock
|
// Re-arm if more checks were pushed while this run held the lease; otherwise park
|
||||||
lockedUntil: null,
|
nextScheduledAt: {
|
||||||
lastFinishedAt: now,
|
$cond: [{ $gt: [{ $size: { $ifNull: ["$pendingChecks", []] } }, 0] }, now, PARKED],
|
||||||
failCount: 0,
|
},
|
||||||
lastFailReason: null,
|
lockedBy: null, // Remove lock
|
||||||
|
lockedUntil: null,
|
||||||
|
lastFinishedAt: now,
|
||||||
|
failCount: 0,
|
||||||
|
lastFailReason: null,
|
||||||
|
runCount: { $add: ["$runCount", 1] },
|
||||||
|
},
|
||||||
},
|
},
|
||||||
$inc: {
|
]
|
||||||
runCount: 1, // Increment run count
|
|
||||||
},
|
|
||||||
}
|
|
||||||
);
|
);
|
||||||
return res.modifiedCount === 1;
|
return res.modifiedCount === 1;
|
||||||
};
|
};
|
||||||
|
|
||||||
upsertEvaluate = async (monitorId: string, now: number) => {
|
upsertEvaluate = async (monitorId: string, pending: PendingCheck[], now: number) => {
|
||||||
const filter = { type: "evaluate", refId: monitorId };
|
const filter = { type: "evaluate", refId: monitorId };
|
||||||
const update = {
|
const update = {
|
||||||
$setOnInsert: {
|
$setOnInsert: {
|
||||||
@@ -181,6 +185,9 @@ class MongoJobsRepository implements IJobsRepository {
|
|||||||
$min: {
|
$min: {
|
||||||
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
||||||
},
|
},
|
||||||
|
$addToSet: {
|
||||||
|
pendingChecks: { $each: pending }, // accumulate pending checks; a retried arm never duplicates an entry
|
||||||
|
},
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||||
@@ -195,6 +202,18 @@ class MongoJobsRepository implements IJobsRepository {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Returns whether this worker still holds the lease. false means another worker has claimed the row and the caller must stop.
|
||||||
|
pullEvaluated = async (id: string, checkIds: string[]) => {
|
||||||
|
if (checkIds.length === 0) return true;
|
||||||
|
|
||||||
|
const res = await JobModel.updateOne(
|
||||||
|
{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
|
||||||
|
{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
|
||||||
|
);
|
||||||
|
return res.matchedCount === 1;
|
||||||
|
};
|
||||||
|
|
||||||
upsertJob = async (job: JobSeed) => {
|
upsertJob = async (job: JobSeed) => {
|
||||||
const res = await JobModel.updateOne(
|
const res = await JobModel.updateOne(
|
||||||
{ _id: job.id },
|
{ _id: job.id },
|
||||||
|
|||||||
@@ -4,6 +4,11 @@ export const LOCK_MS = 60_000;
|
|||||||
export const BACKOFF_MS = 5_000;
|
export const BACKOFF_MS = 5_000;
|
||||||
export const PARKED = Number.MAX_SAFE_INTEGER;
|
export const PARKED = Number.MAX_SAFE_INTEGER;
|
||||||
|
|
||||||
|
export type PendingCheck = {
|
||||||
|
checkId: string;
|
||||||
|
createdAt: number; // epoch ms, copied from Check.createdAt
|
||||||
|
};
|
||||||
|
|
||||||
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
|
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
|
||||||
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
|
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
|
||||||
|
|
||||||
@@ -21,6 +26,9 @@ export type Job = {
|
|||||||
lockedBy: string | null; // workerId holding the lease
|
lockedBy: string | null; // workerId holding the lease
|
||||||
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
|
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
|
||||||
|
|
||||||
|
// Evaluate these rows only. These are checks that are inserted but not yet evaluated
|
||||||
|
pendingChecks: PendingCheck[];
|
||||||
|
|
||||||
// observability (replaces what getMetrics/getJobs)
|
// observability (replaces what getMetrics/getJobs)
|
||||||
runCount: number;
|
runCount: number;
|
||||||
failCount: number;
|
failCount: number;
|
||||||
|
|||||||
@@ -259,7 +259,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
|
|||||||
const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature];
|
const temps = Array.isArray(hardware.cpu.temperature) ? hardware.cpu.temperature : [hardware.cpu.temperature];
|
||||||
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
|
const maxTemp = Math.max(...temps.filter((t: number) => !isNaN(t)));
|
||||||
const threshold = monitor.tempAlertThreshold;
|
const threshold = monitor.tempAlertThreshold;
|
||||||
if (maxTemp >= threshold) {
|
if (maxTemp > threshold) {
|
||||||
breaches.push({
|
breaches.push({
|
||||||
metric: "temp",
|
metric: "temp",
|
||||||
currentValue: maxTemp,
|
currentValue: maxTemp,
|
||||||
|
|||||||
@@ -7,12 +7,14 @@ import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"
|
|||||||
import { ICheckService } from "../domain/checks/check.service.js";
|
import { ICheckService } from "../domain/checks/check.service.js";
|
||||||
import { DockerLog } from "@/domain/docker/docker-log.type.js";
|
import { DockerLog } from "@/domain/docker/docker-log.type.js";
|
||||||
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
|
||||||
|
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||||
const SERVICE_NAME = "BufferService";
|
const SERVICE_NAME = "BufferService";
|
||||||
|
|
||||||
export interface IBufferService {
|
export interface IBufferService {
|
||||||
addToBuffer(check: Check): void;
|
addToBuffer(check: Check): void;
|
||||||
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
|
addGeoCheckToBuffer(geoCheck: GeoCheck): void;
|
||||||
addDockerLogToBuffer(dockerLog: DockerLog): void;
|
addDockerLogToBuffer(dockerLog: DockerLog): void;
|
||||||
|
ingestChecks(checks: Check[]): Promise<void>;
|
||||||
scheduleNextFlush(): void;
|
scheduleNextFlush(): void;
|
||||||
flushBuffer(): Promise<void>;
|
flushBuffer(): Promise<void>;
|
||||||
flushGeoBuffer(): Promise<void>;
|
flushGeoBuffer(): Promise<void>;
|
||||||
@@ -34,6 +36,9 @@ export class BufferService implements IBufferService {
|
|||||||
private dockerLogsService: IDockerLogsService;
|
private dockerLogsService: IDockerLogsService;
|
||||||
private jobsRepository: IJobsRepository;
|
private jobsRepository: IJobsRepository;
|
||||||
|
|
||||||
|
// Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated.
|
||||||
|
private unarmed: Map<string, PendingCheck[]> = new Map();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
logger: ILogger,
|
logger: ILogger,
|
||||||
checkService: ICheckService,
|
checkService: ICheckService,
|
||||||
@@ -99,6 +104,38 @@ export class BufferService implements IBufferService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ingestChecks = async (checks: Check[]) => {
|
||||||
|
if (checks.length > 0) await this.checksService.createChecks(checks);
|
||||||
|
|
||||||
|
// Group by monitor, starting from any entries a previous flush failed to arm
|
||||||
|
const byMonitor = this.unarmed;
|
||||||
|
this.unarmed = new Map();
|
||||||
|
for (const check of checks) {
|
||||||
|
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
|
||||||
|
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
|
||||||
|
byMonitor.set(check.metadata.monitorId, pending);
|
||||||
|
}
|
||||||
|
if (byMonitor.size === 0) return;
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
const ops: Promise<boolean>[] = [];
|
||||||
|
for (const [monitorId, pendingChecks] of byMonitor) {
|
||||||
|
ops.push(
|
||||||
|
this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
|
||||||
|
// The checks are already stored, so a failed arm is retried on the next flush rather than dropped
|
||||||
|
this.unarmed.set(monitorId, pendingChecks);
|
||||||
|
this.logger.error({
|
||||||
|
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`,
|
||||||
|
service: this.SERVICE_NAME,
|
||||||
|
method: "ingestChecks",
|
||||||
|
});
|
||||||
|
return false;
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await Promise.all(ops);
|
||||||
|
};
|
||||||
|
|
||||||
scheduleNextFlush() {
|
scheduleNextFlush() {
|
||||||
if (this.bufferTimer) {
|
if (this.bufferTimer) {
|
||||||
clearTimeout(this.bufferTimer);
|
clearTimeout(this.bufferTimer);
|
||||||
@@ -121,8 +158,9 @@ export class BufferService implements IBufferService {
|
|||||||
}
|
}
|
||||||
}, this.BUFFER_TIMEOUT);
|
}, this.BUFFER_TIMEOUT);
|
||||||
}
|
}
|
||||||
|
|
||||||
async flushBuffer() {
|
async flushBuffer() {
|
||||||
if (this.buffer.length === 0) {
|
if (this.buffer.length === 0 && this.unarmed.size === 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Take the batch first so a write that drains it can't be appended to mid-flush
|
// Take the batch first so a write that drains it can't be appended to mid-flush
|
||||||
@@ -134,11 +172,7 @@ export class BufferService implements IBufferService {
|
|||||||
service: this.SERVICE_NAME,
|
service: this.SERVICE_NAME,
|
||||||
method: "flushBuffer",
|
method: "flushBuffer",
|
||||||
});
|
});
|
||||||
await this.checksService.createChecks(batch);
|
await this.ingestChecks(batch);
|
||||||
// Need to evaluate checks when they are flushed
|
|
||||||
const monitorIds = [...new Set(batch.map((check) => check.metadata.monitorId))];
|
|
||||||
const now = Date.now();
|
|
||||||
await Promise.all(monitorIds.map((monitorId) => this.jobsRepository.upsertEvaluate(monitorId, now)));
|
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
this.logger.error({
|
this.logger.error({
|
||||||
message: error instanceof Error ? error.message : "Unknown error",
|
message: error instanceof Error ? error.message : "Unknown error",
|
||||||
|
|||||||
@@ -120,19 +120,41 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
|||||||
// ********************
|
// ********************
|
||||||
|
|
||||||
private runEvaluate = async (job: Job) => {
|
private runEvaluate = async (job: Job) => {
|
||||||
if (!job.refId) return;
|
if (!job.refId || job.pendingChecks.length === 0) return;
|
||||||
|
const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
|
||||||
|
|
||||||
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
|
||||||
if (!monitor) return;
|
if (!monitor) {
|
||||||
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
|
await this.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
|
||||||
|
|
||||||
let current = monitor;
|
let current = monitor;
|
||||||
for (const check of checks) {
|
for (const check of checks) {
|
||||||
const status = this.checkService.toStatusResponse(check);
|
const status = this.checkService.toStatusResponse(check);
|
||||||
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
|
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
|
||||||
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||||
await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
|
const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||||
|
if (!leaseHeld) {
|
||||||
|
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
|
||||||
|
this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
|
||||||
|
const found = new Set(checks.map((check) => check.id));
|
||||||
|
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
|
||||||
|
if (missing.length > 0) {
|
||||||
|
this.logger.warn({
|
||||||
|
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
|
||||||
|
service: SERVICE_NAME,
|
||||||
|
method: "runEvaluate",
|
||||||
|
});
|
||||||
|
await this.jobsRepository.pullEvaluated(job.id, missing);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
private runGeoCheck = async (job: Job) => {
|
private runGeoCheck = async (job: Job) => {
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
import { describe, expect, it, jest, beforeAll, afterAll, beforeEach, afterEach } from "@jest/globals";
|
||||||
|
import mongoose from "mongoose";
|
||||||
|
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||||
|
import MongoChecksRepository from "../../src/domain/checks/check.repository.mongo.ts";
|
||||||
|
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
||||||
|
import { CheckModel } from "../../src/domain/checks/check.model.ts";
|
||||||
|
import JobModel from "../../src/domain/jobs/job.model.ts";
|
||||||
|
import { CheckService } from "../../src/domain/checks/check.service.ts";
|
||||||
|
import { BufferService } from "../../src/service/bufferService.ts";
|
||||||
|
import { DBQueueWorker } from "../../src/worker/worker.db-queue.ts";
|
||||||
|
import { InMemoryMonitorsRepository } from "../helpers/InMemoryMonitorsRepository.ts";
|
||||||
|
import { createMockLogger } from "../helpers/createMockLogger.ts";
|
||||||
|
import type { Job } from "../../src/domain/jobs/job.type.ts";
|
||||||
|
import type { Check } from "../../src/domain/checks/check.type.ts";
|
||||||
|
import type { Monitor } from "../../src/domain/monitors/monitor.type.ts";
|
||||||
|
import type { MonitorStatusResponse } from "../../src/types/network.ts";
|
||||||
|
import type { ILogger } from "../../src/utils/logger.ts";
|
||||||
|
|
||||||
|
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||||
|
// Regression test for the dropped-check bug in the check-to-evaluate handoff.
|
||||||
|
//
|
||||||
|
// The evaluate step used to be driven by a timestamp cursor: the evaluator read checks with
|
||||||
|
// createdAt strictly after the monitor's lastEvaluatedAt and advanced that cursor to each
|
||||||
|
// check's createdAt. Each worker's BufferService flushes its own in-memory batch on its own
|
||||||
|
// timer, so with two processing workers a later check produced on worker B could be flushed and
|
||||||
|
// evaluated before an earlier check produced on worker A had been flushed. When A's batch landed,
|
||||||
|
// its check was older than the cursor and the evaluate query never returned it. The check existed
|
||||||
|
// in the collection but never reached the status window, stats, incidents or notifications.
|
||||||
|
//
|
||||||
|
// The evaluate row now carries the ids of every stored check, pushed at ingest and pulled as each
|
||||||
|
// is applied, so a late batch is always evaluated. This drives the real time-series query, the
|
||||||
|
// real jobs collection, the real buffer flush and the real DBQueueWorker evaluate path against an
|
||||||
|
// in-process mongod, so the handoff semantics under test are MongoDB's, not a mock's.
|
||||||
|
|
||||||
|
let mongod: MongoMemoryServer;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
mongod = await MongoMemoryServer.create();
|
||||||
|
await mongoose.connect(mongod.getUri());
|
||||||
|
await CheckModel.createCollection(); // timeseries collections must exist before insert
|
||||||
|
await JobModel.init();
|
||||||
|
}, 120_000);
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await mongoose.disconnect();
|
||||||
|
await mongod.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await CheckModel.deleteMany({});
|
||||||
|
await JobModel.deleteMany({});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const MONITOR_ID = new mongoose.Types.ObjectId().toString();
|
||||||
|
const TEAM_ID = new mongoose.Types.ObjectId().toString();
|
||||||
|
const T0 = Date.parse("2026-01-01T00:00:00.000Z");
|
||||||
|
const INTERVAL_MS = 30_000;
|
||||||
|
|
||||||
|
const makeMonitor = (): Monitor =>
|
||||||
|
({
|
||||||
|
id: MONITOR_ID,
|
||||||
|
teamId: TEAM_ID,
|
||||||
|
type: "http",
|
||||||
|
interval: INTERVAL_MS,
|
||||||
|
status: "up",
|
||||||
|
isActive: true,
|
||||||
|
statusWindow: [],
|
||||||
|
statusWindowSize: 5,
|
||||||
|
statusWindowThreshold: 60,
|
||||||
|
recentChecks: [],
|
||||||
|
lastEvaluatedAt: 0,
|
||||||
|
}) as unknown as Monitor;
|
||||||
|
|
||||||
|
const makeStatusResponse = (): MonitorStatusResponse => ({
|
||||||
|
monitorId: MONITOR_ID,
|
||||||
|
teamId: TEAM_ID,
|
||||||
|
type: "http",
|
||||||
|
status: true,
|
||||||
|
code: 200,
|
||||||
|
message: "OK",
|
||||||
|
responseTime: 100,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Production stamps createdAt at the moment the probe ran, on the producing worker; pin it so
|
||||||
|
// the two workers' checks have a known order regardless of wall clock.
|
||||||
|
const producedAt = (check: Check, at: number): Check => {
|
||||||
|
const createdAt = new Date(at).toISOString();
|
||||||
|
return { ...check, createdAt, updatedAt: createdAt };
|
||||||
|
};
|
||||||
|
|
||||||
|
const stubSettingsService = { getSettings: () => ({ nodeEnv: "production" }) };
|
||||||
|
const stubGeoChecksService = { createGeoChecks: jest.fn<any>().mockResolvedValue([]) };
|
||||||
|
const stubDockerLogsService = { createDockerLogs: jest.fn<any>().mockResolvedValue(0) };
|
||||||
|
|
||||||
|
describe("Evaluate cursor with more than one processing worker", () => {
|
||||||
|
let logger: ILogger;
|
||||||
|
let monitorsRepository: InMemoryMonitorsRepository;
|
||||||
|
let checksRepository: MongoChecksRepository;
|
||||||
|
let checkService: CheckService;
|
||||||
|
let bufferA: BufferService;
|
||||||
|
let bufferB: BufferService;
|
||||||
|
let evaluatedCheckIds: string[];
|
||||||
|
let evaluator: DBQueueWorker;
|
||||||
|
let evaluatorJobsRepository: MongoJobsRepository;
|
||||||
|
|
||||||
|
// One BufferService per worker process, each flushing into the same collection and jobs table.
|
||||||
|
const createWorkerBuffer = (workerId: string) =>
|
||||||
|
new BufferService(
|
||||||
|
logger,
|
||||||
|
checkService,
|
||||||
|
stubGeoChecksService as any,
|
||||||
|
stubDockerLogsService as any,
|
||||||
|
stubSettingsService as any,
|
||||||
|
new MongoJobsRepository(workerId)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Claim every due evaluate job and run it through the worker's real per-job path.
|
||||||
|
const runDueEvaluateJobs = async () => {
|
||||||
|
const jobs = await evaluatorJobsRepository.claimDueBatch("evaluate", 20, Date.now());
|
||||||
|
const runJob = (evaluator as unknown as { runJob: (job: Job) => Promise<void> }).runJob;
|
||||||
|
for (const job of jobs) await runJob(job);
|
||||||
|
return jobs.length;
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
logger = createMockLogger() as unknown as ILogger;
|
||||||
|
monitorsRepository = new InMemoryMonitorsRepository();
|
||||||
|
monitorsRepository.seed(makeMonitor());
|
||||||
|
checksRepository = new MongoChecksRepository(logger);
|
||||||
|
checkService = new CheckService(monitorsRepository, logger, checksRepository);
|
||||||
|
bufferA = createWorkerBuffer("worker-a");
|
||||||
|
bufferB = createWorkerBuffer("worker-b");
|
||||||
|
|
||||||
|
evaluatedCheckIds = [];
|
||||||
|
evaluatorJobsRepository = new MongoJobsRepository("worker-b");
|
||||||
|
evaluator = new DBQueueWorker({
|
||||||
|
logger,
|
||||||
|
isDbConnected: () => true,
|
||||||
|
jobsRepository: evaluatorJobsRepository,
|
||||||
|
monitorsRepository,
|
||||||
|
checksRepository,
|
||||||
|
checkService,
|
||||||
|
bufferService: bufferB,
|
||||||
|
checkProducer: { produce: jest.fn<any>() },
|
||||||
|
checkEvaluator: {
|
||||||
|
evaluate: async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
|
||||||
|
evaluatedCheckIds.push(check.id);
|
||||||
|
return {
|
||||||
|
monitor,
|
||||||
|
status,
|
||||||
|
check,
|
||||||
|
statusChange: { monitor, statusChanged: false, prevStatus: monitor.status, code: 200, timestamp: 0 },
|
||||||
|
decision: {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
} as any,
|
||||||
|
geoCheckPipeline: { run: jest.fn<any>() },
|
||||||
|
dispatcher: { dispatch: jest.fn<any>().mockResolvedValue(undefined) },
|
||||||
|
helper: {} as any,
|
||||||
|
queueWorkersRepository: {} as any,
|
||||||
|
queueMode: "worker",
|
||||||
|
queuePrimaryProcesses: true,
|
||||||
|
workerId: "worker-b",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await bufferA.shutdown();
|
||||||
|
await bufferB.shutdown();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("evaluates every persisted check even when a later worker's batch flushes first", async () => {
|
||||||
|
// t=0: worker A holds the check job, probes, and buffers the result. Its flush timer has not fired.
|
||||||
|
const c1 = producedAt(checkService.toCheck(makeStatusResponse())!, T0);
|
||||||
|
bufferA.addToBuffer(c1);
|
||||||
|
|
||||||
|
// t=30s: worker B claims the next run of the same job, probes, and buffers the result.
|
||||||
|
const c2 = producedAt(checkService.toCheck(makeStatusResponse())!, T0 + INTERVAL_MS);
|
||||||
|
bufferB.addToBuffer(c2);
|
||||||
|
|
||||||
|
// t=45s: worker B's flush timer fires first. c2 is inserted and an evaluate job is queued.
|
||||||
|
await bufferB.flushBuffer();
|
||||||
|
expect(await runDueEvaluateJobs()).toBe(1);
|
||||||
|
expect(evaluatedCheckIds).toEqual([c2.id]);
|
||||||
|
|
||||||
|
// t=60s: worker A's flush timer fires. c1 is inserted and the evaluate job is pulled due again.
|
||||||
|
await bufferA.flushBuffer();
|
||||||
|
expect(await runDueEvaluateJobs()).toBe(1);
|
||||||
|
|
||||||
|
// Both checks are durably stored and visible in history.
|
||||||
|
const stored = await CheckModel.find({ "metadata.monitorId": new mongoose.Types.ObjectId(MONITOR_ID) }).lean();
|
||||||
|
expect(stored.map((doc) => String(doc._id)).sort()).toEqual([c1.id, c2.id].sort());
|
||||||
|
|
||||||
|
// Every stored check must reach the evaluator exactly once.
|
||||||
|
expect(evaluatedCheckIds).toHaveLength(2);
|
||||||
|
expect(evaluatedCheckIds).toContain(c1.id);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -3,7 +3,7 @@ import mongoose from "mongoose";
|
|||||||
import { MongoMemoryServer } from "mongodb-memory-server";
|
import { MongoMemoryServer } from "mongodb-memory-server";
|
||||||
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
import MongoJobsRepository from "../../src/domain/jobs/job.repository.mongo.ts";
|
||||||
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
|
import JobModel, { type JobDocument } from "../../src/domain/jobs/job.model.ts";
|
||||||
import { BACKOFF_MS, LOCK_MS, PARKED, type Job, type JobType } from "../../src/domain/jobs/job.type.ts";
|
import { BACKOFF_MS, LOCK_MS, PARKED, type Job, type JobType, type PendingCheck } from "../../src/domain/jobs/job.type.ts";
|
||||||
|
|
||||||
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
// ── Real-Mongo harness ─────────────────────────────────────────────────────────
|
||||||
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
|
// The headline guarantee of the queue — "exactly one claimer wins" — IS MongoDB's
|
||||||
@@ -58,6 +58,9 @@ const ownedBy = (r: MongoJobsRepository) => (r as unknown as { workerId: string
|
|||||||
|
|
||||||
const WORKER_ID = "worker-under-test";
|
const WORKER_ID = "worker-under-test";
|
||||||
|
|
||||||
|
// Pending entries as ingest pushes them: one per inserted check, stamped with the check's createdAt.
|
||||||
|
const pending = (...checkIds: string[]): PendingCheck[] => checkIds.map((checkId, i) => ({ checkId, createdAt: NOW + i }));
|
||||||
|
|
||||||
describe("MongoJobsRepository", () => {
|
describe("MongoJobsRepository", () => {
|
||||||
let repo: MongoJobsRepository;
|
let repo: MongoJobsRepository;
|
||||||
|
|
||||||
@@ -239,6 +242,23 @@ describe("MongoJobsRepository", () => {
|
|||||||
expect(row?.failCount).toBe(2);
|
expect(row?.failCount).toBe(2);
|
||||||
expect(row?.lastFailReason).toBe("timeout");
|
expect(row?.lastFailReason).toBe("timeout");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("re-arms instead of parking when checks were pushed while the run held the lease", async () => {
|
||||||
|
await seedJob({
|
||||||
|
_id: "evaluate:mon-1",
|
||||||
|
type: "evaluate",
|
||||||
|
intervalMs: null,
|
||||||
|
lockedBy: ownedBy(repo),
|
||||||
|
lockedUntil: NOW + LOCK_MS,
|
||||||
|
pendingChecks: pending("c-late"), // arrived mid-run, not yet pulled
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await repo.recordOneShot("evaluate:mon-1", NOW + 1_000)).toBe(true);
|
||||||
|
const row = await readRow("evaluate:mon-1");
|
||||||
|
expect(row?.nextScheduledAt).toBe(NOW + 1_000); // due now, not PARKED
|
||||||
|
expect(row?.lockedBy).toBeNull();
|
||||||
|
expect(row?.runCount).toBe(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// ── renewLocks (lease renewal) ───────────────────────────────────────────────
|
// ── renewLocks (lease renewal) ───────────────────────────────────────────────
|
||||||
@@ -325,10 +345,10 @@ describe("MongoJobsRepository", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// ── upsertEvaluate (handoff + $min coalescing) ───────────────────────────────
|
// ── upsertEvaluate (handoff: $push ids + $min due time) ──────────────────────
|
||||||
describe("upsertEvaluate", () => {
|
describe("upsertEvaluate", () => {
|
||||||
it("creates a single evaluate row keyed by monitor", async () => {
|
it("creates a single evaluate row keyed by monitor carrying the pushed ids", async () => {
|
||||||
await repo.upsertEvaluate("mon-1", NOW);
|
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
|
||||||
|
|
||||||
const row = await readRow("evaluate:mon-1");
|
const row = await readRow("evaluate:mon-1");
|
||||||
expect(row?.type).toBe("evaluate");
|
expect(row?.type).toBe("evaluate");
|
||||||
@@ -336,11 +356,12 @@ describe("MongoJobsRepository", () => {
|
|||||||
expect(row?.nextScheduledAt).toBe(NOW);
|
expect(row?.nextScheduledAt).toBe(NOW);
|
||||||
expect(row?.intervalMs).toBeNull();
|
expect(row?.intervalMs).toBeNull();
|
||||||
expect(row?.isActive).toBe(true);
|
expect(row?.isActive).toBe(true);
|
||||||
|
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
|
||||||
});
|
});
|
||||||
|
|
||||||
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
|
it("coalesces rapid checks: a later call never pushes runAt out ($min)", async () => {
|
||||||
await repo.upsertEvaluate("mon-1", NOW);
|
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
|
||||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
await repo.upsertEvaluate("mon-1", pending("c2"), NOW + 9_000);
|
||||||
|
|
||||||
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
|
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
|
||||||
expect(rows).toHaveLength(1);
|
expect(rows).toHaveLength(1);
|
||||||
@@ -348,19 +369,117 @@ describe("MongoJobsRepository", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
|
it("lowers runAt when an earlier evaluation arrives ($min)", async () => {
|
||||||
await repo.upsertEvaluate("mon-1", NOW + 9_000);
|
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 9_000);
|
||||||
await repo.upsertEvaluate("mon-1", NOW);
|
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
|
||||||
|
|
||||||
const row = await readRow("evaluate:mon-1");
|
const row = await readRow("evaluate:mon-1");
|
||||||
expect(row?.nextScheduledAt).toBe(NOW);
|
expect(row?.nextScheduledAt).toBe(NOW);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("concurrent upserts for one monitor produce exactly one row", async () => {
|
it("never duplicates an entry when the same batch is armed twice (retry after a failed arm)", async () => {
|
||||||
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW);
|
||||||
await Promise.all(workers.map((w) => w.upsertEvaluate("mon-1", NOW)));
|
await repo.upsertEvaluate("mon-1", pending("c1", "c2"), NOW + 1_000);
|
||||||
|
|
||||||
const count = await JobModel.countDocuments({ type: "evaluate", refId: "mon-1" });
|
const row = await readRow("evaluate:mon-1");
|
||||||
expect(count).toBe(1);
|
expect(row?.pendingChecks).toEqual(pending("c1", "c2"));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accumulates ids across flushes so a late batch is never lost", async () => {
|
||||||
|
await repo.upsertEvaluate("mon-1", pending("c2"), NOW); // later check, flushed first
|
||||||
|
await repo.upsertEvaluate("mon-1", pending("c1"), NOW + 15_000); // earlier check, flushed later
|
||||||
|
|
||||||
|
const row = await readRow("evaluate:mon-1");
|
||||||
|
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2", "c1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps pushing onto a row that is currently leased by a worker", async () => {
|
||||||
|
await seedJob({
|
||||||
|
_id: "evaluate:mon-1",
|
||||||
|
type: "evaluate",
|
||||||
|
intervalMs: null,
|
||||||
|
lockedBy: "other-worker",
|
||||||
|
lockedUntil: NOW + LOCK_MS,
|
||||||
|
pendingChecks: pending("c1"),
|
||||||
|
});
|
||||||
|
|
||||||
|
await repo.upsertEvaluate("mon-1", pending("c2"), NOW);
|
||||||
|
|
||||||
|
const row = await readRow("evaluate:mon-1");
|
||||||
|
expect(row?.lockedBy).toBe("other-worker"); // lease untouched
|
||||||
|
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1", "c2"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("concurrent upserts for one monitor produce exactly one row holding every id", async () => {
|
||||||
|
const workers = Array.from({ length: 20 }, (_, i) => new MongoJobsRepository(`worker-${i}`));
|
||||||
|
await Promise.all(workers.map((w, i) => w.upsertEvaluate("mon-1", pending(`c${i}`), NOW)));
|
||||||
|
|
||||||
|
const rows = await JobModel.find({ type: "evaluate", refId: "mon-1" }).lean<JobDocument[]>();
|
||||||
|
expect(rows).toHaveLength(1);
|
||||||
|
expect(rows[0].pendingChecks).toHaveLength(20);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── pullEvaluated (evaluator removes applied ids) ───────────────────────────
|
||||||
|
describe("pullEvaluated", () => {
|
||||||
|
it("removes only the listed ids and leaves the rest", async () => {
|
||||||
|
await seedJob({
|
||||||
|
_id: "evaluate:mon-1",
|
||||||
|
type: "evaluate",
|
||||||
|
intervalMs: null,
|
||||||
|
lockedBy: ownedBy(repo),
|
||||||
|
lockedUntil: NOW + LOCK_MS,
|
||||||
|
pendingChecks: pending("c1", "c2", "c3"),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1", "c3"])).toBe(true);
|
||||||
|
|
||||||
|
const row = await readRow("evaluate:mon-1");
|
||||||
|
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c2"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is fenced on the lease: another worker's row is untouched", async () => {
|
||||||
|
await seedJob({
|
||||||
|
_id: "evaluate:mon-1",
|
||||||
|
type: "evaluate",
|
||||||
|
intervalMs: null,
|
||||||
|
lockedBy: "other-worker",
|
||||||
|
lockedUntil: NOW + LOCK_MS,
|
||||||
|
pendingChecks: pending("c1"),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(false);
|
||||||
|
|
||||||
|
const row = await readRow("evaluate:mon-1");
|
||||||
|
expect(row?.pendingChecks.map((entry) => entry.checkId)).toEqual(["c1"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports the lease as held even when the ids were already pulled", async () => {
|
||||||
|
await seedJob({
|
||||||
|
_id: "evaluate:mon-1",
|
||||||
|
type: "evaluate",
|
||||||
|
intervalMs: null,
|
||||||
|
lockedBy: ownedBy(repo),
|
||||||
|
lockedUntil: NOW + LOCK_MS,
|
||||||
|
pendingChecks: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await repo.pullEvaluated("evaluate:mon-1", ["c1"])).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is a no-op for an empty id list", async () => {
|
||||||
|
await seedJob({
|
||||||
|
_id: "evaluate:mon-1",
|
||||||
|
type: "evaluate",
|
||||||
|
intervalMs: null,
|
||||||
|
lockedBy: ownedBy(repo),
|
||||||
|
lockedUntil: NOW + LOCK_MS,
|
||||||
|
pendingChecks: pending("c1"),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(await repo.pullEvaluated("evaluate:mon-1", [])).toBe(true);
|
||||||
|
|
||||||
|
const row = await readRow("evaluate:mon-1");
|
||||||
|
expect(row?.pendingChecks).toHaveLength(1);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -376,6 +495,7 @@ describe("MongoJobsRepository", () => {
|
|||||||
intervalMs: 60_000,
|
intervalMs: 60_000,
|
||||||
lockedBy: null,
|
lockedBy: null,
|
||||||
lockedUntil: null,
|
lockedUntil: null,
|
||||||
|
pendingChecks: [],
|
||||||
runCount: 0,
|
runCount: 0,
|
||||||
failCount: 0,
|
failCount: 0,
|
||||||
lastFinishedAt: null,
|
lastFinishedAt: null,
|
||||||
@@ -407,6 +527,7 @@ describe("MongoJobsRepository", () => {
|
|||||||
intervalMs: 86_400_000,
|
intervalMs: 86_400_000,
|
||||||
lockedBy: null,
|
lockedBy: null,
|
||||||
lockedUntil: null,
|
lockedUntil: null,
|
||||||
|
pendingChecks: [],
|
||||||
runCount: 0,
|
runCount: 0,
|
||||||
failCount: 0,
|
failCount: 0,
|
||||||
lastFinishedAt: null,
|
lastFinishedAt: null,
|
||||||
@@ -487,7 +608,7 @@ describe("MongoJobsRepository", () => {
|
|||||||
it("deleteById drops every row for the monitor", async () => {
|
it("deleteById drops every row for the monitor", async () => {
|
||||||
await seedJob({ _id: "check:mon-1", type: "check" });
|
await seedJob({ _id: "check:mon-1", type: "check" });
|
||||||
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
await seedJob({ _id: "geo:mon-1", type: "geo-check" });
|
||||||
await repo.upsertEvaluate("mon-1", NOW);
|
await repo.upsertEvaluate("mon-1", pending("c1"), NOW);
|
||||||
|
|
||||||
await repo.deleteById("mon-1");
|
await repo.deleteById("mon-1");
|
||||||
|
|
||||||
|
|||||||
@@ -45,6 +45,8 @@ const makeCheck = (overrides?: Partial<Check>): Check =>
|
|||||||
statusCode: 200,
|
statusCode: 200,
|
||||||
responseTime: 100,
|
responseTime: 100,
|
||||||
message: "OK",
|
message: "OK",
|
||||||
|
createdAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||||
...overrides,
|
...overrides,
|
||||||
}) as Check;
|
}) as Check;
|
||||||
|
|
||||||
@@ -392,9 +394,44 @@ describe("BufferService", () => {
|
|||||||
|
|
||||||
await service.flushBuffer();
|
await service.flushBuffer();
|
||||||
|
|
||||||
|
const createdAt = Date.parse("2026-01-01T00:00:00.000Z");
|
||||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-1", expect.any(Number));
|
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith(
|
||||||
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", expect.any(Number));
|
"mon-1",
|
||||||
|
[
|
||||||
|
{ checkId: "c1", createdAt },
|
||||||
|
{ checkId: "c2", createdAt },
|
||||||
|
],
|
||||||
|
expect.any(Number)
|
||||||
|
);
|
||||||
|
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledWith("mon-2", [{ checkId: "c3", createdAt }], expect.any(Number));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries a failed arm on the next flush without re-storing the checks", async () => {
|
||||||
|
const { service, checkService, jobsRepository, logger } = createService();
|
||||||
|
(jobsRepository.upsertEvaluate as jest.Mock).mockRejectedValueOnce(new Error("arm failed"));
|
||||||
|
service.addToBuffer(makeCheck({ id: "c1" }));
|
||||||
|
|
||||||
|
await service.flushBuffer();
|
||||||
|
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||||
|
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ method: "ingestChecks" }));
|
||||||
|
|
||||||
|
await service.flushBuffer(); // buffer is empty, only the retry runs
|
||||||
|
expect(checkService.createChecks).toHaveBeenCalledTimes(1);
|
||||||
|
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||||
|
expect(jobsRepository.upsertEvaluate).toHaveBeenLastCalledWith("mon-1", [{ checkId: "c1", createdAt: expect.any(Number) }], expect.any(Number));
|
||||||
|
|
||||||
|
await service.flushBuffer(); // nothing left to retry
|
||||||
|
expect(jobsRepository.upsertEvaluate).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ingestChecks with an empty batch neither writes nor arms", async () => {
|
||||||
|
const { service, checkService, jobsRepository } = createService();
|
||||||
|
|
||||||
|
await service.ingestChecks([]);
|
||||||
|
|
||||||
|
expect(checkService.createChecks).not.toHaveBeenCalled();
|
||||||
|
expect(jobsRepository.upsertEvaluate).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not arm the evaluate stage when the check write fails", async () => {
|
it("does not arm the evaluate stage when the check write fails", async () => {
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
|
|||||||
intervalMs: 60000,
|
intervalMs: 60000,
|
||||||
lockedBy: null,
|
lockedBy: null,
|
||||||
lockedUntil: null,
|
lockedUntil: null,
|
||||||
|
pendingChecks: [],
|
||||||
runCount: 0,
|
runCount: 0,
|
||||||
failCount: 0,
|
failCount: 0,
|
||||||
lastFinishedAt: null,
|
lastFinishedAt: null,
|
||||||
@@ -63,12 +64,22 @@ const makeJob = (overrides?: Partial<Job>): Job => ({
|
|||||||
...overrides,
|
...overrides,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// An evaluate row as ingest leaves it: one pending entry per stored check
|
||||||
|
const makeEvaluateJob = (...checkIds: string[]): Job =>
|
||||||
|
makeJob({
|
||||||
|
id: "evaluate:m1",
|
||||||
|
type: "evaluate",
|
||||||
|
intervalMs: null,
|
||||||
|
pendingChecks: checkIds.map((checkId, i) => ({ checkId, createdAt: 1000 + i })),
|
||||||
|
});
|
||||||
|
|
||||||
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
|
const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses?: boolean; mocks?: Record<string, any> }) => {
|
||||||
const jobsRepository = {
|
const jobsRepository = {
|
||||||
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
|
claimDueBatch: jest.fn<any>().mockResolvedValue([]),
|
||||||
upsertJob: jest.fn<any>().mockResolvedValue(true),
|
upsertJob: jest.fn<any>().mockResolvedValue(true),
|
||||||
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
|
upsertCleanupJob: jest.fn<any>().mockResolvedValue(true),
|
||||||
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
|
upsertEvaluate: jest.fn<any>().mockResolvedValue(true),
|
||||||
|
pullEvaluated: jest.fn<any>().mockResolvedValue(true),
|
||||||
recordSuccess: jest.fn<any>().mockResolvedValue(true),
|
recordSuccess: jest.fn<any>().mockResolvedValue(true),
|
||||||
recordFailure: jest.fn<any>().mockResolvedValue(true),
|
recordFailure: jest.fn<any>().mockResolvedValue(true),
|
||||||
recordOneShot: jest.fn<any>().mockResolvedValue(true),
|
recordOneShot: jest.fn<any>().mockResolvedValue(true),
|
||||||
@@ -92,7 +103,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses
|
|||||||
};
|
};
|
||||||
const checkService = {
|
const checkService = {
|
||||||
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
|
toStatusResponse: jest.fn<any>().mockReturnValue({ status: "up" }),
|
||||||
toLastEvaluatedAt: jest.fn<any>().mockReturnValue(12345),
|
|
||||||
};
|
};
|
||||||
const bufferService = {
|
const bufferService = {
|
||||||
addToBuffer: jest.fn<any>(),
|
addToBuffer: jest.fn<any>(),
|
||||||
@@ -369,27 +379,78 @@ describe("DBQueueWorker", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
|
it("parks a one-shot job (intervalMs null) via recordOneShot instead of rescheduling", async () => {
|
||||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
const job = makeEvaluateJob("c1");
|
||||||
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||||
|
|
||||||
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
||||||
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
expect(mocks.jobsRepository.recordSuccess).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("evaluate job dispatches each unevaluated check and advances lastEvaluatedAt", async () => {
|
it("evaluate job loads the row's pending checks, dispatches each, and pulls its id after dispatch", async () => {
|
||||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
const job = makeEvaluateJob("c1");
|
||||||
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
|
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) };
|
||||||
const { mocks } = await start({
|
const { mocks } = await start({
|
||||||
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||||
});
|
});
|
||||||
|
|
||||||
|
expect(checksRepository.findUnevaluatedByMonitorId).toHaveBeenCalledWith("m1", job.pendingChecks);
|
||||||
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
|
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalled();
|
||||||
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
|
expect(mocks.dispatcher.dispatch).toHaveBeenCalled();
|
||||||
expect(mocks.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { lastEvaluatedAt: 12345 });
|
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||||
|
// The monitor cursor is gone: nothing writes lastEvaluatedAt
|
||||||
|
expect(mocks.monitorsRepository.updateById).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("evaluate job with no pending checks does nothing", async () => {
|
||||||
|
const job = makeEvaluateJob();
|
||||||
|
const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } });
|
||||||
|
|
||||||
|
expect(mocks.monitorsRepository.findByIdLean).not.toHaveBeenCalled();
|
||||||
|
expect(mocks.checksRepository.findUnevaluatedByMonitorId).not.toHaveBeenCalled();
|
||||||
|
expect(mocks.jobsRepository.recordOneShot).toHaveBeenCalledWith(job.id, expect.any(Number));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("evaluate job for a deleted monitor drops every pending id", async () => {
|
||||||
|
const job = makeEvaluateJob("c1", "c2");
|
||||||
|
const monitorsRepository = { findByIdLean: jest.fn<any>().mockResolvedValue(null), updateById: jest.fn<any>() };
|
||||||
|
const { mocks } = await start({
|
||||||
|
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, monitorsRepository },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mocks.checkEvaluator.evaluate).not.toHaveBeenCalled();
|
||||||
|
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1", "c2"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("evaluate job stops applying checks once the lease is lost so a second claimer does not double-apply", async () => {
|
||||||
|
const job = makeEvaluateJob("c1", "c2", "c3");
|
||||||
|
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }, { id: "c2" }, { id: "c3" }]) };
|
||||||
|
const jobsRepository = {
|
||||||
|
...createWorker().mocks.jobsRepository,
|
||||||
|
claimDueBatch: claimOnce(job),
|
||||||
|
pullEvaluated: jest.fn<any>().mockResolvedValueOnce(true).mockResolvedValueOnce(false), // lease lost after c2
|
||||||
|
};
|
||||||
|
const { mocks } = await start({ mocks: { jobsRepository, checksRepository } });
|
||||||
|
|
||||||
|
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(2);
|
||||||
|
expect(mocks.dispatcher.dispatch).toHaveBeenCalledTimes(2);
|
||||||
|
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledTimes(2); // no trailing sweep either
|
||||||
|
expect(mocks.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ method: "runEvaluate" }));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("evaluate job pulls ids whose check no longer exists so they do not sit on the row forever", async () => {
|
||||||
|
const job = makeEvaluateJob("c1", "c-gone");
|
||||||
|
const checksRepository = { findUnevaluatedByMonitorId: jest.fn<any>().mockResolvedValue([{ id: "c1" }]) }; // c-gone was cleaned up
|
||||||
|
const { mocks } = await start({
|
||||||
|
mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, checksRepository },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mocks.checkEvaluator.evaluate).toHaveBeenCalledTimes(1);
|
||||||
|
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c1"]);
|
||||||
|
expect(mocks.jobsRepository.pullEvaluated).toHaveBeenCalledWith(job.id, ["c-gone"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
|
it("evaluate job reads the monitor once and threads the post-write monitor across the backlog", async () => {
|
||||||
const job = makeJob({ id: "evaluate:m1", type: "evaluate", intervalMs: null });
|
const job = makeEvaluateJob("c1", "c2");
|
||||||
const initialMonitor = makeMonitor();
|
const initialMonitor = makeMonitor();
|
||||||
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
|
const postWriteMonitor = makeMonitor({ status: "down" }); // what updateStatusWindowAndChecks would return after check c1
|
||||||
const monitorsRepository = {
|
const monitorsRepository = {
|
||||||
|
|||||||
@@ -685,6 +685,22 @@ describe("NotificationMessageBuilder", () => {
|
|||||||
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
|
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// A temperature exactly AT the threshold must not report a breach here, matching
|
||||||
|
// StatusService.computeHardwareStatus's own strict `>` comparison (cpu/memory/disk
|
||||||
|
// already agree with this file on `>`; temp alone used to diverge with `>=`, so an
|
||||||
|
// incident-driving check at exactly the threshold never fired but the notification
|
||||||
|
// content for a breach some OTHER metric triggered could still list temp as breaching).
|
||||||
|
it("does not report temperature breach when exactly at threshold", () => {
|
||||||
|
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: 80 });
|
||||||
|
const response = makeStatusResponse({
|
||||||
|
payload: makeHardwarePayload({ cpu: { usage_percent: 0.5, temperature: [80] } as any }),
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const breaches = builder.extractThresholdBreaches(monitor, response);
|
||||||
|
|
||||||
|
expect(breaches.find((b) => b.metric === "temp")).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
it("skips temperature check when tempAlertThreshold is undefined", () => {
|
it("skips temperature check when tempAlertThreshold is undefined", () => {
|
||||||
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined });
|
const monitor = makeMonitor({ type: "hardware", tempAlertThreshold: undefined });
|
||||||
const response = makeStatusResponse({
|
const response = makeStatusResponse({
|
||||||
|
|||||||
Reference in new issue
Block a user