From c9b99eabb44fc406e86fa597a1bf54be776b8b6d Mon Sep 17 00:00:00 2001 From: fabiangigler Date: Mon, 31 Aug 2026 15:49:13 +0200 Subject: [PATCH 01/35] Localize monitor timing details --- .../design-elements/MonitorStatus.tsx | 10 ++++- .../Components/monitors/MonitorStatBoxes.tsx | 13 +++--- client/src/Utils/TimeUtils.ts | 43 +++++++++++++++++-- client/src/locales/de.json | 1 + client/src/locales/en.json | 1 + 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/client/src/Components/design-elements/MonitorStatus.tsx b/client/src/Components/design-elements/MonitorStatus.tsx index cc0f8df5b..76934f2b3 100644 --- a/client/src/Components/design-elements/MonitorStatus.tsx +++ b/client/src/Components/design-elements/MonitorStatus.tsx @@ -8,8 +8,11 @@ import { typographyLevels } from "@/Utils/Theme/Palette"; import useMediaQuery from "@mui/material/useMediaQuery"; import { LAYOUT } from "@/Utils/Theme/constants"; import { formatDuration } from "@/Utils/TimeUtils"; +import { useTranslation } from "react-i18next"; + export const MonitorStatus = ({ monitor }: { monitor: Monitor }) => { const theme = useTheme(); + const { t, i18n } = useTranslation(); const isSmall = useMediaQuery(theme.breakpoints.down("md")); if (!monitor) { @@ -52,7 +55,12 @@ export const MonitorStatus = ({ monitor }: { monitor: Monitor }) => { <> - Checking every {formatDuration(monitor?.interval, true)} + {t("pages.common.monitors.statBoxes.checkingEvery", { + interval: formatDuration(monitor.interval, { + long: true, + locale: i18n.resolvedLanguage, + }), + })} )} diff --git a/client/src/Components/monitors/MonitorStatBoxes.tsx b/client/src/Components/monitors/MonitorStatBoxes.tsx index 8b201b474..020eda78a 100644 --- a/client/src/Components/monitors/MonitorStatBoxes.tsx +++ b/client/src/Components/monitors/MonitorStatBoxes.tsx @@ -21,7 +21,7 @@ export const MonitorStatBoxes = ({ domainExpiry, }: MonitorStatBoxesProps) => { const theme = useTheme(); - const { t } = useTranslation(); + const { t, i18n } = useTranslation(); if (!monitorStats || !monitor) { return null; } @@ -29,13 +29,16 @@ export const MonitorStatBoxes = ({ const timeOfLastFailure = monitorStats?.timeOfLastFailure || 0; const timeSinceLastFailure = timeOfLastFailure > 0 ? Date.now() - timeOfLastFailure : 0; - // Determine time since last check const timeOfLastCheck = monitorStats?.lastCheckTimestamp || 0; - const timeSinceLastCheck = Date.now() - timeOfLastCheck || 0; const streakTime = formatDuration(timeSinceLastFailure); - - const lastCheckTime = formatDuration(timeSinceLastCheck); + const lastCheckTime = timeOfLastCheck + ? formatDuration(Date.now() - timeOfLastCheck, { + long: true, + relative: true, + locale: i18n.resolvedLanguage, + }) + : t("common.labels.na"); const isActive = monitor?.status === "up" || monitor?.status === "paused" || diff --git a/client/src/Utils/TimeUtils.ts b/client/src/Utils/TimeUtils.ts index 9b809e1da..d82649c81 100644 --- a/client/src/Utils/TimeUtils.ts +++ b/client/src/Utils/TimeUtils.ts @@ -104,9 +104,46 @@ export const formatMs = (ms: number, hasSpace: boolean = true): string => { return formatted; }; -export const formatDuration = (ms: number, verbose: boolean = false, hasSpace = true) => { - if (verbose) { - return prettyMilliseconds(ms, { verbose: true }); +type DurationUnit = "day" | "hour" | "minute" | "second"; + +interface FormatDurationOptions { + long?: boolean; + relative?: boolean; + hasSpace?: boolean; + locale?: string; +} + +const DURATION_UNITS: Array<{ unit: DurationUnit; milliseconds: number }> = [ + { unit: "day", milliseconds: MS_PER_DAY }, + { unit: "hour", milliseconds: MS_PER_HOUR }, + { unit: "minute", milliseconds: MS_PER_MINUTE }, + { unit: "second", milliseconds: MS_PER_SECOND }, +]; + +const getDurationUnit = (ms: number) => + DURATION_UNITS.find(({ milliseconds }) => Math.abs(ms) >= milliseconds) ?? + DURATION_UNITS[DURATION_UNITS.length - 1]; + +export const formatDuration = ( + ms: number, + { long = false, relative = false, hasSpace = true, locale }: FormatDurationOptions = {} +): string => { + if (long || relative) { + const { unit, milliseconds } = getDurationUnit(ms); + const value = Math.sign(ms) * Math.floor(Math.abs(ms) / milliseconds); + + if (relative) { + return new Intl.RelativeTimeFormat(locale, { + numeric: "always", + style: long ? "long" : "narrow", + }).format(-value, unit); + } + + return new Intl.NumberFormat(locale, { + style: "unit", + unit, + unitDisplay: "long", + }).format(value); } const formatted = prettyMilliseconds(ms, { diff --git a/client/src/locales/de.json b/client/src/locales/de.json index 6a39ee1aa..23db1be49 100644 --- a/client/src/locales/de.json +++ b/client/src/locales/de.json @@ -448,6 +448,7 @@ }, "statBoxes": { "activeFor": "Aktiv seit", + "checkingEvery": "Check alle {{interval}}", "certificateExpiry": "Zertifikatablauf", "lastCheck": "Letzter Check", "lastResponseTime": "Letzte Antwortzeit", diff --git a/client/src/locales/en.json b/client/src/locales/en.json index cbebadded..03b69a3bf 100644 --- a/client/src/locales/en.json +++ b/client/src/locales/en.json @@ -545,6 +545,7 @@ }, "statBoxes": { "activeFor": "Active for", + "checkingEvery": "Checking every {{interval}}", "certificateExpiry": "Certificate expiry", "domainExpiry": "Domain expiry", "lastCheck": "Last check", From ede5bc4ecdc362fafe34484757d2db70e2f670ae Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 11:15:20 -0700 Subject: [PATCH 02/35] add docker TLS fields to type, model, access methods to repo --- server/src/domain/monitors/monitor.model.ts | 14 ++++++++++++++ .../monitors/monitor.repository.interface.ts | 5 +++++ .../domain/monitors/monitor.repository.mongo.ts | 14 ++++++++++++++ server/src/domain/monitors/monitor.type.ts | 5 +++++ 4 files changed, 38 insertions(+) diff --git a/server/src/domain/monitors/monitor.model.ts b/server/src/domain/monitors/monitor.model.ts index 2517f5d8e..c04107651 100644 --- a/server/src/domain/monitors/monitor.model.ts +++ b/server/src/domain/monitors/monitor.model.ts @@ -307,6 +307,20 @@ const MonitorSchema = new Schema( type: Boolean, default: false, }, + dockerTlsCa: { + type: String, + }, + dockerTlsCert: { + type: String, + }, + dockerTlsKey: { + type: String, + select: false, + }, + dockerTlsKeySet: { + type: Boolean, + default: false, + }, dnsServer: { type: String, }, diff --git a/server/src/domain/monitors/monitor.repository.interface.ts b/server/src/domain/monitors/monitor.repository.interface.ts index 1b12fef71..5de43aa2c 100644 --- a/server/src/domain/monitors/monitor.repository.interface.ts +++ b/server/src/domain/monitors/monitor.repository.interface.ts @@ -55,6 +55,11 @@ export interface IMonitorsRepository { findMonitorCountByTeamIdAndType(teamId: string, config: TeamQueryConfig): Promise; findMonitorCountByProxyId(proxyId: string): Promise; + // Docker TLS + findDockerTlsKeyById(monitorId: string): Promise; + findAllDockerTlsKeys(): Promise<{ id: string; dockerTlsKey: string }[]>; + updateDockerTlsKey(monitorId: string, dockerTlsKey: string): Promise; // Needed for worker to rotate keys, no teamId/userId + // other findMonitorsSummaryByTeamId(teamId: string, config?: SummaryConfig): Promise; removeNotificationFromMonitors(notificationId: string): Promise; diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index db6b79853..efbcf6b10 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -298,6 +298,20 @@ class MongoMonitorsRepository implements IMonitorsRepository { return { monitors: this.mapDocuments(monitors), deletedCount }; }; + findDockerTlsKeyById = async (monitorId: string): Promise => { + const doc = await MonitorModel.findOne({ _id: monitorId }, { dockerTlsKey: 1 }).lean(); + return doc?.dockerTlsKey ?? null; + }; + + findAllDockerTlsKeys = async (): Promise<{ id: string; dockerTlsKey: string }[]> => { + const docs = await MonitorModel.find({ dockerTlsKeySet: true }, { dockerTlsKey: 1 }).lean(); + return docs.flatMap((doc) => (doc.dockerTlsKey ? [{ id: doc._id.toString(), dockerTlsKey: doc.dockerTlsKey }] : [])); + }; + + updateDockerTlsKey = async (monitorId: string, dockerTlsKey: string): Promise => { + await MonitorModel.updateOne({ _id: monitorId }, { $set: { dockerTlsKey } }); + }; + findMonitorsSummaryByTeamId = async (teamId: string, config: SummaryConfig): Promise => { const match = this.queryBuilder(config, teamId); const pipeline = [ diff --git a/server/src/domain/monitors/monitor.type.ts b/server/src/domain/monitors/monitor.type.ts index 12fcd1036..df8c52d9e 100644 --- a/server/src/domain/monitors/monitor.type.ts +++ b/server/src/domain/monitors/monitor.type.ts @@ -109,6 +109,11 @@ export interface Monitor { geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; dockerLogsEnabled?: boolean; + dockerTlsCa?: string; + dockerTlsCert?: string; + // EncryptionService ciphertext. Normally excluded + dockerTlsKey?: string; + dockerTlsKeySet?: boolean; dnsServer?: string; dnsRecordType?: DnsRecordType; recentChecks: CheckSnapshot[]; From c7626c669ac4d7ab8075a8a6ee7ae46c26be66e9 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 11:29:41 -0700 Subject: [PATCH 03/35] Add docker tls key to check context --- server/src/types/network.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/types/network.ts b/server/src/types/network.ts index bbbd41403..1f8afe267 100644 --- a/server/src/types/network.ts +++ b/server/src/types/network.ts @@ -162,4 +162,5 @@ export type MonitorStatusResponseOverrides = Partial Date: Thu, 10 Sep 2026 11:30:26 -0700 Subject: [PATCH 04/35] add dockerTlsKey to context for docker type monitors --- server/src/worker/worker.check-producer.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/worker/worker.check-producer.ts b/server/src/worker/worker.check-producer.ts index 404978aef..603eec8b6 100644 --- a/server/src/worker/worker.check-producer.ts +++ b/server/src/worker/worker.check-producer.ts @@ -36,6 +36,12 @@ export class CheckProducer implements ICheckProducer { return windows.some((w) => isWindowActive(w, now)); } + private async resolveDockerTlsKey(monitor: Monitor): Promise { + if (monitor.type !== "docker" || !monitor.dockerTlsKeySet || !monitor.id) return undefined; + const dockerTlsKey = await this.monitorsRepository.findDockerTlsKeyById(monitor.id); + return dockerTlsKey ?? undefined; + } + produce = async (monitor: Monitor) => { if (!monitor.id) { throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" }); @@ -61,7 +67,9 @@ export class CheckProducer implements ICheckProducer { // Step 1b: Acquire status const proxyUrl = await this.proxyResolver.resolve(monitor); - const status = await this.networkService.requestStatus(monitor, { proxyUrl }); + const dockerTlsKey = await this.resolveDockerTlsKey(monitor); + + const status = await this.networkService.requestStatus(monitor, { proxyUrl, dockerTlsKey }); if (!status) { throw new Error("No network response"); } From 9c788561a7706910964ef451d125b77903695666 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 11:49:53 -0700 Subject: [PATCH 05/35] update docker provider to add a TLS path --- server/src/service/network/DockerProvider.ts | 95 ++++++++++++++++---- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index 776da3f92..e365534c9 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -1,5 +1,7 @@ +import https from "node:https"; + import { IStatusProvider } from "@/service/network/IStatusProvider.js"; -import { DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js"; +import { CheckContext, DockerStatusPayload, MonitorStatusResponse } from "@/types/network.js"; import { DOCKER_LOG_TAIL_LINES, @@ -23,13 +25,20 @@ import { AppError } from "@/utils/AppError.js"; import Dockerode from "dockerode"; import { timeRequest } from "@/service/network/utils.js"; import { NETWORK_ERROR } from "@/types/network.js"; +import { IEncryptionService } from "@/service/encryptionService.js"; + type DockerodeType = typeof Dockerode; +type DockerOptions = Dockerode.DockerOptions & { agent?: https.Agent }; +type TlsCredentials = { ok: true; key: string } | { ok: false; message: string }; const SERVICE_NAME = "DockerProvider"; const STATS_CONCURRENCY = 5; const DOCKER_LOG_MAX_LINE_BYTES = 4096; const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]"; const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /; +const DOCKER_TLS_DEFAULT_PORT = 2376; +const DOCKER_TLS_URL = /^(tcp|https):\/\/([^\/\s:]+)(?::(\d+))?\/?$/; +const PEM_CERT_BLOCK = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g; export interface DockerError extends Error { statusCode?: number; @@ -42,20 +51,46 @@ export class DockerProvider implements IStatusProvider { constructor( private logger: ILogger, - private DockerLib: DockerodeType + private DockerLib: DockerodeType, + private encryptionService: IEncryptionService ) {} supports(type: MonitorType): boolean { return type === "docker"; } + private resolveTlsCredentials = (monitor: Monitor, ctx?: CheckContext): TlsCredentials | undefined => { + const url = monitor.url?.trim() ?? ""; + if (!DOCKER_TLS_URL.test(url)) return undefined; + if (!ctx?.dockerTlsKey) return { ok: false, message: "Docker TLS key is missing" }; + try { + return { ok: true, key: this.encryptionService.decrypt(ctx.dockerTlsKey) }; + } catch (error) { + if (error instanceof AppError) { + const keyId = typeof error.details?.keyId === "string" ? ` (key id ${error.details.keyId})` : ""; + return { ok: false, message: `${error.message}${keyId}` }; + } + return { ok: false, message: "Docker TLS client key could not be decrypted" }; + } + }; + private isDockerError(error: unknown): error is DockerError { return error instanceof Error && ("statusCode" in error || "reason" in error || "json" in error); } - private toDockerOptions = (monitor: Monitor): Dockerode.DockerOptions => { + private splitCaBundle = (pem: string | undefined): string[] | undefined => { + if (!pem) return undefined; + const blocks = pem.match(PEM_CERT_BLOCK); + return blocks && blocks.length > 0 ? blocks : undefined; + }; + + private invalidUrl = (message: string) => new AppError({ message, status: 422, service: SERVICE_NAME, method: "toDockerOptions" }); + + private toDockerOptions = (monitor: Monitor, credentials?: TlsCredentials): DockerOptions => { const url = monitor.url?.trim(); - if (!url) throw new AppError({ message: "Docker host URL is required", status: 422, service: SERVICE_NAME, method: "toDockerOptions" }); + if (!url) throw this.invalidUrl("Docker host URL is required"); + + // Unix socket if (url.startsWith("unix://")) { const socketPath = url.slice("unix://".length); if (!socketPath.startsWith("/")) @@ -63,7 +98,28 @@ export class DockerProvider implements IStatusProvider { return { socketPath }; } if (url.startsWith("/")) return { socketPath: url }; - throw new AppError({ message: `Invalid Docker host URL: ${url}`, status: 422, service: SERVICE_NAME, method: "toDockerOptions" }); + + // Docker TLS + const match = DOCKER_TLS_URL.exec(url); + if (!match) throw this.invalidUrl(`Invalid Docker host URL: ${url}`); + if (!credentials?.ok) throw this.invalidUrl(credentials?.message ?? "Docker TLS key is missing"); + const [, , host, port] = match; + const ca = monitor.ignoreTlsErrors ? undefined : this.splitCaBundle(monitor.dockerTlsCa); + const cert = monitor.dockerTlsCert; + const key = credentials.key; + return { + host, + port: port ? Number(port) : DOCKER_TLS_DEFAULT_PORT, + protocol: "https", + ca, + cert, + key, + agent: new https.Agent({ ca, cert, key, rejectUnauthorized: !monitor.ignoreTlsErrors }), + socketPath: undefined, + username: undefined, + sshOptions: undefined, + timeout: undefined, + }; }; private async mapWithConcurrency(items: T[], limit: number, fn: (item: T) => Promise): Promise { @@ -260,9 +316,23 @@ export class DockerProvider implements IStatusProvider { return rawLogs.filter((log) => log !== null); }; - handle = async (monitor: Monitor): Promise> => { + private failed = (monitor: Monitor, code: number, message: string, responseTime: number): MonitorStatusResponse => ({ + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: false, + code, + message, + responseTime, + payload: null, + }); + + handle = async (monitor: Monitor, ctx?: CheckContext): Promise> => { try { - const docker = new this.DockerLib(this.toDockerOptions(monitor)); + const credentials = this.resolveTlsCredentials(monitor, ctx); + if (credentials && !credentials.ok) return this.failed(monitor, NETWORK_ERROR, credentials.message, 0); + + const docker = new this.DockerLib(this.toDockerOptions(monitor, credentials)); // Host reachability is the monitor's status; ping latency is the responseTime const { responseTime, error } = await timeRequest(() => docker.ping()); if (error) { @@ -274,16 +344,7 @@ export class DockerProvider implements IStatusProvider { } else if (error instanceof Error) { message = error.message; } - return { - monitorId: monitor.id, - teamId: monitor.teamId, - type: monitor.type, - status: false, - code, - message, - responseTime, - payload: null, - }; + return this.failed(monitor, code, message, responseTime); } const summaries = await docker.listContainers({ all: true }); From 1035e397e05803eb20bb2ac6c46c48a6db2cf46f Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 11:57:43 -0700 Subject: [PATCH 06/35] inject encryptionService into DockerProvider --- server/src/config/services.worker.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index edea2b045..0598a5466 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -59,6 +59,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig jobsRepository, queueWorkersRepository, monitorsRepository, + encryptionService, checksRepository, geoChecksRepository, dockerLogsRepository, @@ -76,7 +77,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath)); const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger); const hardwareProvider = new HardwareProvider(httpProvider); - const dockerProvider = new DockerProvider(logger, Docker); + const dockerProvider = new DockerProvider(logger, Docker, encryptionService); const portProvider = new PortProvider(net); const gameProvider = new GameProvider(logger, GameDig); const grpcProvider = new GrpcProvider(grpc, protoLoader); From f9bc02ed1c4899406e70e0e7703220ed1e8135df Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 11:58:12 -0700 Subject: [PATCH 07/35] update msgs --- server/src/service/network/DockerProvider.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index e365534c9..08674733b 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -37,7 +37,7 @@ const DOCKER_LOG_MAX_LINE_BYTES = 4096; const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]"; const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /; const DOCKER_TLS_DEFAULT_PORT = 2376; -const DOCKER_TLS_URL = /^(tcp|https):\/\/([^\/\s:]+)(?::(\d+))?\/?$/; +const DOCKER_TLS_URL = /^(tcp|https):\/\/([^/\s:]+)(?::(\d+))?\/?$/; const PEM_CERT_BLOCK = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g; export interface DockerError extends Error { @@ -70,7 +70,7 @@ export class DockerProvider implements IStatusProvider { const keyId = typeof error.details?.keyId === "string" ? ` (key id ${error.details.keyId})` : ""; return { ok: false, message: `${error.message}${keyId}` }; } - return { ok: false, message: "Docker TLS client key could not be decrypted" }; + return { ok: false, message: "Docker TLS key could not be decrypted" }; } }; @@ -93,8 +93,7 @@ export class DockerProvider implements IStatusProvider { // Unix socket if (url.startsWith("unix://")) { const socketPath = url.slice("unix://".length); - if (!socketPath.startsWith("/")) - throw new AppError({ message: `Invalid Docker host URL: ${url}`, status: 422, service: SERVICE_NAME, method: "toDockerOptions" }); + if (!socketPath.startsWith("/")) throw this.invalidUrl(`Invalid Docker host URL: ${url}`); return { socketPath }; } if (url.startsWith("/")) return { socketPath: url }; From 131fd1a6b00c01e02004dab0739c9145ffab1f44 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 11:58:26 -0700 Subject: [PATCH 08/35] update tests --- .../providers/network/dockerProvider.test.ts | 169 +++++++++++++++++- 1 file changed, 164 insertions(+), 5 deletions(-) diff --git a/server/test/unit/providers/network/dockerProvider.test.ts b/server/test/unit/providers/network/dockerProvider.test.ts index 4c27985ef..4562db8de 100644 --- a/server/test/unit/providers/network/dockerProvider.test.ts +++ b/server/test/unit/providers/network/dockerProvider.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it, jest } from "@jest/globals"; +import https from "node:https"; import { DockerProvider } from "../../../../src/service/network/DockerProvider.ts"; +import type { IEncryptionService } from "../../../../src/service/encryptionService.ts"; import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts"; import { createMockLogger } from "../../../helpers/createMockLogger.ts"; import { NETWORK_ERROR } from "../../../../src/types/network.ts"; @@ -9,6 +11,25 @@ import { DOCKER_LOG_TAIL_LINES } from "../../../../src/domain/docker/docker.type // ── Fixtures ───────────────────────────────────────────────────────────────── +const CA_PEM = "-----BEGIN CERTIFICATE-----\nca-one\n-----END CERTIFICATE-----"; +const CA_BUNDLE_PEM = `${CA_PEM}\n-----BEGIN CERTIFICATE-----\nca-two\n-----END CERTIFICATE-----\n`; +const CERT_PEM = "-----BEGIN CERTIFICATE-----\nclient\n-----END CERTIFICATE-----"; +const KEY_PEM = "-----BEGIN PRIVATE KEY-----\nstub\n-----END PRIVATE KEY-----"; +const KEY_CIPHERTEXT = "v1.abc123.iv.tag.data"; + +const makeEncryptionService = (overrides?: Partial): IEncryptionService => ({ + isConfigured: jest.fn(() => true), + currentKeyId: jest.fn(() => "abc123"), + keyIdOf: jest.fn(() => "abc123"), + needsReencryption: jest.fn(() => false), + encrypt: jest.fn(() => KEY_CIPHERTEXT), + decrypt: jest.fn(() => KEY_PEM), + ...overrides, +}); + +const makeTlsMonitor = (overrides?: Partial): Monitor => + makeMonitor({ url: "tcp://host", dockerTlsCa: CA_PEM, dockerTlsCert: CERT_PEM, dockerTlsKeySet: true, ...overrides }); + const makeMonitor = (overrides?: Partial): Monitor => ({ id: "mon-1", @@ -60,7 +81,17 @@ const makeLogFrame = (streamType: number, text: string): Buffer => { return Buffer.concat([header, payload]); }; -const setup = (overrides: Partial<{ ping: any; listContainers: any; getContainer: any; inspect: any; stats: any; logs: any }> = {}) => { +const setup = ( + overrides: Partial<{ + ping: any; + listContainers: any; + getContainer: any; + inspect: any; + stats: any; + logs: any; + encryptionService: IEncryptionService; + }> = {} +) => { const inspect = overrides.inspect ?? jest.fn().mockResolvedValue(makeInspect()); const stats = overrides.stats ?? jest.fn().mockResolvedValue(makeStats()); const logs = overrides.logs ?? jest.fn().mockResolvedValue(Buffer.alloc(0)); @@ -72,8 +103,20 @@ const setup = (overrides: Partial<{ ping: any; listContainers: any; getContainer }; const DockerLib = jest.fn().mockReturnValue(instance) as any; const logger = createMockLogger(); - const provider = new DockerProvider(logger as any, DockerLib); - return { provider, logger, DockerLib, ping: instance.ping, listContainers: instance.listContainers, getContainer, inspect, stats, logs }; + const encryptionService = overrides.encryptionService ?? makeEncryptionService(); + const provider = new DockerProvider(logger as any, DockerLib, encryptionService); + return { + provider, + logger, + DockerLib, + encryptionService, + ping: instance.ping, + listContainers: instance.listContainers, + getContainer, + inspect, + stats, + logs, + }; }; // ── Contract ───────────────────────────────────────────────────────────────── @@ -116,6 +159,121 @@ describe("DockerProvider", () => { }); }); + // ── TLS host urls ──────────────────────────────────────────────────── + + describe("tls host urls", () => { + const ctx = { dockerTlsKey: KEY_CIPHERTEXT }; + const optionsPassedTo = (DockerLib: jest.Mock) => DockerLib.mock.calls[0]?.[0] as Record; + const agentOf = (options: Record) => options.agent as https.Agent & { options: Record }; + + it("parses tcp:// into an https connection with the monitor's ca and cert and the decrypted key", async () => { + const { provider, DockerLib, encryptionService } = setup(); + + await provider.handle(makeTlsMonitor(), ctx); + + expect(encryptionService.decrypt).toHaveBeenCalledWith(KEY_CIPHERTEXT); + const options = optionsPassedTo(DockerLib); + expect(options).toMatchObject({ host: "host", port: 2376, protocol: "https", ca: [CA_PEM], cert: CERT_PEM, key: KEY_PEM }); + expect(agentOf(options)).toBeInstanceOf(https.Agent); + expect(agentOf(options).options).toMatchObject({ ca: [CA_PEM], cert: CERT_PEM, key: KEY_PEM, rejectUnauthorized: true }); + }); + + it("parses https:// with an explicit port", async () => { + const { provider, DockerLib } = setup(); + + await provider.handle(makeTlsMonitor({ url: "https://host:2377" }), ctx); + + expect(optionsPassedTo(DockerLib)).toMatchObject({ host: "host", port: 2377, protocol: "https" }); + }); + + it("accepts a trailing slash", async () => { + const { provider, DockerLib } = setup(); + + await provider.handle(makeTlsMonitor({ url: "tcp://host:2376/" }), ctx); + + expect(optionsPassedTo(DockerLib)).toMatchObject({ host: "host", port: 2376 }); + }); + + it("splits a ca bundle into one entry per certificate on the options and the agent", async () => { + const { provider, DockerLib } = setup(); + + await provider.handle(makeTlsMonitor({ dockerTlsCa: CA_BUNDLE_PEM }), ctx); + + const options = optionsPassedTo(DockerLib); + expect(options.ca).toHaveLength(2); + expect(agentOf(options).options.ca).toEqual(options.ca); + }); + + it("omits the ca and stops rejecting unauthorized certificates when ignoreTlsErrors is set", async () => { + const { provider, DockerLib } = setup(); + + await provider.handle(makeTlsMonitor({ ignoreTlsErrors: true }), ctx); + + const options = optionsPassedTo(DockerLib); + expect(options.ca).toBeUndefined(); + expect(agentOf(options).options).toMatchObject({ ca: undefined, rejectUnauthorized: false }); + }); + + it("passes explicit undefined for every field docker-modem would otherwise fill from the environment", async () => { + const { provider, DockerLib } = setup(); + + await provider.handle(makeTlsMonitor(), ctx); + + const options = optionsPassedTo(DockerLib); + for (const field of ["socketPath", "username", "sshOptions", "timeout"]) { + expect(options).toHaveProperty(field, undefined); + } + }); + + it("returns a down check and never constructs a client when the key is missing from the context", async () => { + const { provider, DockerLib, encryptionService } = setup(); + + const result = await provider.handle(makeTlsMonitor()); + + expect(result).toMatchObject({ status: false, code: NETWORK_ERROR, message: "Docker TLS key is missing", payload: null }); + expect(encryptionService.decrypt).not.toHaveBeenCalled(); + expect(DockerLib).not.toHaveBeenCalled(); + }); + + it("returns a down check naming the key id when the stored key cannot be decrypted", async () => { + const decrypt = jest.fn(() => { + throw new AppError({ + message: "No encryption key matches ciphertext", + status: 500, + service: "EncryptionService", + method: "decrypt", + details: { keyId: "abc123" }, + }); + }); + const { provider, DockerLib } = setup({ encryptionService: makeEncryptionService({ decrypt }) }); + + const result = await provider.handle(makeTlsMonitor(), ctx); + + expect(result).toMatchObject({ status: false, code: NETWORK_ERROR, message: "No encryption key matches ciphertext (key id abc123)" }); + expect(DockerLib).not.toHaveBeenCalled(); + }); + + it("returns a generic down check when decryption fails with a non-AppError", async () => { + const decrypt = jest.fn(() => { + throw new Error("boom"); + }); + const { provider, DockerLib } = setup({ encryptionService: makeEncryptionService({ decrypt }) }); + + const result = await provider.handle(makeTlsMonitor(), ctx); + + expect(result).toMatchObject({ status: false, message: "Docker TLS key could not be decrypted" }); + expect(DockerLib).not.toHaveBeenCalled(); + }); + + it("does not touch the encryption service for socket monitors", async () => { + const { provider, encryptionService } = setup(); + + await provider.handle(makeMonitor(), ctx); + + expect(encryptionService.decrypt).not.toHaveBeenCalled(); + }); + }); + // ── Fail-loudly url validation ─────────────────────────────────────── describe("invalid host urls", () => { @@ -125,8 +283,9 @@ describe("DockerProvider", () => { ["unix:// with a relative path", "unix://run/docker.sock", "Invalid Docker host URL"], ["garbage", "not-a-url", "Invalid Docker host URL"], ["a container name (old semantics)", "my-container", "Invalid Docker host URL"], - ["tcp engine urls (unsupported)", "tcp://host:2375", "Invalid Docker host URL"], - ["https engine urls (unsupported)", "https://host", "Invalid Docker host URL"], + ["http engine urls (plaintext tcp is unsupported)", "http://host", "Invalid Docker host URL"], + ["a bare host and port", "host:2376", "Invalid Docker host URL"], + ["a tcp url with a path", "tcp://host/engine", "Invalid Docker host URL"], ["ssh engine urls (unsupported)", "ssh://deploy@host", "Invalid Docker host URL"], ]; From 5eb16fdbdbdbee02a163260307b8596a2f06e177 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 12:23:20 -0700 Subject: [PATCH 09/35] add validation, helpers, tests --- .../src/api/validation/monitorValidation.ts | 17 +++- server/src/service/network/DockerProvider.ts | 12 +-- server/src/utils/dockerHost.ts | 5 + server/src/utils/pem.ts | 28 ++++++ server/test/fixtures/docker-tls/README.md | 10 ++ server/test/fixtures/docker-tls/ca.pem | 11 +++ .../test/fixtures/docker-tls/client-cert.pem | 11 +++ .../test/fixtures/docker-tls/client-key.pem | 5 + .../fixtures/docker-tls/encrypted-key.pem | 8 ++ server/test/fixtures/docker-tls/other-key.pem | 5 + server/test/unit/utils/dockerHost.test.ts | 42 +++++++++ server/test/unit/utils/pem.test.ts | 91 +++++++++++++++++++ .../unit/validation/monitorValidation.test.ts | 17 +++- 13 files changed, 249 insertions(+), 13 deletions(-) create mode 100644 server/src/utils/dockerHost.ts create mode 100644 server/src/utils/pem.ts create mode 100644 server/test/fixtures/docker-tls/README.md create mode 100644 server/test/fixtures/docker-tls/ca.pem create mode 100644 server/test/fixtures/docker-tls/client-cert.pem create mode 100644 server/test/fixtures/docker-tls/client-key.pem create mode 100644 server/test/fixtures/docker-tls/encrypted-key.pem create mode 100644 server/test/fixtures/docker-tls/other-key.pem create mode 100644 server/test/unit/utils/dockerHost.test.ts create mode 100644 server/test/unit/utils/pem.test.ts diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index ddea832ef..3d714c7ba 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -15,6 +15,7 @@ import { import { DateRanges, SortOrders } from "@/types/query.js"; import { DockerContainerStates, DockerHealthStatuses, DockerLogStreams, DockerPortProtocols } from "@/domain/docker/docker.type.js"; import { DOCKER_LOG_PAGE_DEFAULT, DOCKER_LOG_PAGE_MAX } from "@/domain/docker/docker-log.type.js"; +import { isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js"; const httpStatusCode = z.number().refine((code) => HttpStatusCodeSet.has(code), { message: "Must be a valid HTTP status code" }); @@ -120,15 +121,13 @@ const refineProxySelection = (body: { proxyMode?: string; proxyId?: string }, ct } }; -const dockerUrlRegex = /^(?:unix:\/\/\/\S+|\/\S+)$/; - const refineDockerUrl = (data: { type?: string; url?: string }, ctx: z.RefinementCtx) => { if (data.type !== "docker" || data.url === undefined) return; - if (!dockerUrlRegex.test(data.url)) { + if (!isDockerSocketUrl(data.url) && !isDockerTlsUrl(data.url)) { ctx.addIssue({ code: "custom", path: ["url"], - message: "Docker host must be unix:///path or an absolute socket path", + message: "Docker host must be unix:///path, an absolute socket path, or tcp://host[:port]", }); } }; @@ -170,6 +169,9 @@ export const createMonitorBodyValidation = z geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), dockerLogsEnabled: z.boolean().optional(), + dockerTlsCa: z.union([z.string(), z.literal("")]).optional(), + dockerTlsCert: z.union([z.string(), z.literal("")]).optional(), + dockerTlsKey: z.union([z.string(), z.literal("")]).optional(), dnsServer: dnsServerValidation.optional(), dnsRecordType: z.enum(DnsRecordTypes).optional(), }) @@ -215,6 +217,9 @@ export const editMonitorBodyValidation = z geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), dockerLogsEnabled: z.boolean().optional(), + dockerTlsCa: z.union([z.string(), z.literal("")]).optional(), + dockerTlsCert: z.union([z.string(), z.literal("")]).optional(), + dockerTlsKey: z.union([z.string(), z.literal("")]).optional(), dnsServer: dnsServerValidation.optional(), dnsRecordType: z.enum(DnsRecordTypes).optional(), }) @@ -291,7 +296,6 @@ const importedMonitorSchema = z geoCheckLocations: z.array(z.enum(GeoContinents)).default([]), geoCheckInterval: z.number().min(300000).default(300000), dockerLogsEnabled: z.boolean().default(false), - dnsServer: dnsServerValidation.optional(), dnsRecordType: z.enum(DnsRecordTypes).optional(), createdAt: z.string().optional(), @@ -381,6 +385,9 @@ export const monitorResponseSchema = z geoCheckLocations: z.array(z.enum(GeoContinents)), geoCheckInterval: z.number(), dockerLogsEnabled: z.boolean(), + dockerTlsCa: z.string().optional(), + dockerTlsCert: z.string().optional(), + dockerTlsKeySet: z.boolean().optional(), dnsServer: z.string().optional(), dnsRecordType: z.enum(DnsRecordTypes).optional(), teamId: z.string(), diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index 08674733b..c5eeda860 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -26,6 +26,8 @@ import Dockerode from "dockerode"; import { timeRequest } from "@/service/network/utils.js"; import { NETWORK_ERROR } from "@/types/network.js"; import { IEncryptionService } from "@/service/encryptionService.js"; +import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js"; +import { splitCertificateBundle } from "@/utils/pem.js"; type DockerodeType = typeof Dockerode; type DockerOptions = Dockerode.DockerOptions & { agent?: https.Agent }; @@ -37,8 +39,6 @@ const DOCKER_LOG_MAX_LINE_BYTES = 4096; const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]"; const DOCKER_LOG_TS_REGEX = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z /; const DOCKER_TLS_DEFAULT_PORT = 2376; -const DOCKER_TLS_URL = /^(tcp|https):\/\/([^/\s:]+)(?::(\d+))?\/?$/; -const PEM_CERT_BLOCK = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g; export interface DockerError extends Error { statusCode?: number; @@ -60,8 +60,7 @@ export class DockerProvider implements IStatusProvider { } private resolveTlsCredentials = (monitor: Monitor, ctx?: CheckContext): TlsCredentials | undefined => { - const url = monitor.url?.trim() ?? ""; - if (!DOCKER_TLS_URL.test(url)) return undefined; + if (!isDockerTlsUrl(monitor.url)) return undefined; if (!ctx?.dockerTlsKey) return { ok: false, message: "Docker TLS key is missing" }; try { return { ok: true, key: this.encryptionService.decrypt(ctx.dockerTlsKey) }; @@ -79,9 +78,8 @@ export class DockerProvider implements IStatusProvider { } private splitCaBundle = (pem: string | undefined): string[] | undefined => { - if (!pem) return undefined; - const blocks = pem.match(PEM_CERT_BLOCK); - return blocks && blocks.length > 0 ? blocks : undefined; + const blocks = pem ? splitCertificateBundle(pem) : []; + return blocks.length > 0 ? blocks : undefined; }; private invalidUrl = (message: string) => new AppError({ message, status: 422, service: SERVICE_NAME, method: "toDockerOptions" }); diff --git a/server/src/utils/dockerHost.ts b/server/src/utils/dockerHost.ts new file mode 100644 index 000000000..ef95356ee --- /dev/null +++ b/server/src/utils/dockerHost.ts @@ -0,0 +1,5 @@ +export const DOCKER_TLS_URL = /^(tcp|https):\/\/([^/\s:]+)(?::(\d+))?\/?$/; +export const DOCKER_SOCKET_URL = /^(?:unix:\/\/\/\S+|\/\S+)$/; + +export const isDockerTlsUrl = (url: string | undefined): boolean => DOCKER_TLS_URL.test(url?.trim() ?? ""); +export const isDockerSocketUrl = (url: string | undefined): boolean => DOCKER_SOCKET_URL.test(url?.trim() ?? ""); diff --git a/server/src/utils/pem.ts b/server/src/utils/pem.ts new file mode 100644 index 000000000..13e7be106 --- /dev/null +++ b/server/src/utils/pem.ts @@ -0,0 +1,28 @@ +import { createPrivateKey, KeyObject, X509Certificate } from "node:crypto"; + +const PEM_CERT_BLOCK = /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g; + +export const splitCertificateBundle = (pem: string): string[] => pem.match(PEM_CERT_BLOCK) ?? []; + +export const parseCertificates = (pem: string): X509Certificate[] => { + const blocks = splitCertificateBundle(pem); + if (blocks.length === 0) throw new Error("No certificate found; expected one or more PEM CERTIFICATE blocks"); + try { + return blocks.map((block) => new X509Certificate(block)); + } catch { + throw new Error("Certificate is not valid PEM"); + } +}; + +export const parsePrivateKey = (pem: string): KeyObject => { + if (/^-----BEGIN ENCRYPTED PRIVATE KEY-----/m.test(pem) || /^Proc-Type: 4,ENCRYPTED/m.test(pem)) { + throw new Error("Encrypted private keys are not supported; remove the passphrase first"); + } + try { + return createPrivateKey(pem); + } catch { + throw new Error("Private key is not valid PEM"); + } +}; + +export const keyMatchesCertificate = (key: KeyObject, certificate: X509Certificate): boolean => certificate.checkPrivateKey(key); diff --git a/server/test/fixtures/docker-tls/README.md b/server/test/fixtures/docker-tls/README.md new file mode 100644 index 000000000..c4183c6e7 --- /dev/null +++ b/server/test/fixtures/docker-tls/README.md @@ -0,0 +1,10 @@ +Test fixtures for Docker TLS validation. Self-signed, 100-year, not trusted anywhere. + +Regenerate with: + + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 36500 -subj "/CN=Checkmate Test CA" -keyout ca-key.pem -out ca.pem + openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -subj "/CN=checkmate-test-client" -keyout client-key.pem -out client.csr + openssl x509 -req -in client.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -days 36500 -out client-cert.pem + openssl ecparam -name prime256v1 -genkey -noout -out other-key.pem + openssl pkcs8 -topk8 -in client-key.pem -out encrypted-key.pem -passout pass:fixture + rm client.csr ca.srl ca-key.pem diff --git a/server/test/fixtures/docker-tls/ca.pem b/server/test/fixtures/docker-tls/ca.pem new file mode 100644 index 000000000..67f23b873 --- /dev/null +++ b/server/test/fixtures/docker-tls/ca.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBkDCCATWgAwIBAgIUL3SODzLIekF+Tc3iHhHr/7uxvdIwCgYIKoZIzj0EAwIw +HDEaMBgGA1UEAwwRQ2hlY2ttYXRlIFRlc3QgQ0EwIBcNMjYwOTEwMTkxNDUyWhgP +MjEyNjA4MTcxOTE0NTJaMBwxGjAYBgNVBAMMEUNoZWNrbWF0ZSBUZXN0IENBMFkw +EwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEXRJ0Y0jc+pKwXbuioDmbaHOqr5n3oU9x +Y16vYajhUiFeWohEqGRG8lHA9S82KPNmpjthLWUykHvrTsKosBb6IKNTMFEwHQYD +VR0OBBYEFNfGlRFC2MFZrCkoF4UxktbZUve1MB8GA1UdIwQYMBaAFNfGlRFC2MFZ +rCkoF4UxktbZUve1MA8GA1UdEwEB/wQFMAMBAf8wCgYIKoZIzj0EAwIDSQAwRgIh +APOnYaaKk5ED6NV/AkdpNXOEqlWcXwIQzh5Ycj3cjby+AiEApuAGTnk8k4dtVNmB +aVV5bRmq7f3BjhxwZorP+KiIrdI= +-----END CERTIFICATE----- diff --git a/server/test/fixtures/docker-tls/client-cert.pem b/server/test/fixtures/docker-tls/client-cert.pem new file mode 100644 index 000000000..19e6e9fee --- /dev/null +++ b/server/test/fixtures/docker-tls/client-cert.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBgjCCASigAwIBAgIUBx7r8tVOUFNX8c3YYyBxXTNltuUwCgYIKoZIzj0EAwIw +HDEaMBgGA1UEAwwRQ2hlY2ttYXRlIFRlc3QgQ0EwIBcNMjYwOTEwMTkxNDUyWhgP +MjEyNjA4MTcxOTE0NTJaMCAxHjAcBgNVBAMMFWNoZWNrbWF0ZS10ZXN0LWNsaWVu +dDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKeUUNe9kE57nK7wTfnnvmqgzVj1 +ngNPajQYIAh9SAR+pVLAoTWLinWq5YarW1LFAzxyRKl6p2NdKtyLpVYV5BKjQjBA +MB0GA1UdDgQWBBReuZBD6K0kWfgGDkt05zzM24wPejAfBgNVHSMEGDAWgBTXxpUR +QtjBWawpKBeFMZLW2VL3tTAKBggqhkjOPQQDAgNIADBFAiAvfzha4dEVcJlKjH45 +yOkbFXiS/+tU+uTvn+7uutn6QAIhAI2qCXVvxX+i3Jk1xTS7R0mzJNXZXqabjODi +cisS1VJG +-----END CERTIFICATE----- diff --git a/server/test/fixtures/docker-tls/client-key.pem b/server/test/fixtures/docker-tls/client-key.pem new file mode 100644 index 000000000..42e0f795c --- /dev/null +++ b/server/test/fixtures/docker-tls/client-key.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgIHAuDMyYVMOi5PpA +yUPjuNjvy/QOI9dnjQZ+AKhuQO+hRANCAASnlFDXvZBOe5yu8E35575qoM1Y9Z4D +T2o0GCAIfUgEfqVSwKE1i4p1quWGq1tSxQM8ckSpeqdjXSrci6VWFeQS +-----END PRIVATE KEY----- diff --git a/server/test/fixtures/docker-tls/encrypted-key.pem b/server/test/fixtures/docker-tls/encrypted-key.pem new file mode 100644 index 000000000..6be1a4a07 --- /dev/null +++ b/server/test/fixtures/docker-tls/encrypted-key.pem @@ -0,0 +1,8 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIH0MF8GCSqGSIb3DQEFDTBSMDEGCSqGSIb3DQEFDDAkBBDak7FU8VAgafGR1iEl +eFHbAgIIADAMBggqhkiG9w0CCQUAMB0GCWCGSAFlAwQBKgQQCUTF7P/8uj9a5kLU +3nBJdQSBkNpsmjC1FczojCBOn0g+2PlmMGI6MfR3vFvmPRAAF7SHLzQC1lXWhTU0 +3v5yCCdjlC5gTGINKf8RhZQjKym+lIpf+16EAPFLzFbRJPvt/eKq65YiixvQP2PS +vysOG71YZJt1kSHrCOu5EPcCLvD2Kjyt48c/LJssCB81TOdqP1jV9iH29SCv4bnx +KtjXulGg3w== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/server/test/fixtures/docker-tls/other-key.pem b/server/test/fixtures/docker-tls/other-key.pem new file mode 100644 index 000000000..e671633f7 --- /dev/null +++ b/server/test/fixtures/docker-tls/other-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIGnUNFbPMxfa3m7xo9J3Lu9qiss2ZxWpqp91B6X9rQbfoAoGCCqGSM49 +AwEHoUQDQgAEGtbNBALMRIEROLtiEdNda4r/X0Hq1hcY/0V20gW3hEHhKRKkSilH +0LyFWtlZ5uILZh6zXCGWlOPm0PFO9rN/hQ== +-----END EC PRIVATE KEY----- diff --git a/server/test/unit/utils/dockerHost.test.ts b/server/test/unit/utils/dockerHost.test.ts new file mode 100644 index 000000000..aa21c61c0 --- /dev/null +++ b/server/test/unit/utils/dockerHost.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "@jest/globals"; +import { DOCKER_TLS_URL, isDockerSocketUrl, isDockerTlsUrl } from "../../../src/utils/dockerHost.ts"; + +describe("isDockerTlsUrl", () => { + it.each(["tcp://host", "tcp://host:2376", "https://host", "https://host:2377", "tcp://host/", "tcp://10.0.0.1:2376", " tcp://host "])( + "accepts %s", + (url) => { + expect(isDockerTlsUrl(url)).toBe(true); + } + ); + + it.each([ + "http://host", + "host:2376", + "tcp://host/x", + "tcp://host:abc", + "tcp://a:b:c", + "tcp://[::1]:2376", + "tcp://ho st", + "ssh://deploy@host", + "unix:///var/run/docker.sock", + "", + undefined, + ])("rejects %s", (url) => { + expect(isDockerTlsUrl(url)).toBe(false); + }); + + it("captures scheme, host and port for the provider", () => { + expect(DOCKER_TLS_URL.exec("https://host:2377")?.slice(1)).toEqual(["https", "host", "2377"]); + expect(DOCKER_TLS_URL.exec("tcp://host")?.slice(1)).toEqual(["tcp", "host", undefined]); + }); +}); + +describe("isDockerSocketUrl", () => { + it.each(["unix:///var/run/docker.sock", "/var/run/docker.sock", " /run/docker.sock "])("accepts %s", (url) => { + expect(isDockerSocketUrl(url)).toBe(true); + }); + + it.each(["unix://", "unix://relative/path", "tcp://host", "my-container", "", undefined])("rejects %s", (url) => { + expect(isDockerSocketUrl(url)).toBe(false); + }); +}); diff --git a/server/test/unit/utils/pem.test.ts b/server/test/unit/utils/pem.test.ts new file mode 100644 index 000000000..31d9b6d49 --- /dev/null +++ b/server/test/unit/utils/pem.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "@jest/globals"; +import { readFileSync } from "node:fs"; +import { X509Certificate } from "node:crypto"; +import { keyMatchesCertificate, parseCertificates, parsePrivateKey, splitCertificateBundle } from "../../../src/utils/pem.ts"; + +const fixture = (name: string) => readFileSync(new URL(`../../fixtures/docker-tls/${name}`, import.meta.url), "utf8"); + +const CA = fixture("ca.pem"); +const CLIENT_CERT = fixture("client-cert.pem"); +const CLIENT_KEY = fixture("client-key.pem"); +const OTHER_KEY = fixture("other-key.pem"); +const ENCRYPTED_KEY = fixture("encrypted-key.pem"); + +describe("splitCertificateBundle", () => { + it("returns one block for a single certificate", () => { + expect(splitCertificateBundle(CA)).toHaveLength(1); + }); + + it("returns every block of a bundle in order", () => { + const blocks = splitCertificateBundle(`${CA}\n${CLIENT_CERT}`); + expect(blocks).toHaveLength(2); + expect(blocks[0]).toBe(CA.trim()); + expect(blocks[1]).toBe(CLIENT_CERT.trim()); + }); + + it("ignores text outside the blocks", () => { + expect(splitCertificateBundle(`subject=CN=x\n${CA}\ntrailing`)).toHaveLength(1); + }); + + it("returns an empty array when there is no certificate block", () => { + expect(splitCertificateBundle("")).toEqual([]); + expect(splitCertificateBundle(CLIENT_KEY)).toEqual([]); + expect(splitCertificateBundle("-----BEGIN CERTIFICATE-----\nunterminated")).toEqual([]); + }); +}); + +describe("parseCertificates", () => { + it("parses a single certificate", () => { + const [cert] = parseCertificates(CA); + expect(cert).toBeInstanceOf(X509Certificate); + expect(cert?.subject).toContain("CN=Checkmate Test CA"); + }); + + it("parses every certificate in a bundle", () => { + const certs = parseCertificates(`${CA}\n${CLIENT_CERT}`); + expect(certs.map((c) => c.subject)).toEqual([expect.stringContaining("Checkmate Test CA"), expect.stringContaining("checkmate-test-client")]); + }); + + it("rejects input with no certificate block", () => { + expect(() => parseCertificates("")).toThrow("No certificate found"); + expect(() => parseCertificates(CLIENT_KEY)).toThrow("No certificate found"); + }); + + it("rejects a block whose body is not a certificate", () => { + expect(() => parseCertificates("-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----")).toThrow( + "Certificate is not valid PEM" + ); + }); +}); + +describe("parsePrivateKey", () => { + it("parses an unencrypted PKCS#8 key", () => { + expect(parsePrivateKey(CLIENT_KEY).type).toBe("private"); + }); + + it("rejects an encrypted PKCS#8 key with a clear message", () => { + expect(() => parsePrivateKey(ENCRYPTED_KEY)).toThrow("Encrypted private keys are not supported"); + }); + + it("rejects a legacy encrypted key by its Proc-Type header", () => { + const legacy = "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,00\n\nAAAA\n-----END RSA PRIVATE KEY-----"; + expect(() => parsePrivateKey(legacy)).toThrow("Encrypted private keys are not supported"); + }); + + it("rejects input that is not a key", () => { + expect(() => parsePrivateKey("")).toThrow("Private key is not valid PEM"); + expect(() => parsePrivateKey(CA)).toThrow("Private key is not valid PEM"); + }); +}); + +describe("keyMatchesCertificate", () => { + it("is true for the key the certificate was issued to", () => { + const [cert] = parseCertificates(CLIENT_CERT); + expect(keyMatchesCertificate(parsePrivateKey(CLIENT_KEY), cert!)).toBe(true); + }); + + it("is false for an unrelated key", () => { + const [cert] = parseCertificates(CLIENT_CERT); + expect(keyMatchesCertificate(parsePrivateKey(OTHER_KEY), cert!)).toBe(false); + }); +}); diff --git a/server/test/unit/validation/monitorValidation.test.ts b/server/test/unit/validation/monitorValidation.test.ts index 1cd224ab5..c9b7d16f0 100644 --- a/server/test/unit/validation/monitorValidation.test.ts +++ b/server/test/unit/validation/monitorValidation.test.ts @@ -620,8 +620,23 @@ describe("monitorValidation — Docker host url", () => { expect(parsed.url).toBe("/run/docker.sock"); }); + it("accepts tcp:// and https:// engine urls with an optional port and trailing slash", () => { + for (const url of ["tcp://host", "tcp://host:2376", "https://host:2377", "tcp://host/"]) { + expect(createMonitorBodyValidation.parse({ ...baseDockerBody, url }).url).toBe(url); + } + }); + it("rejects container names and unsupported engine urls", () => { - for (const badUrl of ["my-container", "tcp://host:2375", "https://host", "ssh://deploy@host", "unix://relative/path", ""]) { + for (const badUrl of [ + "my-container", + "http://host", + "host:2376", + "tcp://host/x", + "tcp://host:abc", + "ssh://deploy@host", + "unix://relative/path", + "", + ]) { expect(() => createMonitorBodyValidation.parse({ ...baseDockerBody, url: badUrl })).toThrow(); } }); From 7071de294f1df36dcacf9579bd323dd4bd4655f3 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 12:26:12 -0700 Subject: [PATCH 10/35] add docker TLS fields to toEntity --- server/src/domain/monitors/monitor.repository.mongo.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/domain/monitors/monitor.repository.mongo.ts b/server/src/domain/monitors/monitor.repository.mongo.ts index efbcf6b10..385dcebab 100644 --- a/server/src/domain/monitors/monitor.repository.mongo.ts +++ b/server/src/domain/monitors/monitor.repository.mongo.ts @@ -468,6 +468,9 @@ class MongoMonitorsRepository implements IMonitorsRepository { geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, dockerLogsEnabled: doc.dockerLogsEnabled ?? false, + dockerTlsCa: doc.dockerTlsCa ?? undefined, + dockerTlsCert: doc.dockerTlsCert ?? undefined, + dockerTlsKeySet: doc.dockerTlsKeySet ?? false, dnsServer: doc.dnsServer ?? undefined, dnsRecordType: doc.dnsRecordType ?? undefined, createdAt: toDateString(doc.createdAt), From c64c4cc5ff552a218f063027dae218992c104092 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Thu, 10 Sep 2026 12:26:25 -0700 Subject: [PATCH 11/35] update tests --- .../integration/monitorRepository.test.ts | 66 +++++++++++++++++++ .../test/unit/services/checkProducer.test.ts | 44 ++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/server/test/integration/monitorRepository.test.ts b/server/test/integration/monitorRepository.test.ts index ed7a7a7a6..63ef4a105 100644 --- a/server/test/integration/monitorRepository.test.ts +++ b/server/test/integration/monitorRepository.test.ts @@ -123,4 +123,70 @@ describe("MongoMonitorsRepository", () => { expect(monitors[0].recentChecks).toEqual([]); }); }); + + describe("docker tls key access", () => { + const seedDockerMonitor = (overrides: Record = {}) => + MonitorModel.create({ + userId: new mongoose.Types.ObjectId(), + teamId: new mongoose.Types.ObjectId(), + name: "docker", + type: "docker", + url: "tcp://host", + ...overrides, + }); + + it("never returns the key on a normal load, even with the flag set", async () => { + const repo = new MongoMonitorsRepository(); + const doc = await seedDockerMonitor({ dockerTlsKey: "v1.abc123.iv.tag.data", dockerTlsKeySet: true }); + + const monitor = await repo.findByIdLean(doc._id.toString()); + + expect(monitor?.dockerTlsKeySet).toBe(true); + expect(monitor?.dockerTlsKey).toBeUndefined(); + }); + + it("findDockerTlsKeyById returns the stored key despite select: false", async () => { + const repo = new MongoMonitorsRepository(); + const doc = await seedDockerMonitor({ dockerTlsKey: "v1.abc123.iv.tag.data", dockerTlsKeySet: true }); + + await expect(repo.findDockerTlsKeyById(doc._id.toString())).resolves.toBe("v1.abc123.iv.tag.data"); + }); + + it("findDockerTlsKeyById returns null for a monitor without a key and for an unknown id", async () => { + const repo = new MongoMonitorsRepository(); + const doc = await seedDockerMonitor({ url: "unix:///var/run/docker.sock" }); + + await expect(repo.findDockerTlsKeyById(doc._id.toString())).resolves.toBeNull(); + await expect(repo.findDockerTlsKeyById(new mongoose.Types.ObjectId().toString())).resolves.toBeNull(); + }); + + it("findAllDockerTlsKeys returns only rows with the flag set and a key present", async () => { + const repo = new MongoMonitorsRepository(); + const keyed = await seedDockerMonitor({ dockerTlsKey: "v1.abc123.k1", dockerTlsKeySet: true }); + await seedDockerMonitor({ url: "unix:///var/run/docker.sock" }); + await seedDockerMonitor({ dockerTlsKeySet: true }); // flag without key: inconsistent, dropped + await MonitorModel.create({ + userId: new mongoose.Types.ObjectId(), + teamId: new mongoose.Types.ObjectId(), + name: "http", + type: "http", + url: "https://example.com", + }); + + await expect(repo.findAllDockerTlsKeys()).resolves.toEqual([{ id: keyed._id.toString(), dockerTlsKey: "v1.abc123.k1" }]); + }); + + it("updateDockerTlsKey replaces the key and nothing else", async () => { + const repo = new MongoMonitorsRepository(); + const doc = await seedDockerMonitor({ dockerTlsKey: "v1.old.k", dockerTlsKeySet: true, dockerTlsCert: "cert" }); + + await repo.updateDockerTlsKey(doc._id.toString(), "v1.new.k"); + + const after = await MonitorModel.findById(doc._id).select("+dockerTlsKey").lean(); + expect(after?.dockerTlsKey).toBe("v1.new.k"); + expect(after?.dockerTlsCert).toBe("cert"); + expect(after?.dockerTlsKeySet).toBe(true); + expect(after?.name).toBe("docker"); + }); + }); }); diff --git a/server/test/unit/services/checkProducer.test.ts b/server/test/unit/services/checkProducer.test.ts index 7116c5adb..c10cc2f29 100644 --- a/server/test/unit/services/checkProducer.test.ts +++ b/server/test/unit/services/checkProducer.test.ts @@ -22,7 +22,7 @@ const activeWindow = () => { const createProducer = (overrides?: Record) => { const defaults = { logger: createMockLogger(), - monitorsRepository: { updateById: jest.fn().mockResolvedValue({}) }, + monitorsRepository: { updateById: jest.fn().mockResolvedValue({}), findDockerTlsKeyById: jest.fn().mockResolvedValue(null) }, maintenanceWindowsRepository: { findByMonitorId: jest.fn().mockResolvedValue([]) }, checkService: { toCheck: jest.fn().mockReturnValue({ id: "check-1" }) }, networkService: { requestStatus: jest.fn().mockResolvedValue({ monitorId: "m1", status: true, code: 200, message: "OK" }) }, @@ -111,6 +111,48 @@ describe("CheckProducer", () => { // ── acquire / record ────────────────────────────────────────────────────── + // ── docker tls key resolution ───────────────────────────────────────────── + + it("fetches the stored key for a docker monitor with a key set and passes it in the context", async () => { + const { producer, defaults } = createProducer({ + monitorsRepository: { updateById: jest.fn(), findDockerTlsKeyById: jest.fn().mockResolvedValue("v1.abc123.iv.tag.data") }, + }); + const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true }); + + await producer.produce(monitor); + + expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1"); + expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: "v1.abc123.iv.tag.data" }); + }); + + it("does not fetch a key for a docker monitor without one set", async () => { + const { producer, defaults } = createProducer(); + const monitor = makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKeySet: false }); + + await producer.produce(monitor); + + expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled(); + expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined }); + }); + + it("does not fetch a key for non-docker monitors even if the flag is set", async () => { + const { producer, defaults } = createProducer(); + + await producer.produce(makeMonitor({ type: "http", dockerTlsKeySet: true })); + + expect(defaults.monitorsRepository.findDockerTlsKeyById).not.toHaveBeenCalled(); + }); + + it("passes undefined when the flag is set but the repository has no key", async () => { + const { producer, defaults } = createProducer(); + const monitor = makeMonitor({ type: "docker", url: "tcp://host", dockerTlsKeySet: true }); + + await producer.produce(monitor); + + expect(defaults.monitorsRepository.findDockerTlsKeyById).toHaveBeenCalledWith("m1"); + expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(monitor, { proxyUrl: undefined, dockerTlsKey: undefined }); + }); + it("throws when the network response is null", async () => { const { producer } = createProducer({ networkService: { requestStatus: jest.fn().mockResolvedValue(null) }, From b14b2142e362933ca73fbc774f8b563cfc9a9c60 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 10:09:30 -0700 Subject: [PATCH 12/35] add secrets rotation service and tests --- server/src/config/services.shared.ts | 2 +- server/src/config/services.worker.ts | 20 +++ .../{ => encryption}/encryptionService.ts | 0 .../encryption/secretsRotationService.ts | 60 +++++++ server/src/service/network/DockerProvider.ts | 2 +- .../providers/network/dockerProvider.test.ts | 2 +- .../unit/services/encryptionService.test.ts | 4 +- .../services/secretsRotationService.test.ts | 156 ++++++++++++++++++ 8 files changed, 241 insertions(+), 5 deletions(-) rename server/src/service/{ => encryption}/encryptionService.ts (100%) create mode 100644 server/src/service/encryption/secretsRotationService.ts create mode 100644 server/test/unit/services/secretsRotationService.test.ts diff --git a/server/src/config/services.shared.ts b/server/src/config/services.shared.ts index 4f8f63412..3b4fdb147 100644 --- a/server/src/config/services.shared.ts +++ b/server/src/config/services.shared.ts @@ -20,7 +20,7 @@ import { INotificationsService, NotificationsService } from "@/domain/notificati import { IEmailService, EmailService } from "@/service/emailService.js"; import { GlobalPingService } from "@/service/globalPingService.js"; import { ILogger } from "@/utils/logger.js"; -import { IEncryptionService, EncryptionService } from "@/service/encryptionService.js"; +import { IEncryptionService, EncryptionService } from "@/service/encryption/encryptionService.js"; // Notification providers import type { NotificationProviderRegistry } from "@/domain/notifications/notification.service.js"; diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index 0598a5466..8b2f4de07 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -16,6 +16,7 @@ import { SharedServices } from "@/config/services.shared.js"; import { INetworkService, NetworkService } from "@/service/networkService.js"; import { IBufferService, BufferService } from "@/service/bufferService.js"; import { IStatusService, StatusService } from "@/service/statusService.js"; +import { ISecretsRotationService, SecretsRotationService } from "@/service/encryption/secretsRotationService.js"; import { IQueueWorker } from "@/worker/worker.interface.js"; import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; import { WorkerHelper } from "@/worker/worker.helper.js"; @@ -39,6 +40,7 @@ import { GameProvider } from "@/service/network/GameProvider.js"; import { GrpcProvider } from "@/service/network/GrpcProvider.js"; import { WebSocketProvider } from "@/service/network/WebSocketProvider.js"; import { DNSProvider } from "@/service/network/DNSProvider.js"; +import { AppError } from "@/utils/AppError.js"; export interface WorkerServices { worker: IQueueWorker; networkService: INetworkService; @@ -165,5 +167,23 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig workerId, }); + const secretsRotationService = new SecretsRotationService(monitorsRepository, encryptionService, logger); + try { + const result = await secretsRotationService.run(); + logger.debug({ + message: "Secrets rotation run successfully", + service: "WorkerServices", + method: "buildWorker", + details: { ...result }, + }); + } catch (error: unknown) { + logger.warn({ + message: `Could not rotate Docker TLS secrets: ${error instanceof Error ? error.message : String(error)}`, + service: "WorkerServices", + method: "buildWorker", + details: { ...(error instanceof AppError ? error.details : {}) }, + }); + } + return { worker, networkService, bufferService, statusService }; }; diff --git a/server/src/service/encryptionService.ts b/server/src/service/encryption/encryptionService.ts similarity index 100% rename from server/src/service/encryptionService.ts rename to server/src/service/encryption/encryptionService.ts diff --git a/server/src/service/encryption/secretsRotationService.ts b/server/src/service/encryption/secretsRotationService.ts new file mode 100644 index 000000000..c4e05881b --- /dev/null +++ b/server/src/service/encryption/secretsRotationService.ts @@ -0,0 +1,60 @@ +import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js"; +import { IEncryptionService } from "@/service/encryption/encryptionService.js"; +import { AppError } from "@/utils/AppError.js"; +import { ILogger } from "@/utils/logger.js"; + +const SERVICE_NAME = "SecretsRotationService"; + +export interface ISecretsRotationService { + run(): Promise<{ scanned: number; reencrypted: number; skipped: number }>; +} + +export class SecretsRotationService implements ISecretsRotationService { + static SERVICE_NAME = SERVICE_NAME; + + constructor( + private monitorsRepository: IMonitorsRepository, + private encryptionService: IEncryptionService, + private logger: ILogger + ) {} + + run = async () => { + const rows = await this.monitorsRepository.findAllDockerTlsKeys(); + const result = { scanned: rows.length, reencrypted: 0, skipped: 0 }; + if (rows.length === 0) return result; + + if (!this.encryptionService.isConfigured()) { + this.logger.warn({ + message: `${rows.length} monitor(s) have a stored Docker TLS key but ENCRYPTION_KEY is not set; their checks will fail until it is`, + service: SERVICE_NAME, + method: "run", + }); + result.skipped = rows.length; + return result; + } + + for (const monitor of rows) { + const { id, dockerTlsKey } = monitor; + try { + if (!this.encryptionService.needsReencryption(dockerTlsKey)) continue; + const plainText = this.encryptionService.decrypt(dockerTlsKey); + await this.monitorsRepository.updateDockerTlsKey(id, this.encryptionService.encrypt(plainText)); + result.reencrypted += 1; + } catch (error: unknown) { + result.skipped += 1; + this.logger.warn({ + message: `Could not reencrypt Docker TLS key for monitor ${id}: ${error instanceof Error ? error.message : String(error)}`, + service: SERVICE_NAME, + method: "run", + details: { monitorId: id, ...(error instanceof AppError ? error.details : {}) }, + }); + } + } + this.logger.info({ + message: `Docker TLS key rotation: ${result.reencrypted} re-encrypted, ${result.skipped} skipped, ${result.scanned} scanned`, + service: SERVICE_NAME, + method: "run", + }); + return result; + }; +} diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index c5eeda860..d68eebd38 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -25,7 +25,7 @@ import { AppError } from "@/utils/AppError.js"; import Dockerode from "dockerode"; import { timeRequest } from "@/service/network/utils.js"; import { NETWORK_ERROR } from "@/types/network.js"; -import { IEncryptionService } from "@/service/encryptionService.js"; +import { IEncryptionService } from "@/service/encryption/encryptionService.js"; import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js"; import { splitCertificateBundle } from "@/utils/pem.js"; diff --git a/server/test/unit/providers/network/dockerProvider.test.ts b/server/test/unit/providers/network/dockerProvider.test.ts index 4562db8de..f49ecef9e 100644 --- a/server/test/unit/providers/network/dockerProvider.test.ts +++ b/server/test/unit/providers/network/dockerProvider.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, jest } from "@jest/globals"; import https from "node:https"; import { DockerProvider } from "../../../../src/service/network/DockerProvider.ts"; -import type { IEncryptionService } from "../../../../src/service/encryptionService.ts"; +import type { IEncryptionService } from "../../../../src/service/encryption/encryptionService.ts"; import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts"; import { createMockLogger } from "../../../helpers/createMockLogger.ts"; import { NETWORK_ERROR } from "../../../../src/types/network.ts"; diff --git a/server/test/unit/services/encryptionService.test.ts b/server/test/unit/services/encryptionService.test.ts index c700343a3..9797f4939 100644 --- a/server/test/unit/services/encryptionService.test.ts +++ b/server/test/unit/services/encryptionService.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "@jest/globals"; import * as nodeCrypto from "node:crypto"; -import { EncryptionService } from "../../../src/service/encryptionService.ts"; -import type { EncryptionCryptoLib } from "../../../src/service/encryptionService.ts"; +import { EncryptionService } from "../../../src/service/encryption/encryptionService.ts"; +import type { EncryptionCryptoLib } from "../../../src/service/encryption/encryptionService.ts"; import { AppError } from "../../../src/utils/AppError.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; diff --git a/server/test/unit/services/secretsRotationService.test.ts b/server/test/unit/services/secretsRotationService.test.ts new file mode 100644 index 000000000..c0caf9c34 --- /dev/null +++ b/server/test/unit/services/secretsRotationService.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import * as nodeCrypto from "node:crypto"; +import { EncryptionService } from "../../../src/service/encryption/encryptionService.ts"; +import { SecretsRotationService } from "../../../src/service/encryption/secretsRotationService.ts"; +import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +type Row = { id: string; dockerTlsKey: string }; + +const PLAINTEXT = "-----BEGIN PRIVATE KEY-----\nstub\n-----END PRIVATE KEY-----"; + +const newKey = () => nodeCrypto.randomBytes(32); + +const encryptionServiceFor = (keys: Buffer[]) => new EncryptionService(keys, createMockLogger(), nodeCrypto); + +const tamperData = (ciphertext: string) => { + const parts = ciphertext.split("."); + const data = parts[4] ?? ""; + parts[4] = (data[0] === "A" ? "B" : "A") + data.slice(1); + return parts.join("."); +}; + +const setup = (keys: Buffer[], rows: Row[]) => { + const encryptionService = encryptionServiceFor(keys); + const repository = { + findAllDockerTlsKeys: jest.fn(async () => rows), + updateDockerTlsKey: jest.fn(async () => undefined), + }; + const logger = createMockLogger(); + const service = new SecretsRotationService(repository as unknown as IMonitorsRepository, encryptionService, logger); + return { service, encryptionService, repository, logger }; +}; + +const updatedKeyFor = (repository: { updateDockerTlsKey: jest.Mock }, id: string): string => { + const call = repository.updateDockerTlsKey.mock.calls.find(([calledId]) => calledId === id); + expect(call).toBeDefined(); + return (call as [string, string])[1]; +}; + +describe("SecretsRotationService", () => { + it("re-encrypts stale rows under the current key and leaves current rows untouched", async () => { + const current = newKey(); + const previous = newKey(); + const stale = encryptionServiceFor([previous]); + const rows = [ + { id: "m-current", dockerTlsKey: encryptionServiceFor([current]).encrypt(PLAINTEXT) }, + { id: "m-stale-1", dockerTlsKey: stale.encrypt(PLAINTEXT) }, + { id: "m-stale-2", dockerTlsKey: stale.encrypt(PLAINTEXT) }, + ]; + const { service, encryptionService, repository, logger } = setup([current, previous], rows); + + const result = await service.run(); + + expect(result).toEqual({ scanned: 3, reencrypted: 2, skipped: 0 }); + expect(repository.updateDockerTlsKey).toHaveBeenCalledTimes(2); + for (const id of ["m-stale-1", "m-stale-2"]) { + const rotated = updatedKeyFor(repository, id); + expect(encryptionService.keyIdOf(rotated)).toBe(encryptionService.currentKeyId()); + expect(encryptionService.decrypt(rotated)).toBe(PLAINTEXT); + } + expect(repository.updateDockerTlsKey).not.toHaveBeenCalledWith("m-current", expect.anything()); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith( + expect.objectContaining({ message: "Docker TLS key rotation: 2 re-encrypted, 0 skipped, 3 scanned", service: "SecretsRotationService" }) + ); + }); + + it("returns zero counts when every row is already under the current key", async () => { + const current = newKey(); + const rows = [ + { id: "m-1", dockerTlsKey: encryptionServiceFor([current]).encrypt(PLAINTEXT) }, + { id: "m-2", dockerTlsKey: encryptionServiceFor([current]).encrypt(PLAINTEXT) }, + ]; + const { service, repository, logger } = setup([current], rows); + + const result = await service.run(); + + expect(result).toEqual({ scanned: 2, reencrypted: 0, skipped: 0 }); + expect(repository.updateDockerTlsKey).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Docker TLS key rotation: 0 re-encrypted, 0 skipped, 2 scanned" })); + }); + + it("skips a row encrypted under an unknown key and logs its key id", async () => { + const unknown = encryptionServiceFor([newKey()]); + const rows = [{ id: "m-unknown", dockerTlsKey: unknown.encrypt(PLAINTEXT) }]; + const { service, repository, logger } = setup([newKey()], rows); + + const result = await service.run(); + + expect(result).toEqual({ scanned: 1, reencrypted: 0, skipped: 1 }); + expect(repository.updateDockerTlsKey).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining("Could not reencrypt Docker TLS key for monitor m-unknown"), + service: "SecretsRotationService", + details: { monitorId: "m-unknown", keyId: unknown.currentKeyId() }, + }) + ); + }); + + it("continues past a row that fails to decrypt", async () => { + const current = newKey(); + const previous = newKey(); + const stale = encryptionServiceFor([previous]); + const rows = [ + { id: "m-broken", dockerTlsKey: tamperData(stale.encrypt(PLAINTEXT)) }, + { id: "m-stale", dockerTlsKey: stale.encrypt(PLAINTEXT) }, + ]; + const { service, encryptionService, repository, logger } = setup([current, previous], rows); + + const result = await service.run(); + + expect(result).toEqual({ scanned: 2, reencrypted: 1, skipped: 1 }); + expect(repository.updateDockerTlsKey).toHaveBeenCalledTimes(1); + expect(encryptionService.decrypt(updatedKeyFor(repository, "m-stale"))).toBe(PLAINTEXT); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining("Could not reencrypt Docker TLS key for monitor m-broken: Ciphertext failed authentication"), + details: expect.objectContaining({ monitorId: "m-broken" }), + }) + ); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Docker TLS key rotation: 1 re-encrypted, 1 skipped, 2 scanned" })); + }); + + it("warns and skips every row when ENCRYPTION_KEY is not configured", async () => { + const rows = [ + { id: "m-1", dockerTlsKey: encryptionServiceFor([newKey()]).encrypt(PLAINTEXT) }, + { id: "m-2", dockerTlsKey: encryptionServiceFor([newKey()]).encrypt(PLAINTEXT) }, + ]; + const { service, repository, logger } = setup([], rows); + + const result = await service.run(); + + expect(result).toEqual({ scanned: 2, reencrypted: 0, skipped: 2 }); + expect(repository.updateDockerTlsKey).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ + message: "2 monitor(s) have a stored Docker TLS key but ENCRYPTION_KEY is not set; their checks will fail until it is", + service: "SecretsRotationService", + }) + ); + expect(logger.info).not.toHaveBeenCalled(); + }); + + it("logs nothing when ENCRYPTION_KEY is not configured and there are no rows", async () => { + const { service, repository, logger } = setup([], []); + + const result = await service.run(); + + expect(result).toEqual({ scanned: 0, reencrypted: 0, skipped: 0 }); + expect(repository.updateDockerTlsKey).not.toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.info).not.toHaveBeenCalled(); + }); +}); From 2260b4a4d21df89d28768e904a5a7383ce2fe1da Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 11:10:37 -0700 Subject: [PATCH 13/35] add validation for docker TLS fields, encrypt TLS key on create/edit monitor --- .../src/api/validation/monitorValidation.ts | 51 ++++- server/src/config/services.api.ts | 1 + server/src/domain/monitors/monitor.service.ts | 54 ++++++ .../test/unit/services/monitorService.test.ts | 180 ++++++++++++++++++ .../unit/validation/monitorValidation.test.ts | 128 ++++++++++++- 5 files changed, 411 insertions(+), 3 deletions(-) diff --git a/server/src/api/validation/monitorValidation.ts b/server/src/api/validation/monitorValidation.ts index 3d714c7ba..0f90aef99 100644 --- a/server/src/api/validation/monitorValidation.ts +++ b/server/src/api/validation/monitorValidation.ts @@ -16,6 +16,8 @@ import { DateRanges, SortOrders } from "@/types/query.js"; import { DockerContainerStates, DockerHealthStatuses, DockerLogStreams, DockerPortProtocols } from "@/domain/docker/docker.type.js"; import { DOCKER_LOG_PAGE_DEFAULT, DOCKER_LOG_PAGE_MAX } from "@/domain/docker/docker-log.type.js"; import { isDockerSocketUrl, isDockerTlsUrl } from "@/utils/dockerHost.js"; +import { X509Certificate } from "node:crypto"; +import { keyMatchesCertificate, parseCertificates, parsePrivateKey } from "@/utils/pem.js"; const httpStatusCode = z.number().refine((code) => HttpStatusCodeSet.has(code), { message: "Must be a valid HTTP status code" }); @@ -132,6 +134,49 @@ const refineDockerUrl = (data: { type?: string; url?: string }, ctx: z.Refinemen } }; +type DockerTlsFields = { + type?: string; + url?: string; + ignoreTlsErrors?: boolean; + dockerTlsCa?: string; + dockerTlsCert?: string; + dockerTlsKey?: string; +}; + +const refineDockerTls = (mode: "create" | "edit") => (data: DockerTlsFields, ctx: z.RefinementCtx) => { + if (data.type !== "docker" || !isDockerTlsUrl(data.url)) return; + const issue = (path: string, message: string) => ctx.addIssue({ code: "custom", path: [path], message }); + + if (!data.dockerTlsCert) issue("dockerTlsCert", "TLS certificate is required for a TLS Docker host"); + + if (mode === "create" && !data.dockerTlsKey) issue("dockerTlsKey", "TLS key is required for a TLS Docker host"); + + if (!data.ignoreTlsErrors && !data.dockerTlsCa) issue("dockerTlsCa", "CA certificate is required unless TLS errors are ignored"); + + let certificate: X509Certificate | undefined; + + try { + if (data.dockerTlsCa) parseCertificates(data.dockerTlsCa); + } catch (error: unknown) { + issue("dockerTlsCa", error instanceof Error ? error.message : "Docker TLS CA error"); + } + + try { + if (data.dockerTlsCert) [certificate] = parseCertificates(data.dockerTlsCert); + } catch (error) { + issue("dockerTlsCert", error instanceof Error ? error.message : "Docker TLS cert error"); + } + + try { + if (data.dockerTlsKey) { + const key = parsePrivateKey(data.dockerTlsKey); + if (certificate && !keyMatchesCertificate(key, certificate)) issue("dockerTlsKey", "Key does not match certificate"); + } + } catch (error: unknown) { + issue("dockerTlsKey", error instanceof Error ? error.message : "Docker TLS key error"); + } +}; + export const createMonitorBodyValidation = z .object({ _id: z.string().optional(), @@ -180,7 +225,8 @@ export const createMonitorBodyValidation = z .superRefine(refineHeadMatching) .superRefine(refineRegexPattern) .superRefine(refineProxySelection) - .superRefine(refineDockerUrl); + .superRefine(refineDockerUrl) + .superRefine(refineDockerTls("create")); export const editMonitorBodyValidation = z .object({ @@ -228,7 +274,8 @@ export const editMonitorBodyValidation = z .superRefine(refineHeadMatching) .superRefine(refineRegexPattern) .superRefine(refineProxySelection) - .superRefine(refineDockerUrl); + .superRefine(refineDockerUrl) + .superRefine(refineDockerTls("edit")); export const pauseMonitorParamValidation = z.object({ monitorId: z.string().min(1, "Monitor ID is required"), diff --git a/server/src/config/services.api.ts b/server/src/config/services.api.ts index 65607107c..c5f58ac19 100644 --- a/server/src/config/services.api.ts +++ b/server/src/config/services.api.ts @@ -80,6 +80,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A monitorStatsRepository, statusPagesRepository, incidentsRepository, + encryptionService: shared.encryptionService, }); const maintenanceWindowService = new MaintenanceWindowService({ diff --git a/server/src/domain/monitors/monitor.service.ts b/server/src/domain/monitors/monitor.service.ts index 9e4f7f6e5..5d02098ba 100644 --- a/server/src/domain/monitors/monitor.service.ts +++ b/server/src/domain/monitors/monitor.service.ts @@ -33,6 +33,8 @@ import { ILogger } from "@/utils/logger.js"; import { IJobScheduler } from "@/worker/worker.interface.js"; import { DateRange } from "@/types/query.js"; import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js"; +import { IEncryptionService } from "@/service/encryption/encryptionService.js"; +import { isDockerTlsUrl } from "@/utils/dockerHost.js"; const SERVICE_NAME = "MonitorService"; @@ -144,6 +146,7 @@ export class MonitorService implements IMonitorService { private monitorStatsRepository: IMonitorStatsRepository; private statusPagesRepository: IStatusPagesRepository; private incidentsRepository: IIncidentsRepository; + private encryptionService: IEncryptionService; constructor({ scheduler, @@ -156,6 +159,7 @@ export class MonitorService implements IMonitorService { monitorStatsRepository, statusPagesRepository, incidentsRepository, + encryptionService, }: { scheduler: IJobScheduler; logger: ILogger; @@ -167,6 +171,7 @@ export class MonitorService implements IMonitorService { monitorStatsRepository: IMonitorStatsRepository; statusPagesRepository: IStatusPagesRepository; incidentsRepository: IIncidentsRepository; + encryptionService: IEncryptionService; }) { this.scheduler = scheduler; this.logger = logger; @@ -178,6 +183,7 @@ export class MonitorService implements IMonitorService { this.monitorStatsRepository = monitorStatsRepository; this.statusPagesRepository = statusPagesRepository; this.incidentsRepository = incidentsRepository; + this.encryptionService = encryptionService; } createMonitor = async (teamId: string, userId: string, body: Monitor): Promise => { @@ -185,6 +191,10 @@ export class MonitorService implements IMonitorService { if (body.proxyMode !== "custom") { delete body.proxyId; } + + // Encrypt docker TLS keys + this.encryptDockerTls(body, "create"); + const monitor = await this.monitorsRepository.create(body, teamId, userId); if (!monitor) { throw new AppError({ message: "Failed to create monitor", status: 500, service: SERVICE_NAME, method: "createMonitor" }); @@ -537,6 +547,20 @@ export class MonitorService implements IMonitorService { if (unsetProxyId) { delete body.proxyId; } + + // Handle Docker TLS + this.encryptDockerTls(body, "edit"); + if (body.type === "docker" && isDockerTlsUrl(body.url) && body.dockerTlsKey === undefined) { + const stored = await this.monitorsRepository.findById(monitorId, teamId); + if (!stored.dockerTlsKeySet) { + throw new AppError({ + message: "Key is required for a TLS Docker host", + status: 422, + service: SERVICE_NAME, + method: "editMonitor", + }); + } + } const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body, { unsetProxyId }); await this.scheduler.updateJob(editedMonitor); return editedMonitor; @@ -724,4 +748,34 @@ export class MonitorService implements IMonitorService { return { imported: createdMonitors!.length, errors }; }; + + private encryptDockerTls = (body: Partial, mode: "create" | "edit") => { + if (body.type !== "docker" && mode === "create") return; + + if (body.url !== undefined && !isDockerTlsUrl(body.url)) { + body.dockerTlsCa = undefined; + body.dockerTlsCert = undefined; + body.dockerTlsKey = undefined; + body.dockerTlsKeySet = false; + return; + } + + if (!body.dockerTlsKey) { + delete body.dockerTlsKey; // empty field means keeps stored key + return; + } + + if (!this.encryptionService.isConfigured()) { + throw new AppError({ + message: + 'Docker TLS credentials require ENCRYPTION_KEY to be set on the server. Generate one with "openssl rand -base64 32", set the same value for both the API and worker, and restart.', + status: 422, + service: SERVICE_NAME, + method: "encryptDockerTls", + }); + } + + body.dockerTlsKey = this.encryptionService.encrypt(body.dockerTlsKey); + body.dockerTlsKeySet = true; + }; } diff --git a/server/test/unit/services/monitorService.test.ts b/server/test/unit/services/monitorService.test.ts index d3f5dd1a7..31328cd18 100644 --- a/server/test/unit/services/monitorService.test.ts +++ b/server/test/unit/services/monitorService.test.ts @@ -7,6 +7,8 @@ import type { IIncidentsRepository } from "../../../src/domain/incidents/inciden import type { IMonitorStatsRepository } from "../../../src/domain/monitor-stats/monitor-stats.repository.interface.ts"; import type { IMonitorsRepository } from "../../../src/domain/monitors/monitor.repository.interface.ts"; import type { IStatusPagesRepository } from "../../../src/domain/status-pages/status-page-repository.interface.ts"; +import type { IEncryptionService } from "../../../src/service/encryption/encryptionService.ts"; +import { AppError } from "../../../src/utils/AppError.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; const createMonitorsRepositoryMock = () => @@ -70,6 +72,17 @@ const createJobQueueMock = () => ({ deleteJob: jest.fn(), }); +const createEncryptionServiceMock = (overrides: Partial = {}) => + ({ + isConfigured: jest.fn(() => true), + currentKeyId: jest.fn(() => "key-1"), + keyIdOf: jest.fn(() => "key-1"), + needsReencryption: jest.fn(() => false), + encrypt: jest.fn((plaintext: string) => `enc:${plaintext}`), + decrypt: jest.fn((ciphertext: string) => ciphertext.replace(/^enc:/, "")), + ...overrides, + }) as unknown as IEncryptionService; + const TEAM_ID = "team-1"; const MONITOR_ID = "monitor-1"; const USER_ID = "user-1"; @@ -98,6 +111,7 @@ const createService = ( geoChecksRepository?: IGeoChecksRepository; dockerLogsRepository?: IDockerLogsRepository; incidentsRepository?: IIncidentsRepository; + encryptionService?: IEncryptionService; jobQueue?: ReturnType; logger?: ReturnType; games?: Record; @@ -116,6 +130,7 @@ const createService = ( monitorStatsRepository: overrides.monitorStatsRepository ?? createMonitorStatsRepositoryMock(), statusPagesRepository: overrides.statusPagesRepository ?? createStatusPagesRepositoryMock(), incidentsRepository: overrides.incidentsRepository ?? createIncidentsRepositoryMock(), + encryptionService: overrides.encryptionService ?? createEncryptionServiceMock(), }); return { service, jobQueue, logger }; }; @@ -1893,6 +1908,7 @@ describe("MonitorService", () => { monitorStatsRepository, statusPagesRepository, incidentsRepository, + encryptionService: createEncryptionServiceMock(), }); const result = await service.deleteAllMonitors({ teamId: TEAM_ID }); @@ -1981,3 +1997,167 @@ describe("MonitorService", () => { }); }); }); + +describe("MonitorService — Docker TLS credentials", () => { + const KEY_PEM = "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----"; + const tlsBody = { type: "docker" as const, url: "tcp://host:2376", dockerTlsCa: "ca-pem", dockerTlsCert: "cert-pem", dockerTlsKey: KEY_PEM }; + + const expectAppError = async (fn: () => Promise, status: number, message: string | RegExp) => { + let caught: unknown; + try { + await fn(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(AppError); + expect((caught as AppError).status).toBe(status); + expect((caught as AppError).message).toMatch(message); + }; + + describe("createMonitor", () => { + it("encrypts the key and sets dockerTlsKeySet for a TLS url", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.create as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" })); + const encryptionService = createEncryptionServiceMock(); + const { service } = createService({ monitorsRepository, encryptionService }); + + await service.createMonitor(TEAM_ID, USER_ID, makeMonitor(tlsBody) as any); + + expect(encryptionService.encrypt).toHaveBeenCalledWith(KEY_PEM); + expect(monitorsRepository.create).toHaveBeenCalledWith( + expect.objectContaining({ dockerTlsCa: "ca-pem", dockerTlsCert: "cert-pem", dockerTlsKey: `enc:${KEY_PEM}`, dockerTlsKeySet: true }), + TEAM_ID, + USER_ID + ); + }); + + it("does not encrypt anything for a socket url and leaves dockerTlsKeySet false", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.create as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" })); + const encryptionService = createEncryptionServiceMock(); + const { service } = createService({ monitorsRepository, encryptionService }); + + await service.createMonitor(TEAM_ID, USER_ID, makeMonitor({ type: "docker", url: "unix:///var/run/docker.sock" }) as any); + + expect(encryptionService.encrypt).not.toHaveBeenCalled(); + const [created] = (monitorsRepository.create as jest.Mock).mock.calls[0] as [Record]; + expect(created.dockerTlsKeySet).toBe(false); + expect(created.dockerTlsKey).toBeUndefined(); + }); + + it("ignores TLS fields on non-docker monitors", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.create as jest.Mock).mockResolvedValue(makeMonitor()); + const encryptionService = createEncryptionServiceMock(); + const { service } = createService({ monitorsRepository, encryptionService }); + + await service.createMonitor(TEAM_ID, USER_ID, makeMonitor({ dockerTlsKey: KEY_PEM }) as any); + + expect(encryptionService.encrypt).not.toHaveBeenCalled(); + expect(monitorsRepository.create).toHaveBeenCalledWith(expect.objectContaining({ dockerTlsKey: KEY_PEM }), TEAM_ID, USER_ID); + }); + + it("rejects with a 422 naming ENCRYPTION_KEY when encryption is not configured", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + const encryptionService = createEncryptionServiceMock({ isConfigured: jest.fn(() => false) as any }); + const { service } = createService({ monitorsRepository, encryptionService }); + + await expectAppError(() => service.createMonitor(TEAM_ID, USER_ID, makeMonitor(tlsBody) as any), 422, /ENCRYPTION_KEY/); + expect(encryptionService.encrypt).not.toHaveBeenCalled(); + expect(monitorsRepository.create).not.toHaveBeenCalled(); + }); + }); + + describe("editMonitor", () => { + it("keeps the stored key when the key field is blank", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker", dockerTlsKeySet: true })); + (monitorsRepository.updateById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" })); + const encryptionService = createEncryptionServiceMock(); + const { service } = createService({ monitorsRepository, encryptionService }); + + await service.editMonitor({ teamId: TEAM_ID, monitorId: MONITOR_ID, body: { ...tlsBody, dockerTlsKey: "" } }); + + expect(encryptionService.encrypt).not.toHaveBeenCalled(); + const [, , patch] = (monitorsRepository.updateById as jest.Mock).mock.calls[0] as [string, string, Record]; + expect(patch).not.toHaveProperty("dockerTlsKey"); + expect(patch).not.toHaveProperty("dockerTlsKeySet"); + expect(patch).toMatchObject({ dockerTlsCa: "ca-pem", dockerTlsCert: "cert-pem" }); + }); + + it("rejects a blank key with a 422 when no key is stored", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.findById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker", dockerTlsKeySet: false })); + const { service } = createService({ monitorsRepository }); + + await expectAppError( + () => service.editMonitor({ teamId: TEAM_ID, monitorId: MONITOR_ID, body: { ...tlsBody, dockerTlsKey: "" } }), + 422, + /required for a TLS Docker host/ + ); + expect(monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + + it("encrypts a replacement key without reading the stored monitor", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.updateById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" })); + const encryptionService = createEncryptionServiceMock(); + const { service } = createService({ monitorsRepository, encryptionService }); + + await service.editMonitor({ teamId: TEAM_ID, monitorId: MONITOR_ID, body: tlsBody }); + + expect(monitorsRepository.findById).not.toHaveBeenCalled(); + expect(monitorsRepository.updateById).toHaveBeenCalledWith( + MONITOR_ID, + TEAM_ID, + expect.objectContaining({ dockerTlsKey: `enc:${KEY_PEM}`, dockerTlsKeySet: true }), + { unsetProxyId: false } + ); + }); + + it("encrypts a replacement key when the body omits the url", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.updateById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" })); + const encryptionService = createEncryptionServiceMock(); + const { service } = createService({ monitorsRepository, encryptionService }); + + await service.editMonitor({ teamId: TEAM_ID, monitorId: MONITOR_ID, body: { type: "docker", dockerTlsKey: KEY_PEM } }); + + expect(encryptionService.encrypt).toHaveBeenCalledWith(KEY_PEM); + expect(monitorsRepository.updateById).toHaveBeenCalledWith( + MONITOR_ID, + TEAM_ID, + expect.objectContaining({ dockerTlsKey: `enc:${KEY_PEM}`, dockerTlsKeySet: true }), + { unsetProxyId: false } + ); + }); + + it("clears dockerTlsKeySet when the url changes back to a socket", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + (monitorsRepository.updateById as jest.Mock).mockResolvedValue(makeMonitor({ type: "docker" })); + const encryptionService = createEncryptionServiceMock(); + const { service } = createService({ monitorsRepository, encryptionService }); + + await service.editMonitor({ + teamId: TEAM_ID, + monitorId: MONITOR_ID, + body: { type: "docker", url: "unix:///var/run/docker.sock", dockerTlsKey: KEY_PEM }, + }); + + expect(encryptionService.encrypt).not.toHaveBeenCalled(); + expect(monitorsRepository.findById).not.toHaveBeenCalled(); + const [, , patch] = (monitorsRepository.updateById as jest.Mock).mock.calls[0] as [string, string, Record]; + expect(patch.dockerTlsKeySet).toBe(false); + expect(patch.dockerTlsKey).toBeUndefined(); + }); + + it("rejects with a 422 naming ENCRYPTION_KEY when encryption is not configured", async () => { + const monitorsRepository = createMonitorsRepositoryMock(); + const encryptionService = createEncryptionServiceMock({ isConfigured: jest.fn(() => false) as any }); + const { service } = createService({ monitorsRepository, encryptionService }); + + await expectAppError(() => service.editMonitor({ teamId: TEAM_ID, monitorId: MONITOR_ID, body: tlsBody }), 422, /ENCRYPTION_KEY/); + expect(monitorsRepository.updateById).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/server/test/unit/validation/monitorValidation.test.ts b/server/test/unit/validation/monitorValidation.test.ts index c9b7d16f0..5b626d329 100644 --- a/server/test/unit/validation/monitorValidation.test.ts +++ b/server/test/unit/validation/monitorValidation.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "@jest/globals"; +import { readFileSync } from "node:fs"; import { createMonitorBodyValidation, editMonitorBodyValidation, @@ -621,8 +622,10 @@ describe("monitorValidation — Docker host url", () => { }); it("accepts tcp:// and https:// engine urls with an optional port and trailing slash", () => { + const fixture = (name: string) => readFileSync(new URL(`../../fixtures/docker-tls/${name}`, import.meta.url), "utf8"); + const tlsFields = { dockerTlsCa: fixture("ca.pem"), dockerTlsCert: fixture("client-cert.pem"), dockerTlsKey: fixture("client-key.pem") }; for (const url of ["tcp://host", "tcp://host:2376", "https://host:2377", "tcp://host/"]) { - expect(createMonitorBodyValidation.parse({ ...baseDockerBody, url }).url).toBe(url); + expect(createMonitorBodyValidation.parse({ ...baseDockerBody, ...tlsFields, url }).url).toBe(url); } }); @@ -673,3 +676,126 @@ describe("monitorValidation — Docker host url", () => { }); }); }); + +describe("monitorValidation — Docker TLS credentials", () => { + const fixture = (name: string) => readFileSync(new URL(`../../fixtures/docker-tls/${name}`, import.meta.url), "utf8"); + const CA = fixture("ca.pem"); + const CLIENT_CERT = fixture("client-cert.pem"); + const CLIENT_KEY = fixture("client-key.pem"); + const OTHER_KEY = fixture("other-key.pem"); + const ENCRYPTED_KEY = fixture("encrypted-key.pem"); + + const baseTlsBody = { + name: "Docker TLS check", + type: "docker" as const, + url: "tcp://host:2376", + }; + const fullTlsBody = { ...baseTlsBody, dockerTlsCa: CA, dockerTlsCert: CLIENT_CERT, dockerTlsKey: CLIENT_KEY }; + + const issuesFor = (result: ReturnType) => + result.success ? [] : result.error.issues.map((issue) => ({ path: issue.path.join("."), message: issue.message })); + + describe("createMonitorBodyValidation", () => { + it("accepts a CA, client certificate, and matching key", () => { + const parsed = createMonitorBodyValidation.parse(fullTlsBody); + expect(parsed.dockerTlsCa).toBe(CA); + expect(parsed.dockerTlsCert).toBe(CLIENT_CERT); + expect(parsed.dockerTlsKey).toBe(CLIENT_KEY); + }); + + it("does not require TLS fields for a socket url", () => { + expect(createMonitorBodyValidation.safeParse({ ...baseTlsBody, url: "unix:///var/run/docker.sock" }).success).toBe(true); + }); + + it("requires the client certificate, key, and CA for a TLS url", () => { + const issues = issuesFor(createMonitorBodyValidation.safeParse(baseTlsBody)); + expect(issues).toEqual( + expect.arrayContaining([ + { path: "dockerTlsCert", message: "TLS certificate is required for a TLS Docker host" }, + { path: "dockerTlsKey", message: "TLS key is required for a TLS Docker host" }, + { path: "dockerTlsCa", message: "CA certificate is required unless TLS errors are ignored" }, + ]) + ); + }); + + it("makes the CA optional when TLS errors are ignored", () => { + const result = createMonitorBodyValidation.safeParse({ + ...baseTlsBody, + ignoreTlsErrors: true, + dockerTlsCert: CLIENT_CERT, + dockerTlsKey: CLIENT_KEY, + }); + expect(result.success).toBe(true); + }); + + it("still requires the client certificate and key when TLS errors are ignored", () => { + const issues = issuesFor(createMonitorBodyValidation.safeParse({ ...baseTlsBody, ignoreTlsErrors: true })); + expect(issues.map((issue) => issue.path)).toEqual(expect.arrayContaining(["dockerTlsCert", "dockerTlsKey"])); + expect(issues.map((issue) => issue.path)).not.toContain("dockerTlsCa"); + }); + + it("rejects a CA that is not a PEM certificate", () => { + const issues = issuesFor(createMonitorBodyValidation.safeParse({ ...fullTlsBody, dockerTlsCa: "not a certificate" })); + expect(issues).toEqual([{ path: "dockerTlsCa", message: "No certificate found; expected one or more PEM CERTIFICATE blocks" }]); + }); + + it("rejects a client certificate that is not a PEM certificate", () => { + const issues = issuesFor(createMonitorBodyValidation.safeParse({ ...fullTlsBody, dockerTlsCert: "not a certificate" })); + expect(issues).toEqual([{ path: "dockerTlsCert", message: "No certificate found; expected one or more PEM CERTIFICATE blocks" }]); + }); + + it("rejects a key that is not a PEM private key", () => { + const issues = issuesFor(createMonitorBodyValidation.safeParse({ ...fullTlsBody, dockerTlsKey: "not a key" })); + expect(issues).toEqual([{ path: "dockerTlsKey", message: "Private key is not valid PEM" }]); + }); + + it("rejects an encrypted private key", () => { + const issues = issuesFor(createMonitorBodyValidation.safeParse({ ...fullTlsBody, dockerTlsKey: ENCRYPTED_KEY })); + expect(issues).toEqual([{ path: "dockerTlsKey", message: "Encrypted private keys are not supported; remove the passphrase first" }]); + }); + + it("rejects a key that does not match the client certificate", () => { + const issues = issuesFor(createMonitorBodyValidation.safeParse({ ...fullTlsBody, dockerTlsKey: OTHER_KEY })); + expect(issues).toEqual([{ path: "dockerTlsKey", message: "Key does not match certificate" }]); + }); + + it("accepts a CA bundle with more than one certificate", () => { + const result = createMonitorBodyValidation.safeParse({ ...fullTlsBody, dockerTlsCa: `${CA}\n${CLIENT_CERT}` }); + expect(result.success).toBe(true); + }); + }); + + describe("editMonitorBodyValidation", () => { + it("accepts a TLS url without a key so a stored key is kept", () => { + const result = editMonitorBodyValidation.safeParse({ ...baseTlsBody, dockerTlsCa: CA, dockerTlsCert: CLIENT_CERT }); + expect(result.success).toBe(true); + }); + + it("accepts a blank key so a stored key is kept", () => { + const result = editMonitorBodyValidation.safeParse({ ...baseTlsBody, dockerTlsCa: CA, dockerTlsCert: CLIENT_CERT, dockerTlsKey: "" }); + expect(result.success).toBe(true); + }); + + it("still requires the client certificate and CA", () => { + const issues = issuesFor(editMonitorBodyValidation.safeParse(baseTlsBody)); + expect(issues.map((issue) => issue.path)).toEqual(expect.arrayContaining(["dockerTlsCert", "dockerTlsCa"])); + expect(issues.map((issue) => issue.path)).not.toContain("dockerTlsKey"); + }); + + it("validates a replacement key against the client certificate", () => { + const issues = issuesFor( + editMonitorBodyValidation.safeParse({ ...baseTlsBody, dockerTlsCa: CA, dockerTlsCert: CLIENT_CERT, dockerTlsKey: OTHER_KEY }) + ); + expect(issues).toEqual([{ path: "dockerTlsKey", message: "Key does not match certificate" }]); + }); + }); + + describe("importMonitorsBodyValidation", () => { + it("strips TLS credentials from imported monitors instead of validating them", () => { + const parsed = importMonitorsBodyValidation.parse({ monitors: [fullTlsBody] }); + expect(parsed.monitors[0]).not.toHaveProperty("dockerTlsCa"); + expect(parsed.monitors[0]).not.toHaveProperty("dockerTlsCert"); + expect(parsed.monitors[0]).not.toHaveProperty("dockerTlsKey"); + }); + }); +}); From 9d72d20dd2f1d8dfcde7eba7f6efe2f633d91882 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 11:20:47 -0700 Subject: [PATCH 14/35] docker TLS validation --- client/src/Validation/monitor.ts | 51 ++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 3a1f479dd..f8ec5c322 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -108,10 +108,30 @@ const portSchema = baseSchema.extend({ .max(65535, "Port must be at most 65535"), }); -// Docker monitor schema +// Docker monitor schema. The url forms mirror DOCKER_TLS_URL and +// DOCKER_SOCKET_URL in server/src/utils/dockerHost.ts. +const dockerTlsUrlRegex = /^(tcp|https):\/\/([^/\s:]+)(?::(\d+))?\/?$/; +const dockerSocketUrlRegex = /^(?:unix:\/\/\/\S+|\/\S+)$/; + +export const isDockerTlsUrl = (url: string | undefined): boolean => + dockerTlsUrlRegex.test(url?.trim() ?? ""); + const dockerSchema = baseSchema.extend({ type: z.literal("docker"), - url: z.string().min(1, "Docker host URL is required"), + url: z + .string() + .min(1, "Docker host URL is required") + .refine( + (url) => dockerSocketUrlRegex.test(url.trim()) || isDockerTlsUrl(url), + "Docker host must be unix:///path, an absolute socket path, or tcp://host[:port]" + ), + ignoreTlsErrors: z.boolean(), + dockerTlsCa: z.string(), + dockerTlsCert: z.string(), + // Never prefilled; blank on edit keeps the stored key. + dockerTlsKey: z.string(), + // Read-only mirror of the server flag; stripped by the server on submit. + dockerTlsKeySet: z.boolean(), dockerLogsEnabled: z.boolean().register(monitorStepRegistry, { step: 1 }), }); @@ -217,7 +237,8 @@ const monitorSchemaUnion = z.discriminatedUnion("type", [ dnsSchema, ]); -// Mirrors the server's refineProxySelection (monitorValidation.ts) +// Mirrors the server's refineProxySelection and refineDockerTls +// (monitorValidation.ts). PEM contents are only parsed on the server. export const monitorSchema = monitorSchemaUnion.superRefine((data, ctx) => { if (data.type === "http" && data.proxyMode === "custom" && !data.proxyId) { ctx.addIssue({ @@ -226,6 +247,30 @@ export const monitorSchema = monitorSchemaUnion.superRefine((data, ctx) => { message: "A proxy must be selected when proxy mode is custom", }); } + + if (data.type === "docker" && isDockerTlsUrl(data.url)) { + if (!data.dockerTlsCert.trim()) { + ctx.addIssue({ + code: "custom", + path: ["dockerTlsCert"], + message: "Client certificate is required for a TLS Docker host", + }); + } + if (!data.dockerTlsKeySet && !data.dockerTlsKey.trim()) { + ctx.addIssue({ + code: "custom", + path: ["dockerTlsKey"], + message: "Client key is required for a TLS Docker host", + }); + } + if (!data.ignoreTlsErrors && !data.dockerTlsCa.trim()) { + ctx.addIssue({ + code: "custom", + path: ["dockerTlsCa"], + message: "CA certificate is required unless TLS errors are ignored", + }); + } + } }); export type MonitorFormData = z.infer; From 5c0279a6a71b3549d8453fd4dd59a48de76be23d Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 11:21:13 -0700 Subject: [PATCH 15/35] add dockerTLS to form --- client/src/Hooks/useMonitorForm.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 043796ccb..89ed0f066 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -77,6 +77,12 @@ export const getMonitorDefaults = ( ...base, type: "docker", url: data?.url || "", + ignoreTlsErrors: data?.ignoreTlsErrors || false, + dockerTlsCa: data?.dockerTlsCa || "", + dockerTlsCert: data?.dockerTlsCert || "", + // The key is write-only; the API never returns it. + dockerTlsKey: "", + dockerTlsKeySet: data?.dockerTlsKeySet ?? false, dockerLogsEnabled: data?.dockerLogsEnabled ?? false, }; break; From 727b904ca9ce4cc29e68b29c0971d600d05b2c0b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 11:21:39 -0700 Subject: [PATCH 16/35] add docker TLS to monitor type --- client/src/Types/Monitor.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index c81df9c82..858c827d5 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -160,6 +160,9 @@ export interface Monitor { geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; dockerLogsEnabled?: boolean; + dockerTlsCa?: string; + dockerTlsCert?: string; + dockerTlsKeySet?: boolean; dnsServer?: string; dnsRecordType?: DnsRecordType; recentChecks: CheckSnapshot[]; From 8c9cb4be776051ebc405d253f8d7e3ef7801b8df Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 11:23:11 -0700 Subject: [PATCH 17/35] update create monitor form --- client/src/Pages/CreateMonitor/index.tsx | 70 ++++++++++++++++++++++++ client/src/locales/en.json | 25 ++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 3658276cd..3291ab090 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -50,6 +50,7 @@ import type { AppSettingsResponse } from "@/Types/Settings"; import { stepFieldsFor, monitorStepCount, + isDockerTlsUrl, type MonitorFormData, } from "@/Validation/monitor"; import { FormNumberField } from "@/Components/inputs/forms/FormNumberField"; @@ -66,6 +67,14 @@ const httpMethodOptions = HttpMethods.map((method) => ({ label: method, })); +const PEM_FIELD_ROWS = 6; + +// Hides the pasted key while the field is not focused. Ignored by browsers +// without -webkit-text-security, which then show the key as plain text. +const maskedInputSx = { + "&:not(.Mui-focused) textarea": { WebkitTextSecurity: "disc" }, +}; + interface GeneralSettingsConfig { urlLabel: string; urlPlaceholder: string; @@ -300,6 +309,7 @@ const CreateMonitorPage = () => { const showStep = (step: number) => isEditMode || currentStep === step; const watchedType = watch("type") as MonitorType; + const watchedUrl = watch("url") as string; const watchedMethod = watch("method") as HttpMethod | undefined; const watchedProxyMode = watch("proxyMode") as ProxyMode | undefined; const watchedUseAdvancedMatching = watch("useAdvancedMatching") as boolean; @@ -334,6 +344,8 @@ const CreateMonitorPage = () => { [watchedType, t] ); + const pemInputSx = { fontFamily: theme.typography.fontFamilyMonospace }; + const { post, loading: isCreating } = usePost(); const { patch, loading: isUpdating } = usePatch(); const isSubmitting = isCreating || isUpdating; @@ -689,6 +701,64 @@ const CreateMonitorPage = () => { /> )} + {showStep(0) && watchedType === "docker" && isDockerTlsUrl(watchedUrl) && ( + + + + + + + {t("pages.createMonitor.form.dockerTls.option.ignoreTls.description")} + + + } + /> + )} + {showStep(1) && ( supported games list for protocol details.", "pagespeed": "Track page load performance, Core Web Vitals, and optimization scores for your website.", "grpc": "Enter the hostname and port of the gRPC server, optionally specify a service name for the health check, and add a display name.", @@ -834,6 +834,29 @@ "description": "This will permanently delete all monitoring history, statistics, and check data for your team. This action cannot be undone.", "confirm": "Yes, clear all stats" } + }, + "dockerTls": { + "title": "TLS credentials", + "description": "Connect to a remote Docker daemon over mutual TLS. Paste the PEM files you would pass to docker --tlsverify. The client key is encrypted before it is stored and is never shown again.", + "option": { + "ca": { + "label": "CA certificate", + "placeholder": "-----BEGIN CERTIFICATE-----" + }, + "cert": { + "label": "Client certificate", + "placeholder": "-----BEGIN CERTIFICATE-----" + }, + "key": { + "label": "Client key", + "placeholder": "-----BEGIN PRIVATE KEY-----", + "stored": "A key is stored. Leave blank to keep it." + }, + "ignoreTls": { + "label": "Ignore TLS/SSL errors", + "description": "Skips verifying the daemon's certificate, so the CA certificate becomes optional. The client certificate and key are still required because the daemon verifies them." + } + } } } }, From 912f1f960d5667194f326ec445b7c1872a638579 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 11:50:26 -0700 Subject: [PATCH 18/35] add missing timeout --- server/src/service/network/DockerProvider.ts | 11 +++++++---- .../providers/network/dockerProvider.test.ts | 19 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index d68eebd38..71fd4a618 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -30,10 +30,12 @@ import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js"; import { splitCertificateBundle } from "@/utils/pem.js"; type DockerodeType = typeof Dockerode; -type DockerOptions = Dockerode.DockerOptions & { agent?: https.Agent }; +// @types/dockerode omits agent and connectionTimeout; docker-modem forwards both +type DockerOptions = Dockerode.DockerOptions & { agent?: https.Agent; connectionTimeout?: number }; type TlsCredentials = { ok: true; key: string } | { ok: false; message: string }; const SERVICE_NAME = "DockerProvider"; +const TIMEOUT_MS = 10000; const STATS_CONCURRENCY = 5; const DOCKER_LOG_MAX_LINE_BYTES = 4096; const DOCKER_LOG_TRUNCATION_MARKER = " …[truncated]"; @@ -92,9 +94,9 @@ export class DockerProvider implements IStatusProvider { if (url.startsWith("unix://")) { const socketPath = url.slice("unix://".length); if (!socketPath.startsWith("/")) throw this.invalidUrl(`Invalid Docker host URL: ${url}`); - return { socketPath }; + return { socketPath, timeout: TIMEOUT_MS, connectionTimeout: TIMEOUT_MS }; } - if (url.startsWith("/")) return { socketPath: url }; + if (url.startsWith("/")) return { socketPath: url, timeout: TIMEOUT_MS, connectionTimeout: TIMEOUT_MS }; // Docker TLS const match = DOCKER_TLS_URL.exec(url); @@ -115,7 +117,8 @@ export class DockerProvider implements IStatusProvider { socketPath: undefined, username: undefined, sshOptions: undefined, - timeout: undefined, + timeout: TIMEOUT_MS, + connectionTimeout: TIMEOUT_MS, }; }; diff --git a/server/test/unit/providers/network/dockerProvider.test.ts b/server/test/unit/providers/network/dockerProvider.test.ts index f49ecef9e..58b2d7903 100644 --- a/server/test/unit/providers/network/dockerProvider.test.ts +++ b/server/test/unit/providers/network/dockerProvider.test.ts @@ -27,6 +27,9 @@ const makeEncryptionService = (overrides?: Partial): IEncryp ...overrides, }); +const TIMEOUT_MS = 10000; +const TIMEOUTS = { timeout: TIMEOUT_MS, connectionTimeout: TIMEOUT_MS }; + const makeTlsMonitor = (overrides?: Partial): Monitor => makeMonitor({ url: "tcp://host", dockerTlsCa: CA_PEM, dockerTlsCert: CERT_PEM, dockerTlsKeySet: true, ...overrides }); @@ -139,7 +142,7 @@ describe("DockerProvider", () => { await provider.handle(makeMonitor({ url: "unix:///run/docker.sock" })); - expect(DockerLib).toHaveBeenCalledWith({ socketPath: "/run/docker.sock" }); + expect(DockerLib).toHaveBeenCalledWith({ socketPath: "/run/docker.sock", ...TIMEOUTS }); }); it("parses bare absolute paths into socketPath", async () => { @@ -147,7 +150,7 @@ describe("DockerProvider", () => { await provider.handle(makeMonitor({ url: "/var/run/docker.sock" })); - expect(DockerLib).toHaveBeenCalledWith({ socketPath: "/var/run/docker.sock" }); + expect(DockerLib).toHaveBeenCalledWith({ socketPath: "/var/run/docker.sock", ...TIMEOUTS }); }); it("trims surrounding whitespace", async () => { @@ -155,7 +158,7 @@ describe("DockerProvider", () => { await provider.handle(makeMonitor({ url: " /var/run/docker.sock " })); - expect(DockerLib).toHaveBeenCalledWith({ socketPath: "/var/run/docker.sock" }); + expect(DockerLib).toHaveBeenCalledWith({ socketPath: "/var/run/docker.sock", ...TIMEOUTS }); }); }); @@ -220,11 +223,19 @@ describe("DockerProvider", () => { await provider.handle(makeTlsMonitor(), ctx); const options = optionsPassedTo(DockerLib); - for (const field of ["socketPath", "username", "sshOptions", "timeout"]) { + for (const field of ["socketPath", "username", "sshOptions"]) { expect(options).toHaveProperty(field, undefined); } }); + it("bounds every request with the provider timeout so a filtered port fails fast", async () => { + const { provider, DockerLib } = setup(); + + await provider.handle(makeTlsMonitor(), ctx); + + expect(optionsPassedTo(DockerLib)).toMatchObject(TIMEOUTS); + }); + it("returns a down check and never constructs a client when the key is missing from the context", async () => { const { provider, DockerLib, encryptionService } = setup(); From ba0d685c6dd9216052c83c91a4bac2071d669069 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Fri, 11 Sep 2026 12:07:45 -0700 Subject: [PATCH 19/35] commetns --- client/src/Validation/monitor.ts | 4 ++-- server/src/service/network/DockerProvider.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index f8ec5c322..8a2b6e726 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -128,9 +128,9 @@ const dockerSchema = baseSchema.extend({ ignoreTlsErrors: z.boolean(), dockerTlsCa: z.string(), dockerTlsCert: z.string(), - // Never prefilled; blank on edit keeps the stored key. + // blank on edit keeps the stored key. dockerTlsKey: z.string(), - // Read-only mirror of the server flag; stripped by the server on submit. + // Read only mirror of the server flag, stripped on submit dockerTlsKeySet: z.boolean(), dockerLogsEnabled: z.boolean().register(monitorStepRegistry, { step: 1 }), }); diff --git a/server/src/service/network/DockerProvider.ts b/server/src/service/network/DockerProvider.ts index 71fd4a618..aa8070649 100644 --- a/server/src/service/network/DockerProvider.ts +++ b/server/src/service/network/DockerProvider.ts @@ -30,7 +30,6 @@ import { DOCKER_TLS_URL, isDockerTlsUrl } from "@/utils/dockerHost.js"; import { splitCertificateBundle } from "@/utils/pem.js"; type DockerodeType = typeof Dockerode; -// @types/dockerode omits agent and connectionTimeout; docker-modem forwards both type DockerOptions = Dockerode.DockerOptions & { agent?: https.Agent; connectionTimeout?: number }; type TlsCredentials = { ok: true; key: string } | { ok: false; message: string }; From 406518e0a5351dac0f6ba9979124f7f064adc1f1 Mon Sep 17 00:00:00 2001 From: Jan Kahmen <36455663+kah-ja@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:08:33 +0000 Subject: [PATCH 20/35] validate invite token type on registration --- server/src/api/controllers/authController.ts | 3 ++- server/src/api/validation/authValidation.ts | 2 ++ .../unit/validation/authValidation.test.ts | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 server/test/unit/validation/authValidation.test.ts diff --git a/server/src/api/controllers/authController.ts b/server/src/api/controllers/authController.ts index ab8d1fc61..a28393089 100755 --- a/server/src/api/controllers/authController.ts +++ b/server/src/api/controllers/authController.ts @@ -5,6 +5,7 @@ import { requireTeamId, requireUserEmail, requireUserId, requireUserRoles } from import { registrationBodyValidation, + registerInviteTokenValidation, loginValidation, recoveryValidation, recoveryTokenBodyValidation, @@ -48,7 +49,7 @@ class AuthController implements IAuthController { registerUser = catchAsync(async (req: Request, res: Response) => { const newUser = req.body.user; - const newUserToken = req.body.token; + const newUserToken = registerInviteTokenValidation.parse(req.body.token); if (newUser?.email) { const newUserEmail = requireUserEmail(newUser.email); newUser.email = newUserEmail.toLowerCase(); diff --git a/server/src/api/validation/authValidation.ts b/server/src/api/validation/authValidation.ts index be30272e6..9a3162d91 100644 --- a/server/src/api/validation/authValidation.ts +++ b/server/src/api/validation/authValidation.ts @@ -23,6 +23,8 @@ export const registrationBodyValidation = z.object({ inviteToken: z.string().optional().default(""), }); +export const registerInviteTokenValidation = z.string().optional().default(""); + export const recoveryValidation = z.object({ email: z.email("Must be a valid email address"), }); diff --git a/server/test/unit/validation/authValidation.test.ts b/server/test/unit/validation/authValidation.test.ts new file mode 100644 index 000000000..89cdf60ac --- /dev/null +++ b/server/test/unit/validation/authValidation.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "@jest/globals"; +import { registerInviteTokenValidation } from "../../../src/api/validation/authValidation.ts"; + +describe("authValidation", () => { + describe("registerInviteTokenValidation", () => { + it("accepts a valid token string", () => { + expect(registerInviteTokenValidation.parse("a".repeat(64))).toBe("a".repeat(64)); + }); + + it("defaults an absent token to an empty string", () => { + expect(registerInviteTokenValidation.parse(undefined)).toBe(""); + }); + + it("rejects a NoSQL operator object in place of a token", () => { + expect(() => registerInviteTokenValidation.parse({ $ne: null })).toThrow(); + }); + + it("rejects an array token", () => { + expect(() => registerInviteTokenValidation.parse(["a", "b"])).toThrow(); + }); + }); +}); From cebe9350f7729facd4076b1b56a37c3938c8a88b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Sat, 12 Sep 2026 14:07:31 -0700 Subject: [PATCH 21/35] update readme --- README.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 19205f853..89ed11279 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ JWT_SECRET="$(openssl rand -hex 32)" docker compose up -d Then open http://localhost:52345. If the app is reached at another origin (domain or LAN IP), set `CLIENT_HOST` accordingly. To build the image yourself, run `docker build -f docker/Dockerfile -t checkmate .` from a checkout. For TLS, put any reverse proxy (Caddy, Traefik, nginx) in front of port 52345. There are also 1-click installation options like [Repocloud](https://repocloud.io/details/Checkmate), -[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [K8s](../../charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). +[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). Note that the Helm chart has not yet been migrated to the all-in-one image: it still deploys the legacy `checkmate-client`, `checkmate-backend` and `checkmate-mongo` images, pinned at v3.8.1. ### Configuration @@ -103,9 +103,16 @@ The image is configured entirely through environment variables on the server con |---|---|---| | `DB_CONNECTION_STRING` | Yes | MongoDB connection string, e.g. `mongodb://mongodb:27017/uptime_db` | | `JWT_SECRET` | Yes | Secret used to sign auth tokens; generate one with `openssl rand -hex 32` | -| `ENCRYPTION_KEY` | No | Encrypts stored Docker TLS client keys at rest; generate one with `openssl rand -base64 32`. Comma-separated list: the first key encrypts, every key decrypts. Must be identical on the API and every worker. To rotate without downtime, deploy `OLD_KEY,NEW_KEY` everywhere, then `NEW_KEY,OLD_KEY` everywhere, wait for the worker to re-encrypt every row, then drop `OLD_KEY`. | | `CLIENT_HOST` | Yes | The URL users reach the app at, e.g. `https://checkmate.example.com`; used for CORS and for links in notifications and emails | +| `ENCRYPTION_KEY` | No | Encrypts stored Docker TLS client keys at rest; generate one with `openssl rand -base64 32`. Comma-separated list: the first key encrypts, every key decrypts. Must be identical on the API and every worker. To rotate without downtime, deploy `OLD_KEY,NEW_KEY` everywhere, then `NEW_KEY,OLD_KEY` everywhere, wait for the worker to re-encrypt every row, then drop `OLD_KEY`. | +| `PORT` | No | Port the API and web client are served on (default `52345`) | +| `HEALTH_PORT` | No | Port for the `/livez`, `/readyz` and `/metrics` endpoints, served by any process that runs the job worker (default `52346`) | +| `NODE_ENV` | No | `development`, `production` or `test` (default `development`). `development` disables the general API rate limiter; set `production` on real deployments | | `LOG_LEVEL` | No | Server log level: `error`, `warn`, `info`, or `debug` (default `debug`) | +| `TOKEN_TTL` | No | Lifetime of issued auth tokens, e.g. `12h` or `7d` (default `99d`) | +| `QUEUE_MODE` | No | `primary` (default) runs the API, the web client and the job scheduler; `worker` runs a job-processing worker only, with no API | +| `QUEUE_PRIMARY_PROCESSES` | No | `true` (default) or `false`. Whether a `primary` node also processes monitoring jobs itself; set `false` when dedicated `worker` nodes handle all checks. Ignored in `worker` mode | +| `STATUS_PAGE_THEMES_ENABLED` | No | `true` (default) or `false`. When `false`, status pages ignore theme settings and always render the default theme | The web client needs no configuration by default: it calls the API on the same origin it was served from (`/api/v1`). For setups where the defaults don't apply — for example, the API is reached through a different origin than the page — the server renders overrides into the client at runtime via these optional variables: @@ -119,7 +126,7 @@ The web client needs no configuration by default: it calls the API on the same o See full installation instructions in the [Checkmate documentation portal](https://checkmate.so/docs). -Alternatively, you can also use [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) (South Africa), [Cloudzy](https://cloudzy.com/marketplace/checkmate) or [Pikapods](https://www.pikapods.com/) to quickly spin off a Checkmate instance. If you would like to monitor your server infrastructure, you'll need [Capture agent](https://github.com/bluewave-labs/capture). Capture repository also contains the installation instructions. +Alternatively, you can also use [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md) (legacy images, pinned at v3.8.1), [Sive Host](https://sive.host) (South Africa), [Cloudzy](https://cloudzy.com/marketplace/checkmate) or [Pikapods](https://www.pikapods.com/) to quickly spin off a Checkmate instance. If you would like to monitor your server infrastructure, you'll need [Capture agent](https://github.com/bluewave-labs/capture). Capture repository also contains the installation instructions. ### Using a Custom CA @@ -134,7 +141,7 @@ Thanks to extensive optimizations, Checkmate operates with an exceptionally smal ![image](https://github.com/user-attachments/assets/37e04a75-d83a-488f-b25c-025511b492c9) -You can see the memory footprint of MongoDB and Redis on the same server (398Mb and 15Mb) for the same amount of servers: +You can see the memory footprint of MongoDB on the same server (398Mb) for the same amount of servers: ![image](https://github.com/user-attachments/assets/3b469e85-e675-4040-a162-3f24c1afc751) @@ -152,16 +159,16 @@ Feel free to ask questions or share your ideas - we'd love to hear from you! ## Features - Completely open source, deployable on your servers or home devices (e.g Raspberry Pi 4 or 5) -- Several monitoring options: Uptime, Docker, Ping, SSL, Port, Game server +- Several monitoring options: HTTP (with SSL certificate expiry), Ping, Port, DNS, Docker, gRPC, WebSocket, Game server - Page speed monitoring - Infrastructure monitoring (memory, disk usage, CPU performance, network etc) - requires [Capture](https://github.com/bluewave-labs/capture) agent - Selective disk monitoring with mountpoint selection - Incidents at a glance -- Status pages with 4 beautiful themes -- E-mail, Webhooks, Discord, Slack, PagerDuty, Matrix, Rocket.Chat, Microsoft Teams, Telegram, Pushover, Twilio (SMS) notifications +- Status pages with 5 beautiful themes +- E-mail, Webhooks, Discord, Slack, PagerDuty, Matrix, Rocket.Chat, Microsoft Teams, Telegram, Pushover, ntfy, SignalGrid, Twilio (SMS) notifications - Scheduled maintenance - JSON query monitoring -- Multi-language support for Arabic, Chinese (Simplified), Chinese (Traditional, Taiwan), Czech, English, Finnish, French, German, Japanese, Portuguese (Brazil), Russian, Spanish, Thai, Turkish, Ukrainian, and Vietnamese +- Multi-language support for Arabic, Catalan, Chinese (Simplified), Chinese (Traditional, Taiwan), Czech, English, Finnish, French, German, Italian, Japanese, Polish, Portuguese (Brazil), Russian, Spanish, Thai, Turkish, Ukrainian, and Vietnamese ## Monitor Lifecycle @@ -214,7 +221,7 @@ Feel free to ask questions or share your ideas - we'd love to hear from you! We are [Alex](http://github.com/ajhollid) (team lead), [Gorkem](http://github.com/gorkem-bwl/), [Aryaman](https://github.com/Br0wnHammer), [Malena](https://github.com/malenacaroline) and [Mert](https://github.com/mertssmnoglu) helping individuals and businesses monitor their infra and servers. -We pride ourselves on building strong connections with contributors at every level. Despite being a young project, Checkmate has already earned almost 11K+ stars and attracted 150+ contributors from around the globe. +We pride ourselves on building strong connections with contributors at every level. Despite being a young project, Checkmate has already earned almost 11K stars and attracted 150+ contributors from around the globe. Our repo is starred by employees from **Google, Microsoft, Intel, Cisco, Tencent, Electronic Arts, ByteDance, JP Morgan Chase, Deloitte, Accenture, Foxconn, Broadcom, China Telecom, Barclays, Capgemini, Wipro, Cloudflare, Dassault Systèmes and NEC**, so don’t hold back — jump in, contribute and learn with us! From 581ec9382b38d6193fa8946141633e1281c8aabb Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Sat, 12 Sep 2026 14:16:14 -0700 Subject: [PATCH 22/35] docker monitor instructions --- README.md | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/README.md b/README.md index 89ed11279..1b5b74dc9 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,113 @@ Alternatively, you can also use [Coolify](https://coolify.io/), [Elestio](https: If you need to monitor internal HTTPS endpoints with certificates from private Certificate Authorities (like Smallstep), see our [Custom CA Trust Guide](./docs/custom-ca-trust.md) for Docker configuration options. +### Docker monitors + +A Docker monitor connects to a Docker daemon and reports on every container it runs. The daemon's ping response decides whether the monitor is up or down and its latency is the response time. Each check also records every container's state, health, CPU and memory usage, restart count, published ports and mounts. Enabling **Collect container logs** additionally stores the latest 200 log lines per container on every check; logs are kept for 7 days. + +The **Docker host** field accepts two forms: + +| Host | Example | Notes | +|---|---|---| +| Local socket | `unix:///var/run/docker.sock` | Also accepts a bare absolute path such as `/var/run/docker.sock`. Use this for the daemon on the same machine Checkmate runs on. | +| Remote daemon | `tcp://docker.example.com:2376` | Always uses mutual TLS; the port defaults to `2376`. Unencrypted daemons on `2375` are not supported. | + +**Monitoring the local socket.** The reference Compose file does not mount the socket, so add it and grant the container the host's `docker` group. The image runs as an unprivileged user and cannot read the socket otherwise. Find the group id with `stat -c %g /var/run/docker.sock`, then: + +```yaml +services: + checkmate: + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + group_add: + - "989" # the gid printed by stat +``` + +**Monitoring a remote daemon.** Set the host to `tcp://host:port` and fill in the **TLS credentials** section of the monitor form with the same PEM files you would pass to `docker --tlsverify`: + +- **CA certificate**: the CA that signed the daemon's server certificate. Required unless **Ignore TLS/SSL errors** is on, which skips verifying the daemon's identity. +- **Client certificate**: the certificate the daemon uses to authenticate Checkmate. +- **Client key**: the matching private key, unencrypted (no passphrase). Checkmate checks that it matches the certificate, encrypts it with `ENCRYPTION_KEY`, and never shows it again. Leave the field blank when editing to keep the stored key. + +TLS Docker monitors require `ENCRYPTION_KEY` to be set on the server (see [Configuration](#configuration)). Saving one without it fails with an error telling you so. If the key is ever removed or rotated incorrectly, affected checks fail with a decryption error until it is restored. + +If the daemon's certificate is signed by a private CA that you also want the rest of Checkmate to trust, see the [Custom CA Trust Guide](./docs/custom-ca-trust.md); for Docker monitors alone, the **CA certificate** field is enough. + +### Securing the Docker daemon with TLS + +The Docker daemon does not enable TLS by default. Follow these steps on the Docker host to generate a CA, a server certificate, and a client certificate for Checkmate. This is the procedure from [Docker's own guide](https://docs.docker.com/engine/security/protect-access/), condensed. Replace `docker.example.com` and `203.0.113.10` with your daemon's DNS name and IP. + +**1. Create a CA.** The CA key gets a passphrase; keep it offline once the certificates are issued. + +```bash +openssl genrsa -aes256 -out ca-key.pem 4096 +openssl req -new -x509 -days 365 -key ca-key.pem -sha256 -subj "/CN=docker-ca" -out ca.pem +``` + +**2. Create the server certificate.** The subject alternative names must cover every name or address Checkmate will use to reach the daemon. + +```bash +openssl genrsa -out server-key.pem 4096 +openssl req -subj "/CN=docker.example.com" -sha256 -new -key server-key.pem -out server.csr +cat > server-ext.cnf < client-ext.cnf +openssl x509 -req -days 365 -sha256 -in client.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial \ + -out cert.pem -extfile client-ext.cnf +rm client.csr server.csr server-ext.cnf client-ext.cnf +chmod 0400 ca-key.pem key.pem server-key.pem +chmod 0444 ca.pem server-cert.pem cert.pem +``` + +**4. Point the daemon at the certificates.** Move `ca.pem`, `server-cert.pem` and `server-key.pem` to `/etc/docker/certs/` and configure `/etc/docker/daemon.json`: + +```json +{ + "hosts": ["unix:///var/run/docker.sock", "tcp://0.0.0.0:2376"], + "tls": true, + "tlsverify": true, + "tlscacert": "/etc/docker/certs/ca.pem", + "tlscert": "/etc/docker/certs/server-cert.pem", + "tlskey": "/etc/docker/certs/server-key.pem" +} +``` + +On distributions where the systemd unit already passes `-H fd://` (Debian, Ubuntu and derivatives), the daemon refuses to start with `hosts` set in both places. Remove the flag from the unit with an override, then restart: + +```bash +sudo systemctl edit docker.service +``` + +```ini +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd +``` + +```bash +sudo systemctl daemon-reload && sudo systemctl restart docker +``` + +Open port `2376` on the host firewall only to the machine running Checkmate. + +**5. Verify from the Checkmate host**, then paste `ca.pem`, `cert.pem` and `key.pem` into the monitor form: + +```bash +docker --tlsverify --tlscacert=ca.pem --tlscert=cert.pem --tlskey=key.pem \ + -H=docker.example.com:2376 version +``` + For more documentation, see the [docs directory](./docs/). From b31ffa5318fa8b8951dbbe7504eb930d2fda5538 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Sat, 12 Sep 2026 14:46:05 -0700 Subject: [PATCH 23/35] reoptimize svg --- client/src/assets/Images/sushi_404.svg | 1454 +----------------------- 1 file changed, 4 insertions(+), 1450 deletions(-) diff --git a/client/src/assets/Images/sushi_404.svg b/client/src/assets/Images/sushi_404.svg index ae99ac3eb..3770b5f1c 100644 --- a/client/src/assets/Images/sushi_404.svg +++ b/client/src/assets/Images/sushi_404.svg @@ -1,1451 +1,5 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Design by Annie Hsu - - - - - - + + + \ No newline at end of file From 9ad20daa517464ab0cabfe6191a0ff8fb8a16c91 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Sat, 12 Sep 2026 14:56:07 -0700 Subject: [PATCH 24/35] bump minor version, fix unused import --- client/package-lock.json | 4 +- client/package.json | 2 +- server/openapi.json | 406 +++++++++++++++++++++++++-- server/src/config/services.worker.ts | 2 +- 4 files changed, 389 insertions(+), 25 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index f6adb320d..ef226f507 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1,12 +1,12 @@ { "name": "client", - "version": "3.11.0", + "version": "3.12.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "client", - "version": "3.11.0", + "version": "3.12.0", "dependencies": { "@csstools/css-tokenizer": "^3.0.4", "@emotion/react": "^11.13.3", diff --git a/client/package.json b/client/package.json index 16f06aced..a9e53f93c 100644 --- a/client/package.json +++ b/client/package.json @@ -1,7 +1,7 @@ { "name": "client", "private": true, - "version": "3.11.0", + "version": "3.12.0", "type": "module", "scripts": { "lockfile": "docker run --rm -v \"$PWD\":/w -w /w $(grep -m1 -o 'node:[^ ]*' ../docker/Dockerfile) npm install --package-lock-only", diff --git a/server/openapi.json b/server/openapi.json index cf058ec4c..4b4faf896 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -422,9 +422,6 @@ "secret": { "type": "string" }, - "sshPrivateKey": { - "type": "string" - }, "cpuAlertThreshold": { "type": "number" }, @@ -466,6 +463,18 @@ "geoCheckInterval": { "type": "number" }, + "dockerLogsEnabled": { + "type": "boolean" + }, + "dockerTlsCa": { + "type": "string" + }, + "dockerTlsCert": { + "type": "string" + }, + "dockerTlsKeySet": { + "type": "boolean" + }, "dnsServer": { "type": "string" }, @@ -513,6 +522,7 @@ "geoCheckEnabled", "geoCheckLocations", "geoCheckInterval", + "dockerLogsEnabled", "teamId", "userId", "createdAt", @@ -640,9 +650,6 @@ "secret": { "type": "string" }, - "sshPrivateKey": { - "type": "string" - }, "cpuAlertThreshold": { "type": "number" }, @@ -684,6 +691,18 @@ "geoCheckInterval": { "type": "number" }, + "dockerLogsEnabled": { + "type": "boolean" + }, + "dockerTlsCa": { + "type": "string" + }, + "dockerTlsCert": { + "type": "string" + }, + "dockerTlsKeySet": { + "type": "boolean" + }, "dnsServer": { "type": "string" }, @@ -733,6 +752,7 @@ "geoCheckEnabled", "geoCheckLocations", "geoCheckInterval", + "dockerLogsEnabled", "teamId", "userId", "createdAt", @@ -929,6 +949,9 @@ { "$ref": "#/components/schemas/PushoverNotification" }, + { + "$ref": "#/components/schemas/SignalgridNotification" + }, { "$ref": "#/components/schemas/TwilioNotification" }, @@ -949,6 +972,7 @@ "teams": "#/components/schemas/TeamsNotification", "telegram": "#/components/schemas/TelegramNotification", "pushover": "#/components/schemas/PushoverNotification", + "signalgrid": "#/components/schemas/SignalgridNotification", "twilio": "#/components/schemas/TwilioNotification", "ntfy": "#/components/schemas/NtfyNotification" } @@ -1380,6 +1404,34 @@ "accessToken": "a1234567890abcdef" } }, + "SignalgridNotification": { + "type": "object", + "properties": { + "notificationName": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "enum": ["signalgrid"] + }, + "address": { + "type": "string", + "minLength": 1 + }, + "accessToken": { + "type": "string", + "minLength": 1 + } + }, + "required": ["notificationName", "type", "address", "accessToken"], + "example": { + "notificationName": "Signalgrid", + "type": "signalgrid", + "address": "your-channel", + "accessToken": "your-client-key" + } + }, "TwilioNotification": { "type": "object", "properties": { @@ -7090,9 +7142,6 @@ "secret": { "type": "string" }, - "sshPrivateKey": { - "type": "string" - }, "cpuAlertThreshold": { "type": "number" }, @@ -7134,6 +7183,18 @@ "geoCheckInterval": { "type": "number" }, + "dockerLogsEnabled": { + "type": "boolean" + }, + "dockerTlsCa": { + "type": "string" + }, + "dockerTlsCert": { + "type": "string" + }, + "dockerTlsKeySet": { + "type": "boolean" + }, "dnsServer": { "type": "string" }, @@ -7183,6 +7244,7 @@ "geoCheckEnabled", "geoCheckLocations", "geoCheckInterval", + "dockerLogsEnabled", "teamId", "userId", "createdAt", @@ -7655,9 +7717,6 @@ "secret": { "type": "string" }, - "sshPrivateKey": { - "type": "string" - }, "cpuAlertThreshold": { "type": "number" }, @@ -7699,6 +7758,18 @@ "geoCheckInterval": { "type": "number" }, + "dockerLogsEnabled": { + "type": "boolean" + }, + "dockerTlsCa": { + "type": "string" + }, + "dockerTlsCert": { + "type": "string" + }, + "dockerTlsKeySet": { + "type": "boolean" + }, "dnsServer": { "type": "string" }, @@ -7748,6 +7819,7 @@ "geoCheckEnabled", "geoCheckLocations", "geoCheckInterval", + "dockerLogsEnabled", "teamId", "userId", "createdAt", @@ -7984,6 +8056,231 @@ } } }, + "/monitors/docker/details/{monitorId}/containers/{containerName}/logs": { + "get": { + "tags": ["monitors"], + "summary": "Get stored log lines for one container on a Docker host monitor, newest first", + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true, + "name": "monitorId", + "in": "path" + }, + { + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true, + "name": "containerName", + "in": "path" + }, + { + "schema": { + "type": "string", + "format": "date-time" + }, + "required": false, + "name": "before", + "in": "query" + }, + { + "schema": { + "type": "string", + "format": "date-time" + }, + "required": false, + "name": "after", + "in": "query" + }, + { + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 50, + "default": 20 + }, + "required": false, + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "enum": [true] + }, + "msg": { + "type": "string" + }, + "data": { + "type": "object", + "properties": { + "logs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "metadata": { + "type": "object", + "properties": { + "monitorId": { + "type": "string" + }, + "teamId": { + "type": "string" + }, + "containerId": { + "type": "string" + }, + "containerName": { + "type": "string" + } + }, + "required": ["monitorId", "teamId", "containerId", "containerName"] + }, + "lines": { + "type": "array", + "items": { + "type": "object", + "properties": { + "ts": { + "type": "string" + }, + "stream": { + "type": "string", + "enum": ["stdout", "stderr"] + }, + "text": { + "type": "string" + } + }, + "required": ["ts", "stream", "text"] + } + }, + "gap": { + "type": "boolean" + }, + "checkedAt": { + "type": "string" + }, + "expiry": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "updatedAt": { + "type": "string" + } + }, + "required": ["id", "metadata", "lines", "gap", "checkedAt", "expiry", "createdAt", "updatedAt"] + } + }, + "nextCursor": { + "type": "string", + "nullable": true + } + }, + "required": ["logs", "nextCursor"] + } + }, + "required": ["success", "msg", "data"] + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "enum": [false] + }, + "msg": { + "type": "string" + }, + "data": { + "nullable": true + } + }, + "required": ["success", "msg"] + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "enum": [false] + }, + "msg": { + "type": "string" + }, + "data": { + "nullable": true + } + }, + "required": ["success", "msg"] + } + } + } + }, + "500": { + "description": "Internal server error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "enum": [false] + }, + "msg": { + "type": "string" + }, + "data": { + "nullable": true + } + }, + "required": ["success", "msg"] + } + } + } + } + } + } + }, "/monitors/pagespeed/details/{monitorId}": { "get": { "tags": ["monitors"], @@ -8908,9 +9205,6 @@ "secret": { "type": "string" }, - "sshPrivateKey": { - "type": "string" - }, "jsonPath": { "anyOf": [ { @@ -9011,6 +9305,42 @@ "type": "number", "minimum": 300000 }, + "dockerLogsEnabled": { + "type": "boolean" + }, + "dockerTlsCa": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [""] + } + ] + }, + "dockerTlsCert": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [""] + } + ] + }, + "dockerTlsKey": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [""] + } + ] + }, "dnsServer": { "type": "string", "minLength": 1 @@ -9604,9 +9934,6 @@ "secret": { "type": "string" }, - "sshPrivateKey": { - "type": "string" - }, "cpuAlertThreshold": { "type": "number", "default": 100 @@ -9702,6 +10029,10 @@ "minimum": 300000, "default": 300000 }, + "dockerLogsEnabled": { + "type": "boolean", + "default": false + }, "dnsServer": { "type": "string", "minLength": 1 @@ -10137,9 +10468,6 @@ "secret": { "type": "string" }, - "sshPrivateKey": { - "type": "string" - }, "ignoreTlsErrors": { "type": "boolean" }, @@ -10269,6 +10597,42 @@ "type": "number", "minimum": 300000 }, + "dockerLogsEnabled": { + "type": "boolean" + }, + "dockerTlsCa": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [""] + } + ] + }, + "dockerTlsCert": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [""] + } + ] + }, + "dockerTlsKey": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [""] + } + ] + }, "dnsServer": { "type": "string", "minLength": 1 diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index 8b2f4de07..acd4a944a 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -16,7 +16,7 @@ import { SharedServices } from "@/config/services.shared.js"; import { INetworkService, NetworkService } from "@/service/networkService.js"; import { IBufferService, BufferService } from "@/service/bufferService.js"; import { IStatusService, StatusService } from "@/service/statusService.js"; -import { ISecretsRotationService, SecretsRotationService } from "@/service/encryption/secretsRotationService.js"; +import { SecretsRotationService } from "@/service/encryption/secretsRotationService.js"; import { IQueueWorker } from "@/worker/worker.interface.js"; import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js"; import { WorkerHelper } from "@/worker/worker.helper.js"; From 4bb366f5abbbcb1e9b553770e9d12de06b1508a6 Mon Sep 17 00:00:00 2001 From: fabiangigler Date: Wed, 26 Aug 2026 15:26:52 +0200 Subject: [PATCH 25/35] Add custom monitor interval options --- .../inputs/forms/FormAutoCompleteField.tsx | 9 ++- client/src/Pages/CreateMonitor/index.tsx | 63 +++++++++++++++---- client/src/Types/Monitor.ts | 2 + client/src/Validation/monitor.ts | 4 +- .../test/unit/services/jobScheduler.test.ts | 15 +++++ 5 files changed, 80 insertions(+), 13 deletions(-) diff --git a/client/src/Components/inputs/forms/FormAutoCompleteField.tsx b/client/src/Components/inputs/forms/FormAutoCompleteField.tsx index 1dbc65118..6c1a4fd94 100644 --- a/client/src/Components/inputs/forms/FormAutoCompleteField.tsx +++ b/client/src/Components/inputs/forms/FormAutoCompleteField.tsx @@ -5,9 +5,10 @@ import { type FieldValues, } from "react-hook-form"; import { Autocomplete } from "@/Components/inputs"; +import type { AutocompleteProps } from "@mui/material/Autocomplete"; export interface AutocompleteOption { - id: string; + id: string | number; name: string; } @@ -16,6 +17,8 @@ interface FormAutocompleteProps["filterOptions"]; + getOptionDisabled?: AutocompleteProps["getOptionDisabled"]; } export const FormAutocompleteField = < @@ -26,6 +29,8 @@ export const FormAutocompleteField = < options, fieldLabel, placeholder, + filterOptions, + getOptionDisabled, }: FormAutocompleteProps) => { const { control } = useFormContext(); return ( @@ -43,6 +48,8 @@ export const FormAutocompleteField = < }} fieldLabel={fieldLabel} placeholder={placeholder} + filterOptions={filterOptions} + getOptionDisabled={getOptionDisabled} error={!!fieldState.error} helperText={fieldState.error?.message ?? ""} /> diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 3291ab090..380329c4b 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -38,6 +38,7 @@ import { PageSpeedStrategies, DnsRecordTypes, MonitorIntervalOptions, + MIN_MONITOR_INTERVAL_MS, DefaultMonitorMatchMethod, MonitorMatchMethods, GeoCheckIntervalOptions, @@ -58,9 +59,14 @@ import { FormTextField } from "@/Components/inputs/forms/FormTextField"; import { FormRadioGroup } from "@/Components/inputs/forms/FormRadioGroupField"; import { FormMultiSelectField } from "@/Components/inputs/forms/FormMultiSelectField"; import { FormSelectField } from "@/Components/inputs/forms/FormSelectField"; +import { + FormAutocompleteField, + type AutocompleteOption, +} from "@/Components/inputs/forms/FormAutoCompleteField"; import { FormSliderField } from "@/Components/inputs/forms/FormSliderField"; import { FormSwitchField } from "@/Components/inputs/forms/FormSwitchField"; import type { ProxyResponse } from "@/Types/Proxy"; +import { formatDuration } from "@/Utils/TimeUtils"; const httpMethodOptions = HttpMethods.map((method) => ({ value: method, @@ -75,6 +81,36 @@ const maskedInputSx = { "&:not(.Mui-focused) textarea": { WebkitTextSecurity: "disc" }, }; +interface IntervalOption extends AutocompleteOption { + id: number; +} + +const INTERVAL_UNIT_MULTIPLIERS = [1000, 60000, 3600000, 86400000] as const; + +const toIntervalOption = (intervalMs: number): IntervalOption => ({ + id: intervalMs, + name: formatDuration(intervalMs, true), +}); + +const filterIntervalOptions = ( + options: IntervalOption[], + inputValue: string +): IntervalOption[] => { + const input = inputValue.trim(); + if (input === "" || options.some((option) => option.name === input)) { + return options; + } + + const amount = Number(input); + if (!Number.isFinite(amount) || amount <= 0) { + return []; + } + + return INTERVAL_UNIT_MULTIPLIERS.map((multiplier) => amount * multiplier) + .filter(Number.isSafeInteger) + .map(toIntervalOption); +}; + interface GeneralSettingsConfig { urlLabel: string; urlPlaceholder: string; @@ -314,6 +350,7 @@ const CreateMonitorPage = () => { const watchedProxyMode = watch("proxyMode") as ProxyMode | undefined; const watchedUseAdvancedMatching = watch("useAdvancedMatching") as boolean; const watchGeoCheckEnabled = watch("geoCheckEnabled") as boolean; + const watchedInterval = watch("interval"); // Steps without an advanced section drop it, so the last step is the form's. const totalSteps = monitorStepCount(watchedType); @@ -471,16 +508,16 @@ const CreateMonitorPage = () => { [t] ); - const intervalOptions = useMemo( - () => - MonitorIntervalOptions.map((option) => ({ - value: option.value, - label: t( - `pages.createMonitor.form.frequency.option.frequency.value.${option.labelKey}` - ), - })), - [t] - ); + const intervalOptions = useMemo(() => { + const options = MonitorIntervalOptions.map(({ value }) => toIntervalOption(value)); + if ( + typeof watchedInterval === "number" && + !options.some((option) => option.id === watchedInterval) + ) { + options.push(toIntervalOption(watchedInterval)); + } + return options; + }, [watchedInterval]); const matchMethodOptions = useMemo( () => @@ -764,12 +801,16 @@ const CreateMonitorPage = () => { title={t("pages.createMonitor.form.frequency.title")} subtitle={t("pages.createMonitor.form.frequency.description")} rightContent={ - + filterIntervalOptions(options, inputValue) + } + getOptionDisabled={(option) => option.id < MIN_MONITOR_INTERVAL_MS} /> } /> diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index 858c827d5..892e2b549 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -60,6 +60,8 @@ export const monitorTypeLabelKey: Record & pagespeed: "optionPagespeed", }; +export const MIN_MONITOR_INTERVAL_MS = 10000; + export const MonitorIntervalOptions = [ { value: 15000, labelKey: "fifteenSeconds" }, { value: 30000, labelKey: "thirtySeconds" }, diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 8a2b6e726..1637d5297 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -6,6 +6,7 @@ import { HttpMethods, PageSpeedStrategies, ProxyModes, + MIN_MONITOR_INTERVAL_MS, type MonitorType, } from "@/Types/Monitor"; import { ALL_HTTP_STATUS_CODES } from "@/Utils/statusCode"; @@ -27,7 +28,8 @@ const baseSchema = z.object({ description: z.string().optional(), interval: z .number() - .min(15000, "Interval must be at least 15 seconds") + .int("Interval must be a whole number of milliseconds") + .min(MIN_MONITOR_INTERVAL_MS, "Interval must be at least 10 seconds") .register(monitorStepRegistry, { step: 1 }), notifications: z.array(z.string()).register(monitorStepRegistry, { step: 1 }), tags: z.array(z.string()).register(monitorStepRegistry, { step: 1 }), diff --git a/server/test/unit/services/jobScheduler.test.ts b/server/test/unit/services/jobScheduler.test.ts index 90c8c2850..c4a162dd6 100644 --- a/server/test/unit/services/jobScheduler.test.ts +++ b/server/test/unit/services/jobScheduler.test.ts @@ -143,6 +143,21 @@ describe("JobScheduler", () => { }); }); + describe("addJob", () => { + it("preserves an arbitrary monitor interval in the scheduled job", async () => { + const { scheduler, mocks } = createScheduler(); + + await scheduler.addJob("m1", makeMonitor({ interval: 75000 })); + + expect(mocks.jobsRepository.upsertJob).toHaveBeenCalledWith( + expect.objectContaining({ + id: "check:m1", + intervalMs: 75000, + }) + ); + }); + }); + // Regression: a heartbeat upsert racing shutdown's deleteById could resurrect the // registry row, leaving a ghost worker that inflates the alive count until TTL. // shutdown() must await the in-flight beat and issue the delete as the LAST write. From 5da472da1493de3d760418a3127eac576cd863bb Mon Sep 17 00:00:00 2001 From: fabiangigler Date: Thu, 27 Aug 2026 12:16:26 +0200 Subject: [PATCH 26/35] Prevent clearing monitor intervals --- client/src/Components/inputs/forms/FormAutoCompleteField.tsx | 4 ++++ client/src/Pages/CreateMonitor/index.tsx | 1 + 2 files changed, 5 insertions(+) diff --git a/client/src/Components/inputs/forms/FormAutoCompleteField.tsx b/client/src/Components/inputs/forms/FormAutoCompleteField.tsx index 6c1a4fd94..3af48efb3 100644 --- a/client/src/Components/inputs/forms/FormAutoCompleteField.tsx +++ b/client/src/Components/inputs/forms/FormAutoCompleteField.tsx @@ -19,6 +19,7 @@ interface FormAutocompleteProps["filterOptions"]; getOptionDisabled?: AutocompleteProps["getOptionDisabled"]; + disableClearable?: boolean; } export const FormAutocompleteField = < @@ -31,6 +32,7 @@ export const FormAutocompleteField = < placeholder, filterOptions, getOptionDisabled, + disableClearable, }: FormAutocompleteProps) => { const { control } = useFormContext(); return ( @@ -44,12 +46,14 @@ export const FormAutocompleteField = < isOptionEqualToValue={(option, value) => option.id === value.id} value={options.find((o) => o.id === field.value) ?? null} onChange={(_: unknown, newValue: O | null) => { + if (newValue === null && disableClearable) return; field.onChange(newValue?.id ?? ""); }} fieldLabel={fieldLabel} placeholder={placeholder} filterOptions={filterOptions} getOptionDisabled={getOptionDisabled} + disableClearable={disableClearable} error={!!fieldState.error} helperText={fieldState.error?.message ?? ""} /> diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 380329c4b..921f3deb3 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -811,6 +811,7 @@ const CreateMonitorPage = () => { filterIntervalOptions(options, inputValue) } getOptionDisabled={(option) => option.id < MIN_MONITOR_INTERVAL_MS} + disableClearable /> } /> From ec01a5dfe4791992f7178238b0515a3fd62a77cf Mon Sep 17 00:00:00 2001 From: Craig Lawson Date: Mon, 14 Sep 2026 10:26:09 +0100 Subject: [PATCH 27/35] Add egress state and probe service (#3937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the EgressService that probes a configurable set of reliability targets (ping, TCP or HTTP) when asked, persists a single instance-wide egress state document, and runs a recovery poll loop while egress is degraded. Adds the four egress settings fields with validation. The worker constructs and initialises the service but nothing calls assessAfterFailure yet, so behaviour is unchanged. 🔧 --- server/openapi.json | 25 ++ .../src/api/validation/settingsValidation.ts | 14 + server/src/config/services.api.ts | 5 + server/src/config/services.shared.ts | 5 + server/src/config/services.worker.ts | 9 +- .../domain/app-settings/app-settings.model.ts | 5 + .../app-settings.repository.mongo.ts | 5 + .../domain/app-settings/app-settings.type.ts | 5 + .../src/domain/egress/egress-state.model.ts | 40 ++ .../egress-state.repository.interface.ts | 12 + .../egress/egress-state.repository.mongo.ts | 67 +++ .../src/domain/egress/egress-state.service.ts | 20 + server/src/domain/egress/egress.service.ts | 290 +++++++++++++ server/src/domain/egress/egress.type.ts | 33 ++ .../test/unit/services/egressService.test.ts | 404 ++++++++++++++++++ .../unit/services/egressStateService.test.ts | 70 +++ .../validation/settingsValidation.test.ts | 72 ++++ 17 files changed, 1080 insertions(+), 1 deletion(-) create mode 100644 server/src/domain/egress/egress-state.model.ts create mode 100644 server/src/domain/egress/egress-state.repository.interface.ts create mode 100644 server/src/domain/egress/egress-state.repository.mongo.ts create mode 100644 server/src/domain/egress/egress-state.service.ts create mode 100644 server/src/domain/egress/egress.service.ts create mode 100644 server/src/domain/egress/egress.type.ts create mode 100644 server/test/unit/services/egressService.test.ts create mode 100644 server/test/unit/services/egressStateService.test.ts create mode 100644 server/test/unit/validation/settingsValidation.test.ts diff --git a/server/openapi.json b/server/openapi.json index 4b4faf896..d0ae81f26 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -13294,6 +13294,31 @@ "globalProxyId": { "type": "string", "nullable": true + }, + "egressCheckEnabled": { + "type": "boolean" + }, + "egressCheckTargets": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^(https?:\\/\\/\\S+|[A-Za-z0-9.\\-:[\\]]+)$" + }, + "maxItems": 10 + }, + "egressPollIntervalSeconds": { + "type": "integer", + "minimum": 5, + "maximum": 600 + }, + "egressNotifications": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f\\d]{24}$/i" + } } } } diff --git a/server/src/api/validation/settingsValidation.ts b/server/src/api/validation/settingsValidation.ts index 50c75b043..33fe433fc 100644 --- a/server/src/api/validation/settingsValidation.ts +++ b/server/src/api/validation/settingsValidation.ts @@ -1,6 +1,15 @@ import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js"; +import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js"; import { z } from "zod"; +// A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL. +const egressTargetValidation = z + .string() + .trim() + .min(1) + .max(253) + .regex(/^(https?:\/\/\S+|[A-Za-z0-9.\-:[\]]+)$/, "Invalid egress target"); + //**************************************** // Settings Validations //**************************************** @@ -42,6 +51,11 @@ export const updateAppSettingsBodyValidation = z .optional(), globalProxyEnabled: z.boolean().optional(), globalProxyId: z.string().nullable().optional(), + + egressCheckEnabled: z.boolean().optional(), + egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS + egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(), + egressNotifications: z.array(z.string().regex(/^[a-f\d]{24}$/i, "Invalid notification id")).optional(), }) .strip() .superRefine((body, ctx) => { diff --git a/server/src/config/services.api.ts b/server/src/config/services.api.ts index c5f58ac19..e929fbfe8 100644 --- a/server/src/config/services.api.ts +++ b/server/src/config/services.api.ts @@ -7,6 +7,7 @@ import { ITagsService, TagsService } from "@/domain/tags/tag.service.js"; import { IUserService, UserService } from "@/domain/users/user.service.js"; import { IJobScheduler } from "@/worker/worker.interface.js"; import { ProxiesService, IProxiesService } from "@/domain/proxies/proxy.service.js"; +import { EgressStateService, IEgressStateService } from "@/domain/egress/egress-state.service.js"; import { SharedServices } from "@/config/services.shared.js"; // Third-party @@ -24,6 +25,7 @@ export interface ApiServices extends SharedServices { tagsService: ITagsService; diagnosticService: IDiagnosticService; proxiesService: IProxiesService; + egressStateService: IEgressStateService; } export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => { @@ -48,6 +50,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A maintenanceWindowsRepository, jobsRepository, proxiesRepository, + egressStateRepository, } = shared; const userService = new UserService({ @@ -100,6 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A const tagsService = new TagsService(tagsRepository, monitorsRepository); const diagnosticService = new DiagnosticService(db); const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService); + const egressStateService = new EgressStateService(egressStateRepository); return { ...shared, worker: jobScheduler, @@ -111,5 +115,6 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A tagsService, diagnosticService, proxiesService, + egressStateService, }; }; diff --git a/server/src/config/services.shared.ts b/server/src/config/services.shared.ts index 3b4fdb147..a1a69566b 100644 --- a/server/src/config/services.shared.ts +++ b/server/src/config/services.shared.ts @@ -56,6 +56,7 @@ import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; import { IUsersRepository } from "@/domain/users/user.repository.interface.js"; import { IProxiesRepository } from "@/domain/proxies/proxy.repository.interface.js"; +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; // Mongo repository implementations import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js"; @@ -76,6 +77,7 @@ import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js"; import MongoTeamsRepository from "@/domain/teams/team.repository.model.js"; import MongoUsersRepository from "@/domain/users/user.repository.mongo.js"; import MongoProxiesRepository from "@/domain/proxies/proxy.repository.mongo.js"; +import MongoEgressStateRepository from "@/domain/egress/egress-state.repository.mongo.js"; import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js"; // Shared infrastructure + business services that both the API and the worker process construct. @@ -114,6 +116,7 @@ export interface SharedServices { teamsRepository: ITeamsRepository; maintenanceWindowsRepository: IMaintenanceWindowsRepository; proxiesRepository: IProxiesRepository; + egressStateRepository: IEgressStateRepository; } export const buildShared = async ({ @@ -157,6 +160,7 @@ export const buildShared = async ({ const teamsRepository = new MongoTeamsRepository(); const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository(); const proxiesRepository = new MongoProxiesRepository(); + const egressStateRepository = new MongoEgressStateRepository(); // Inject settings repository into settings service (now that DB is connected) (settingsService as SettingsService).setRepository(settingsRepository); @@ -246,6 +250,7 @@ export const buildShared = async ({ teamsRepository, maintenanceWindowsRepository, proxiesRepository, + egressStateRepository, }; return sharedServices; }; diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index acd4a944a..83a875f05 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -28,6 +28,7 @@ import { IncidentReactor } from "@/worker/reactors/reactor.incident.js"; import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { DBQueueWorker } from "@/worker/worker.db-queue.js"; import { ProxyResolver } from "@/service/network/ProxyResolver.js"; +import { IEgressService, EgressService } from "@/domain/egress/egress.service.js"; // Network providers import { PingProvider } from "@/service/network/PingProvider.js"; import { HttpProvider } from "@/service/network/HttpProvider.js"; @@ -46,6 +47,7 @@ export interface WorkerServices { networkService: INetworkService; bufferService: IBufferService; statusService: IStatusService; + egressService: IEgressService; } export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise => { @@ -70,6 +72,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig teamsRepository, maintenanceWindowsRepository, proxiesRepository, + egressStateRepository, } = shared; // *********************** @@ -103,6 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository); const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); const monitorStatusPolicy = new MonitorStatusPolicy(); + const egressService = new EgressService(settingsService, egressStateRepository, networkService, logger); // *********************** // Reactors and dispatcher @@ -185,5 +189,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig }); } - return { worker, networkService, bufferService, statusService }; + // Resume recovery polling if the instance was mid-episode when it last stopped + await egressService.init(); + + return { worker, networkService, bufferService, statusService, egressService }; }; diff --git a/server/src/domain/app-settings/app-settings.model.ts b/server/src/domain/app-settings/app-settings.model.ts index 3dd3c3abe..dfe138b8f 100644 --- a/server/src/domain/app-settings/app-settings.model.ts +++ b/server/src/domain/app-settings/app-settings.model.ts @@ -1,5 +1,6 @@ import { Schema, model, type Types } from "mongoose"; import type { Settings, SettingsThresholds } from "@/domain/app-settings/app-settings.type.js"; +import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js"; interface AppSettingsDocument extends Omit { _id: Types.ObjectId; @@ -42,6 +43,10 @@ const AppSettingsSchema = new Schema( globalThresholds: { type: thresholdsSchema }, globalProxyEnabled: { type: Boolean, default: false }, globalProxyId: { type: String, default: null }, + egressCheckEnabled: { type: Boolean, default: false }, + egressCheckTargets: { type: [String], default: () => [...DEFAULT_EGRESS_TARGETS] }, + egressPollIntervalSeconds: { type: Number, default: DEFAULT_EGRESS_POLL_INTERVAL_SECONDS }, + egressNotifications: { type: [String], default: () => [] }, // Notification ids, stored as strings like globalProxyId }, { timestamps: true } ); diff --git a/server/src/domain/app-settings/app-settings.repository.mongo.ts b/server/src/domain/app-settings/app-settings.repository.mongo.ts index d80289a7b..f2ef3351c 100644 --- a/server/src/domain/app-settings/app-settings.repository.mongo.ts +++ b/server/src/domain/app-settings/app-settings.repository.mongo.ts @@ -3,6 +3,7 @@ import { ISettingsRepository } from "@/domain/app-settings/app-settings-reposito import type { Settings, SettingsUpdate } from "@/domain/app-settings/app-settings.type.js"; import { AppSettingsModel, type AppSettingsDocument } from "@/domain/app-settings/app-settings.model.js"; import { toStringId, toDateString } from "@/utils/mongoMappers.js"; +import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS } from "@/domain/egress/egress.type.js"; class MongoSettingsRepository implements ISettingsRepository { private toEntity = (doc: AppSettingsDocument): Settings => { @@ -30,6 +31,10 @@ class MongoSettingsRepository implements ISettingsRepository { globalThresholds: doc.globalThresholds ?? undefined, globalProxyEnabled: doc.globalProxyEnabled ?? false, globalProxyId: doc.globalProxyId ?? undefined, + egressCheckEnabled: doc.egressCheckEnabled ?? false, + egressCheckTargets: doc.egressCheckTargets ?? [...DEFAULT_EGRESS_TARGETS], + egressPollIntervalSeconds: doc.egressPollIntervalSeconds ?? DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, + egressNotifications: doc.egressNotifications ?? [], createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; diff --git a/server/src/domain/app-settings/app-settings.type.ts b/server/src/domain/app-settings/app-settings.type.ts index e4d2f9559..9dc4eae45 100644 --- a/server/src/domain/app-settings/app-settings.type.ts +++ b/server/src/domain/app-settings/app-settings.type.ts @@ -51,6 +51,11 @@ export interface Settings { globalThresholds?: SettingsThresholds; globalProxyEnabled: boolean; globalProxyId?: string | null; + // Egress self-check (see domain/egress) + egressCheckEnabled: boolean; + egressCheckTargets: string[]; + egressPollIntervalSeconds: number; // recovery poll cadence while degraded, not a general schedule + egressNotifications: string[]; // Notification ids to receive the "egress recovered" alert createdAt: string; updatedAt: string; } diff --git a/server/src/domain/egress/egress-state.model.ts b/server/src/domain/egress/egress-state.model.ts new file mode 100644 index 000000000..14f7cafe8 --- /dev/null +++ b/server/src/domain/egress/egress-state.model.ts @@ -0,0 +1,40 @@ +import { Schema, model, type Types } from "mongoose"; +import { EgressStatuses, type EgressProbeResult, type EgressState } from "@/domain/egress/egress.type.js"; + +interface EgressStateDocument extends Omit { + _id: Types.ObjectId; + singleton: boolean; + degradedSince: Date | null; + lastRecoveredAt: Date | null; + lastProbeAt: Date | null; + createdAt: Date; + updatedAt: Date; +} + +const probeResultSchema = new Schema( + { + target: { type: String, required: true }, + reachable: { type: Boolean, required: true }, + responseTime: { type: Number, default: 0 }, + message: { type: String }, + }, + { _id: false } +); + +const EgressStateSchema = new Schema( + { + singleton: { type: Boolean, required: true, unique: true, default: true }, + status: { type: String, enum: EgressStatuses, required: true, default: "ok" }, + degradedSince: { type: Date, default: null }, + lastRecoveredAt: { type: Date, default: null }, + lastProbeAt: { type: Date, default: null }, + lastProbeResults: { type: [probeResultSchema], default: () => [] }, + }, + { timestamps: true } +); + +const EgressStateModel = model("EgressState", EgressStateSchema); + +export type { EgressStateDocument }; +export { EgressStateModel }; +export default EgressStateModel; diff --git a/server/src/domain/egress/egress-state.repository.interface.ts b/server/src/domain/egress/egress-state.repository.interface.ts new file mode 100644 index 000000000..533aad656 --- /dev/null +++ b/server/src/domain/egress/egress-state.repository.interface.ts @@ -0,0 +1,12 @@ +import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js"; + +export interface IEgressStateRepository { + // read + findSingleton(): Promise; // creates the "ok" singleton on first call + // update + recordProbe(results: EgressProbeResult[], now: Date): Promise; + // Atomic transitions. Each returns the updated state when this caller performed the transition, + // or null when the state was already on the target side (another worker got there first). + markDegraded(results: EgressProbeResult[], now: Date): Promise; + markRecovered(results: EgressProbeResult[], now: Date): Promise; +} diff --git a/server/src/domain/egress/egress-state.repository.mongo.ts b/server/src/domain/egress/egress-state.repository.mongo.ts new file mode 100644 index 000000000..606551e5a --- /dev/null +++ b/server/src/domain/egress/egress-state.repository.mongo.ts @@ -0,0 +1,67 @@ +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type.js"; +import { EgressStateModel, type EgressStateDocument } from "@/domain/egress/egress-state.model.js"; +import { toStringId, toDateString } from "@/utils/mongoMappers.js"; + +const SERVICE_NAME = "EgressStateRepository"; + +class MongoEgressStateRepository implements IEgressStateRepository { + static SERVICE_NAME = SERVICE_NAME; + + private toEntity = (doc: EgressStateDocument): EgressState => { + return { + id: toStringId(doc._id), + status: doc.status, + degradedSince: doc.degradedSince ? toDateString(doc.degradedSince) : null, + lastRecoveredAt: doc.lastRecoveredAt ? toDateString(doc.lastRecoveredAt) : null, + lastProbeAt: doc.lastProbeAt ? toDateString(doc.lastProbeAt) : null, + lastProbeResults: (doc.lastProbeResults ?? []).map((result) => ({ + target: result.target, + reachable: result.reachable, + responseTime: result.responseTime ?? 0, + ...(result.message !== undefined && { message: result.message }), + })), + createdAt: toDateString(doc.createdAt), + updatedAt: toDateString(doc.updatedAt), + }; + }; + + findSingleton = async () => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $setOnInsert: { singleton: true, status: "ok" } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + }; + + recordProbe = async (results: EgressProbeResult[], now: Date) => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $set: { lastProbeAt: now, lastProbeResults: results }, $setOnInsert: { singleton: true, status: "ok" } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + }; + + markDegraded = async (results: EgressProbeResult[], now: Date) => { + // Only the caller that flips ok -> degraded sees a document back; everyone else sees null. + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true, status: "ok" }, + { $set: { status: "degraded", degradedSince: now, lastProbeAt: now, lastProbeResults: results } }, + { new: true } + ).lean(); + return doc ? this.toEntity(doc) : null; + }; + + markRecovered = async (results: EgressProbeResult[], now: Date) => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true, status: "degraded" }, + { $set: { status: "ok", lastRecoveredAt: now, lastProbeAt: now, lastProbeResults: results } }, + { new: true } + ).lean(); + return doc ? this.toEntity(doc) : null; + }; +} + +export default MongoEgressStateRepository; diff --git a/server/src/domain/egress/egress-state.service.ts b/server/src/domain/egress/egress-state.service.ts new file mode 100644 index 000000000..35800c7ac --- /dev/null +++ b/server/src/domain/egress/egress-state.service.ts @@ -0,0 +1,20 @@ +import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { EgressState } from "@/domain/egress/egress.type.js"; + +export interface IEgressStateService { + getState(): Promise; +} + +const SERVICE_NAME = "EgressStateService"; + +// Read-only view of the instance-global egress state for the API process. Transitions are +// owned by the worker (egress.service.ts); this exists so the controller never touches the repository. +export class EgressStateService implements IEgressStateService { + static SERVICE_NAME = SERVICE_NAME; + + constructor(private egressStateRepository: IEgressStateRepository) {} + + getState = async (): Promise => { + return this.egressStateRepository.findSingleton(); + }; +} diff --git a/server/src/domain/egress/egress.service.ts b/server/src/domain/egress/egress.service.ts new file mode 100644 index 000000000..cb3b79ebb --- /dev/null +++ b/server/src/domain/egress/egress.service.ts @@ -0,0 +1,290 @@ +import type { Monitor } from "@/domain/monitors/monitor.type.js"; +import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; +import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { INetworkService } from "@/service/networkService.js"; +import type { ILogger } from "@/utils/logger.js"; +import { + DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, + DEFAULT_EGRESS_TARGETS, + type EgressProbeResult, + type EgressStatus, +} from "@/domain/egress/egress.type.js"; +import { timeRequest } from "@/service/network/utils.js"; + +const SERVICE_NAME = "EgressService"; +const PROBE_TIMEOUT_MS = 5000; + +// Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs. +const PROBE_MONITOR_ID = "egress-probe"; +const PROBE_TEAM_ID = "system"; + +// "[host]:port" or "host:port" where host has no colons (so a bare IPv6 address is not mistaken for host:port). +const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/; + +export interface IEgressService { + probeTargets(targets: string[]): Promise; + assessAfterFailure(): Promise; + startRecoveryPolling(): void; + stop(): void; + init(): Promise; +} + +export class EgressService implements IEgressService { + static SERVICE_NAME = SERVICE_NAME; + + private polling = false; + private pollTimer: NodeJS.Timeout | null = null; + + constructor( + private settingsService: ISettingsService, + private egressStateRepository: IEgressStateRepository, + private networkService: INetworkService, + private logger: ILogger + ) {} + + // **************************** + // Probing + // **************************** + + // Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor. + private toProbeMonitor = (target: string): Monitor => { + const base = { + id: PROBE_MONITOR_ID, + teamId: PROBE_TEAM_ID, + name: `Egress probe: ${target}`, + method: "GET", + useAdvancedMatching: false, + ignoreTlsErrors: false, + customUpCodes: [], + }; + + if (/^https?:\/\//i.test(target)) { + return { ...base, type: "http", url: target } as unknown as Monitor; + } + + const hostPort = HOST_PORT_PATTERN.exec(target); + if (hostPort) { + const host = hostPort[1] ?? hostPort[2]; + return { ...base, type: "port", url: host, port: Number(hostPort[3]) } as unknown as Monitor; + } + + return { ...base, type: "ping", url: target } as unknown as Monitor; + }; + + private probeTarget = async (target: string): Promise => { + const monitor = this.toProbeMonitor(target); + let timer: NodeJS.Timeout | undefined; + + const { response, responseTime, error } = await timeRequest(() => + Promise.race([ + this.networkService.requestStatus(monitor), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS); + }), + ]).finally(() => clearTimeout(timer)) + ); + + if (error || !response) { + return { + target, + reachable: false, + responseTime, + message: error instanceof Error ? error.message : error ? String(error) : "No response", + }; + } + + return { + target, + reachable: response.status === true, + responseTime: response.responseTime ?? responseTime, + message: response.message, + }; + }; + + probeTargets = async (targets: string[]): Promise => { + return await Promise.all(targets.map((target) => this.probeTarget(target))); + }; + + private resolveTargets = (configured: string[] | undefined): string[] => { + const targets = (configured ?? []).map((target) => target.trim()).filter((target) => target.length > 0); + // An empty list would make "all unreachable" vacuously true, so fall back to the defaults. + return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS]; + }; + + // **************************** + // Event-triggered assessment + // **************************** + + assessAfterFailure = async (): Promise => { + try { + const settings = await this.settingsService.getDBSettings(); + if (!settings.egressCheckEnabled) { + return null; + } + + const state = await this.egressStateRepository.findSingleton(); + if (state.status === "degraded") { + // Recovery detection belongs to the poll loop. Make sure one is running in this process in case + // the worker that entered the degraded state is no longer around. + this.startRecoveryPolling(); + return "degraded"; + } + + const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); + const now = new Date(); + + if (results.some((result) => result.reachable)) { + await this.egressStateRepository.recordProbe(results, now); + return "ok"; + } + + const degraded = await this.egressStateRepository.markDegraded(results, now); + this.logger.warn({ + message: degraded + ? "Instance egress degraded: every reliability target is unreachable. Monitor failures will not be counted until it recovers" + : "Instance egress degraded (transition already recorded by another worker)", + service: SERVICE_NAME, + method: "assessAfterFailure", + details: { results }, + }); + this.startRecoveryPolling(); + return "degraded"; + } catch (error: unknown) { + // The egress check must never break check production; treat an internal failure as "unknown" and carry on. + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "assessAfterFailure", + stack: error instanceof Error ? error.stack : undefined, + }); + return null; + } + }; + + // **************************** + // Recovery poll loop + // **************************** + + startRecoveryPolling = () => { + if (this.polling) { + return; + } + this.polling = true; + this.logger.info({ message: "Starting egress recovery polling", service: SERVICE_NAME, method: "startRecoveryPolling" }); + void this.scheduleNextTick(); + }; + + stop = () => { + this.polling = false; + if (this.pollTimer) { + clearTimeout(this.pollTimer); + this.pollTimer = null; + } + }; + + private readPollIntervalMs = async () => { + try { + const settings = await this.settingsService.getDBSettings(); + const seconds = settings.egressPollIntervalSeconds; + return (Number.isFinite(seconds) && seconds > 0 ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000; + } catch (error: unknown) { + this.logger.warn({ + message: `Could not read egress poll interval, using default: ${error instanceof Error ? error.message : String(error)}`, + service: SERVICE_NAME, + method: "readPollIntervalMs", + }); + return DEFAULT_EGRESS_POLL_INTERVAL_SECONDS * 1000; + } + }; + + private scheduleNextTick = async () => { + if (!this.polling) { + return; + } + const intervalMs = await this.readPollIntervalMs(); + if (!this.polling) { + return; + } + this.pollTimer = setTimeout(() => void this.tick(), intervalMs); + // Never hold the process open for this loop; the state is persisted and init() resumes it on restart. + this.pollTimer.unref(); + }; + + private tick = async () => { + this.pollTimer = null; + if (!this.polling) { + return; + } + + try { + const state = await this.egressStateRepository.findSingleton(); + if (state.status !== "degraded") { + this.logger.info({ message: "Egress no longer degraded, stopping recovery polling", service: SERVICE_NAME, method: "tick" }); + this.stop(); + return; + } + + const settings = await this.settingsService.getDBSettings(); + const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); + const now = new Date(); + + if (!results.some((result) => result.reachable)) { + await this.egressStateRepository.recordProbe(results, now); + return; + } + + const recovered = await this.egressStateRepository.markRecovered(results, now); + this.stop(); + if (!recovered) { + // Another process performed the transition. + return; + } + + this.logger.info({ + message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`, + service: SERVICE_NAME, + method: "tick", + details: { results }, + }); + } catch (error: unknown) { + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "tick", + stack: error instanceof Error ? error.stack : undefined, + }); + } finally { + // No-op once stop() has been called; otherwise keep the loop alive through errors. + await this.scheduleNextTick(); + } + }; + + // **************************** + // Lifecycle + // **************************** + + init = async () => { + try { + const settings = await this.settingsService.getDBSettings(); + if (!settings.egressCheckEnabled) { + return; + } + const state = await this.egressStateRepository.findSingleton(); + if (state.status === "degraded") { + this.logger.warn({ + message: `Instance egress was degraded at startup (since ${state.degradedSince ?? "unknown"}), resuming recovery polling`, + service: SERVICE_NAME, + method: "init", + }); + this.startRecoveryPolling(); + } + } catch (error: unknown) { + this.logger.error({ + message: error instanceof Error ? error.message : String(error), + service: SERVICE_NAME, + method: "init", + stack: error instanceof Error ? error.stack : undefined, + }); + } + }; +} diff --git a/server/src/domain/egress/egress.type.ts b/server/src/domain/egress/egress.type.ts new file mode 100644 index 000000000..386116ef7 --- /dev/null +++ b/server/src/domain/egress/egress.type.ts @@ -0,0 +1,33 @@ +// Egress self-check: detects loss of the instance's own outbound connectivity so that +// a burst of simultaneous monitor failures is not reported as a burst of real outages. + +export const EgressStatuses = ["ok", "degraded"] as const; +export type EgressStatus = (typeof EgressStatuses)[number]; + +// Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect, +// http(s):// URLs are fetched. Any one target being reachable means egress is fine. +export const DEFAULT_EGRESS_TARGETS = ["1.1.1.1", "8.8.8.8"] as const; +export const DEFAULT_EGRESS_POLL_INTERVAL_SECONDS = 30; +export const MIN_EGRESS_POLL_INTERVAL_SECONDS = 5; +export const MAX_EGRESS_POLL_INTERVAL_SECONDS = 600; +export const MAX_EGRESS_TARGETS = 10; + +// Instance-global singleton persisted in MongoDB so that every worker process and the API +// process share one view of the state (the queue worker may run separately from the API). +export interface EgressState { + id: string; + status: EgressStatus; + degradedSince: string | null; // ISO timestamp of the current or most recent degraded entry + lastRecoveredAt: string | null; // ISO timestamp of the most recent recovery + lastProbeAt: string | null; // ISO timestamp of the most recent reliability probe + lastProbeResults: EgressProbeResult[]; + createdAt: string; + updatedAt: string; +} + +export interface EgressProbeResult { + target: string; + reachable: boolean; + responseTime: number; // ms + message?: string; +} diff --git a/server/test/unit/services/egressService.test.ts b/server/test/unit/services/egressService.test.ts new file mode 100644 index 000000000..8c4fbf183 --- /dev/null +++ b/server/test/unit/services/egressService.test.ts @@ -0,0 +1,404 @@ +import { afterEach, describe, expect, it, jest } from "@jest/globals"; +import { EgressService } from "../../../src/domain/egress/egress.service.ts"; +import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; +import { createMockLogger } from "../../helpers/createMockLogger.ts"; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const makeState = (overrides?: Partial): EgressState => ({ + id: "egress-1", + status: "ok", + degradedSince: null, + lastRecoveredAt: null, + lastProbeAt: null, + lastProbeResults: [], + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + ...overrides, +}); + +const makeSettings = (overrides?: Record) => ({ + egressCheckEnabled: true, + egressCheckTargets: ["1.1.1.1", "8.8.8.8"], + egressPollIntervalSeconds: 30, + egressNotifications: ["notif-1"], + ...overrides, +}); + +// requestStatus mock that answers per target url +const statusFor = (reachableTargets: string[]) => + jest.fn().mockImplementation(async (monitor: any) => ({ + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: reachableTargets.includes(monitor.url), + code: reachableTargets.includes(monitor.url) ? 200 : 5000, + message: reachableTargets.includes(monitor.url) ? "Success" : "Ping failed", + responseTime: 12, + })); + +const createService = (overrides?: Record) => { + const defaults = { + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) }, + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState()), + recordProbe: jest.fn().mockResolvedValue(makeState()), + markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), + markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })), + }, + networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) }, + logger: createMockLogger(), + ...overrides, + }; + const service = new EgressService( + defaults.settingsService as any, + defaults.egressStateRepository as any, + defaults.networkService as any, + defaults.logger as any + ); + return { service, defaults }; +}; + +// Lets the setTimeout chain in startRecoveryPolling settle (settings read, then timer armed) before advancing time. +const runOneTick = async (intervalMs: number) => { + await jest.advanceTimersByTimeAsync(0); + await jest.advanceTimersByTimeAsync(intervalMs); +}; + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("EgressService", () => { + afterEach(() => { + jest.useRealTimers(); + }); + + // ── probeTargets ────────────────────────────────────────────────────────── + + describe("probeTargets", () => { + it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => { + const { service, defaults } = createService(); + + await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443"]); + + const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any); + expect(monitors).toEqual([ + expect.objectContaining({ type: "ping", url: "1.1.1.1" }), + expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }), + expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }), + expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }), + ]); + }); + + it("reports reachable only when the provider says status is true", async () => { + const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } }); + + const results = await service.probeTargets(["1.1.1.1", "8.8.8.8"]); + + expect(results).toEqual([ + expect.objectContaining({ target: "1.1.1.1", reachable: true, responseTime: 12 }), + expect.objectContaining({ target: "8.8.8.8", reachable: false }), + ]); + }); + + it("counts a thrown provider error as unreachable", async () => { + const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } }); + + const results = await service.probeTargets(["1.1.1.1"]); + + expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: "ENETUNREACH" })]); + }); + + it("counts a hung provider as unreachable once the probe timeout elapses", async () => { + jest.useFakeTimers(); + const { service } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } }); + + const pending = service.probeTargets(["1.1.1.1"]); + await jest.advanceTimersByTimeAsync(5000); + const results = await pending; + + expect(results).toEqual([expect.objectContaining({ target: "1.1.1.1", reachable: false, message: expect.stringContaining("timed out") })]); + }); + }); + + // ── assessAfterFailure ──────────────────────────────────────────────────── + + describe("assessAfterFailure", () => { + it("returns null and does not probe or record when the feature is disabled", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBeNull(); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + }); + + it("returns ok and records the probe when any target is reachable", async () => { + const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("ok"); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledWith( + [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: true })], + expect.any(Date) + ); + expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); + }); + + it("marks degraded, warns and returns degraded when every target is unreachable", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("degraded"); + expect(defaults.egressStateRepository.markDegraded).toHaveBeenCalledWith( + [expect.objectContaining({ target: "1.1.1.1", reachable: false }), expect.objectContaining({ target: "8.8.8.8", reachable: false })], + expect.any(Date) + ); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") })); + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); + service.stop(); + }); + + it("returns degraded without probing when the persisted state is already degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("degraded"); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); + service.stop(); + }); + + it("falls back to the default targets when the configured list is empty", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckTargets: [] })) }, + }); + + await service.assessAfterFailure(); + + const urls = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url); + expect(urls).toEqual(["1.1.1.1", "8.8.8.8"]); + }); + + it("returns null and logs when the assessment itself fails", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockRejectedValue(new Error("db down")) }, + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBeNull(); + expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db down", method: "assessAfterFailure" })); + }); + }); + + // ── recovery polling ────────────────────────────────────────────────────── + + describe("recovery polling", () => { + it("marks recovered once and stops polling when a target becomes reachable", async () => { + jest.useFakeTimers(); + const recovered = makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" }); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(recovered), + }, + networkService: { requestStatus: statusFor(["1.1.1.1"]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })); + + // Loop has stopped: a further interval produces no more work + await runOneTick(30_000); + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + }); + + it("does not log a recovery when another process already recorded it", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(null), + }, + networkService: { requestStatus: statusFor(["1.1.1.1"]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.logger.info).not.toHaveBeenCalledWith( + expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }) + ); + }); + + it("records the probe and keeps polling while every target stays unreachable", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); + expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); + service.stop(); + }); + + it("stops polling when the persisted state is no longer degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "ok" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + await runOneTick(30_000); + expect(defaults.egressStateRepository.findSingleton).toHaveBeenCalledTimes(1); + }); + + it("survives a failing tick and reschedules", async () => { + jest.useFakeTimers(); + const findSingleton = jest + .fn() + .mockRejectedValueOnce(new Error("db blip")) + .mockResolvedValue(makeState({ status: "degraded" })); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton, + recordProbe: jest.fn().mockResolvedValue(makeState()), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(30_000); + expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db blip", method: "tick" })); + + await runOneTick(30_000); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + + it("re-reads the poll interval from settings on each tick", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: 5 })) }, + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + await runOneTick(5_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + + it("is idempotent: a second start does not arm a second loop", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn(), + markRecovered: jest.fn(), + }, + networkService: { requestStatus: statusFor([]) }, + }); + + service.startRecoveryPolling(); + service.startRecoveryPolling(); + await runOneTick(30_000); + + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + service.stop(); + }); + }); + + // ── init ────────────────────────────────────────────────────────────────── + + describe("init", () => { + it("resumes polling when the feature is enabled and the persisted state is degraded", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ + egressStateRepository: { + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn().mockResolvedValue(makeState()), + }, + }); + + await service.init(); + + expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); + service.stop(); + }); + + it("does nothing when the persisted state is ok", async () => { + const { service, defaults } = createService(); + + await service.init(); + + expect(defaults.logger.info).not.toHaveBeenCalled(); + }); + + it("does nothing when the feature is disabled", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + }); + + await service.init(); + + expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/server/test/unit/services/egressStateService.test.ts b/server/test/unit/services/egressStateService.test.ts new file mode 100644 index 000000000..85c32f32d --- /dev/null +++ b/server/test/unit/services/egressStateService.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, jest } from "@jest/globals"; +import { EgressStateService } from "../../../src/domain/egress/egress-state.service.ts"; +import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const createEgressStateRepo = () => ({ + findSingleton: jest.fn(), + recordProbe: jest.fn(), + markDegraded: jest.fn(), + markRecovered: jest.fn(), +}); + +const createService = () => { + const egressStateRepository = createEgressStateRepo(); + const service = new EgressStateService(egressStateRepository as any); + return { service, egressStateRepository }; +}; + +const makeState = (overrides?: Partial): EgressState => ({ + id: "egress-1", + status: "ok", + degradedSince: null, + lastRecoveredAt: null, + lastProbeAt: null, + lastProbeResults: [], + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + ...overrides, +}); + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe("EgressStateService", () => { + describe("getState", () => { + it("returns the singleton state from the repository", async () => { + const { service, egressStateRepository } = createService(); + const state = makeState({ + status: "degraded", + degradedSince: "2026-01-01T00:05:00Z", + lastProbeAt: "2026-01-01T00:05:00Z", + lastProbeResults: [{ target: "1.1.1.1", reachable: false, responseTime: 0, message: "timeout" }], + }); + (egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(state); + + const result = await service.getState(); + + expect(egressStateRepository.findSingleton).toHaveBeenCalledTimes(1); + expect(result).toEqual(state); + }); + + it("does not call any transition method", async () => { + const { service, egressStateRepository } = createService(); + (egressStateRepository.findSingleton as jest.Mock).mockResolvedValue(makeState()); + + await service.getState(); + + expect(egressStateRepository.recordProbe).not.toHaveBeenCalled(); + expect(egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(egressStateRepository.markRecovered).not.toHaveBeenCalled(); + }); + + it("propagates repository errors", async () => { + const { service, egressStateRepository } = createService(); + (egressStateRepository.findSingleton as jest.Mock).mockRejectedValue(new Error("db down")); + + await expect(service.getState()).rejects.toThrow("db down"); + }); + }); +}); diff --git a/server/test/unit/validation/settingsValidation.test.ts b/server/test/unit/validation/settingsValidation.test.ts new file mode 100644 index 000000000..36e25c3b2 --- /dev/null +++ b/server/test/unit/validation/settingsValidation.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "@jest/globals"; +import { updateAppSettingsBodyValidation } from "../../../src/api/validation/settingsValidation.ts"; +import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "../../../src/domain/egress/egress.type.ts"; + +const validNotificationId = "64b7f0c2a1d2e3f4a5b6c7d8"; + +describe("settingsValidation", () => { + describe("updateAppSettingsBodyValidation egress fields", () => { + it("accepts a full set of valid egress settings", () => { + const parsed = updateAppSettingsBodyValidation.parse({ + egressCheckEnabled: true, + egressCheckTargets: ["1.1.1.1", "dns.google:53", "https://example.com/health"], + egressPollIntervalSeconds: 30, + egressNotifications: [validNotificationId], + }); + + expect(parsed.egressCheckEnabled).toBe(true); + expect(parsed.egressCheckTargets).toEqual(["1.1.1.1", "dns.google:53", "https://example.com/health"]); + expect(parsed.egressPollIntervalSeconds).toBe(30); + expect(parsed.egressNotifications).toEqual([validNotificationId]); + }); + + it("accepts a body with no egress fields", () => { + expect(() => updateAppSettingsBodyValidation.parse({ checkTTL: 30 })).not.toThrow(); + }); + + it("accepts an empty target list (the worker falls back to the default targets)", () => { + expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: [] }).egressCheckTargets).toEqual([]); + }); + + it("rejects more targets than the maximum", () => { + const targets = Array.from({ length: MAX_EGRESS_TARGETS + 1 }, (_, i) => `10.0.0.${i}`); + + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow(); + }); + + it("rejects targets that are not a host, host:port or http(s) URL", () => { + for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm"]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow(); + } + }); + + it("accepts the poll interval boundaries", () => { + expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MIN_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe( + MIN_EGRESS_POLL_INTERVAL_SECONDS + ); + expect(updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: MAX_EGRESS_POLL_INTERVAL_SECONDS }).egressPollIntervalSeconds).toBe( + MAX_EGRESS_POLL_INTERVAL_SECONDS + ); + }); + + it("rejects out-of-range and non-integer poll intervals", () => { + for (const interval of [MIN_EGRESS_POLL_INTERVAL_SECONDS - 1, MAX_EGRESS_POLL_INTERVAL_SECONDS + 1, 0, -5, 30.5, "30"]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressPollIntervalSeconds: interval })).toThrow(); + } + }); + + it("accepts an empty notification list", () => { + expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]); + }); + + it("rejects notification ids that are not 24-character hex", () => { + for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow(); + } + }); + + it("rejects a non-boolean enabled flag", () => { + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckEnabled: "yes" })).toThrow(); + }); + }); +}); From 0f474d82d66e8dbd8426af968da3cd4bbb40b293 Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 14 Sep 2026 09:45:09 -0700 Subject: [PATCH 28/35] use exact unit for duration --- client/src/Utils/TimeUtils.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/client/src/Utils/TimeUtils.ts b/client/src/Utils/TimeUtils.ts index d82649c81..2fc9412a4 100644 --- a/client/src/Utils/TimeUtils.ts +++ b/client/src/Utils/TimeUtils.ts @@ -120,30 +120,36 @@ const DURATION_UNITS: Array<{ unit: DurationUnit; milliseconds: number }> = [ { unit: "second", milliseconds: MS_PER_SECOND }, ]; +const FALLBACK_DURATION_UNIT = DURATION_UNITS[DURATION_UNITS.length - 1]; + const getDurationUnit = (ms: number) => DURATION_UNITS.find(({ milliseconds }) => Math.abs(ms) >= milliseconds) ?? - DURATION_UNITS[DURATION_UNITS.length - 1]; + FALLBACK_DURATION_UNIT; + +const getExactDurationUnit = (ms: number) => + DURATION_UNITS.find(({ milliseconds }) => ms !== 0 && ms % milliseconds === 0) ?? + FALLBACK_DURATION_UNIT; export const formatDuration = ( ms: number, { long = false, relative = false, hasSpace = true, locale }: FormatDurationOptions = {} ): string => { - if (long || relative) { + if (relative) { const { unit, milliseconds } = getDurationUnit(ms); const value = Math.sign(ms) * Math.floor(Math.abs(ms) / milliseconds); + return new Intl.RelativeTimeFormat(locale, { + numeric: "always", + style: long ? "long" : "narrow", + }).format(-value, unit); + } - if (relative) { - return new Intl.RelativeTimeFormat(locale, { - numeric: "always", - style: long ? "long" : "narrow", - }).format(-value, unit); - } - + if (long) { + const { unit, milliseconds } = getExactDurationUnit(ms); return new Intl.NumberFormat(locale, { style: "unit", unit, unitDisplay: "long", - }).format(value); + }).format(ms / milliseconds); } const formatted = prettyMilliseconds(ms, { From 308672d558c3bc560f1aba204ae8c31efcb6a94b Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Mon, 14 Sep 2026 09:45:24 -0700 Subject: [PATCH 29/35] pass locale to formatting --- client/src/Pages/CreateMonitor/index.tsx | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 921f3deb3..31ba8d3aa 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -87,14 +87,18 @@ interface IntervalOption extends AutocompleteOption { const INTERVAL_UNIT_MULTIPLIERS = [1000, 60000, 3600000, 86400000] as const; -const toIntervalOption = (intervalMs: number): IntervalOption => ({ +const toIntervalOption = ( + intervalMs: number, + locale: string | undefined +): IntervalOption => ({ id: intervalMs, - name: formatDuration(intervalMs, true), + name: formatDuration(intervalMs, { long: true, locale }), }); const filterIntervalOptions = ( options: IntervalOption[], - inputValue: string + inputValue: string, + locale: string | undefined ): IntervalOption[] => { const input = inputValue.trim(); if (input === "" || options.some((option) => option.name === input)) { @@ -108,7 +112,7 @@ const filterIntervalOptions = ( return INTERVAL_UNIT_MULTIPLIERS.map((multiplier) => amount * multiplier) .filter(Number.isSafeInteger) - .map(toIntervalOption); + .map((ms) => toIntervalOption(ms, locale)); }; interface GeneralSettingsConfig { @@ -288,7 +292,7 @@ const getGeneralSettingsConfig = ( const CreateMonitorPage = () => { const theme = useTheme(); - const { t } = useTranslation(); + const { t, i18n } = useTranslation(); const { monitorId } = useParams(); const location = useLocation(); const navigate = useNavigate(); @@ -509,15 +513,17 @@ const CreateMonitorPage = () => { ); const intervalOptions = useMemo(() => { - const options = MonitorIntervalOptions.map(({ value }) => toIntervalOption(value)); + const options = MonitorIntervalOptions.map(({ value }) => + toIntervalOption(value, i18n.resolvedLanguage) + ); if ( typeof watchedInterval === "number" && !options.some((option) => option.id === watchedInterval) ) { - options.push(toIntervalOption(watchedInterval)); + options.push(toIntervalOption(watchedInterval, i18n.resolvedLanguage)); } return options; - }, [watchedInterval]); + }, [watchedInterval, i18n.resolvedLanguage]); const matchMethodOptions = useMemo( () => @@ -808,7 +814,7 @@ const CreateMonitorPage = () => { )} options={intervalOptions} filterOptions={(options, { inputValue }) => - filterIntervalOptions(options, inputValue) + filterIntervalOptions(options, inputValue, i18n.resolvedLanguage) } getOptionDisabled={(option) => option.id < MIN_MONITOR_INTERVAL_MS} disableClearable From 46eb0b4063b26d20176cd2250b33457dff68eccf Mon Sep 17 00:00:00 2001 From: Craig Lawson Date: Mon, 14 Sep 2026 20:41:58 +0100 Subject: [PATCH 30/35] Replace the egress recovery timer with a scheduled job (#3937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed check still calls assessAfterFailure, which now shares one in-flight assessment between concurrent failures (same cache shape as ProxyResolver). When egress degrades it upserts a global "egress" job; DBQueueWorker runs checkRecovery for that type at the configured interval, and the job deletes its own row once any target answers. The unref'd timer, startRecoveryPolling, stop and init are gone: a degraded episode survives a restart through the jobs collection. deleteByIdAndType accepts a null refId for global rows. 🔧 --- server/src/config/services.worker.ts | 6 +- server/src/domain/egress/egress.service.ts | 197 +++++-------- .../domain/jobs/job.repository.interface.ts | 4 +- .../src/domain/jobs/job.repository.mongo.ts | 4 +- server/src/domain/jobs/job.type.ts | 4 +- server/src/worker/worker.db-queue.ts | 9 + server/src/worker/worker.job-scheduler.ts | 1 + .../test/unit/services/dbQueueWorker.test.ts | 14 +- .../test/unit/services/egressService.test.ts | 273 +++++++----------- 9 files changed, 203 insertions(+), 309 deletions(-) diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index 83a875f05..b97bb56f3 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -106,7 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository); const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); const monitorStatusPolicy = new MonitorStatusPolicy(); - const egressService = new EgressService(settingsService, egressStateRepository, networkService, logger); + const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, logger); // *********************** // Reactors and dispatcher @@ -165,6 +165,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig geoCheckPipeline, dispatcher: reactorDispatcher, helper: workerHelper, + egressService, queueWorkersRepository, queueMode: envSettings.queueMode, queuePrimaryProcesses: envSettings.queuePrimaryProcesses, @@ -189,8 +190,5 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig }); } - // Resume recovery polling if the instance was mid-episode when it last stopped - await egressService.init(); - return { worker, networkService, bufferService, statusService, egressService }; }; diff --git a/server/src/domain/egress/egress.service.ts b/server/src/domain/egress/egress.service.ts index cb3b79ebb..9ca98f4b2 100644 --- a/server/src/domain/egress/egress.service.ts +++ b/server/src/domain/egress/egress.service.ts @@ -1,6 +1,8 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; +import { jobId, type JobSeed } from "@/domain/jobs/job.type.js"; import type { INetworkService } from "@/service/networkService.js"; import type { ILogger } from "@/utils/logger.js"; import { @@ -13,6 +15,11 @@ import { timeRequest } from "@/service/network/utils.js"; const SERVICE_NAME = "EgressService"; const PROBE_TIMEOUT_MS = 5000; +// When egress is down every monitor fails at once, so concurrent failures share one assessment +// rather than each probing. Same in-flight cache shape as ProxyResolver. +const ASSESSMENT_TTL_MS = PROBE_TIMEOUT_MS; +// Global job row (refId null) that re-probes while degraded. Inserted when egress degrades, removed on recovery. +const RECOVERY_JOB_ID = jobId("egress", null); // Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs. const PROBE_MONITOR_ID = "egress-probe"; @@ -23,21 +30,21 @@ const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/; export interface IEgressService { probeTargets(targets: string[]): Promise; + // Called by the check producer when a check fails. Null when the feature is off or the assessment itself failed. assessAfterFailure(): Promise; - startRecoveryPolling(): void; - stop(): void; - init(): Promise; + // Handler for the "egress" job: re-probes while degraded and removes the job once egress is back. + checkRecovery(): Promise; } export class EgressService implements IEgressService { static SERVICE_NAME = SERVICE_NAME; - private polling = false; - private pollTimer: NodeJS.Timeout | null = null; + private assessment: { value: Promise; expiresAt: number } | null = null; constructor( private settingsService: ISettingsService, private egressStateRepository: IEgressStateRepository, + private jobsRepository: IJobsRepository, private networkService: INetworkService, private logger: ILogger ) {} @@ -111,11 +118,31 @@ export class EgressService implements IEgressService { return targets.length > 0 ? targets : [...DEFAULT_EGRESS_TARGETS]; }; + private toIntervalMs = (seconds: number | undefined): number => { + const valid = typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0; + return (valid ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000; + }; + // **************************** // Event-triggered assessment // **************************** - assessAfterFailure = async (): Promise => { + assessAfterFailure = (): Promise => { + const now = Date.now(); + if (this.assessment && this.assessment.expiresAt > now) { + return this.assessment.value; + } + // Cache the in-flight promise so concurrent failures share one probe + const value = this.assess(); + this.assessment = { value, expiresAt: now + ASSESSMENT_TTL_MS }; + // Don't cache an internal failure, so the next failing check retries + void value.then((status) => { + if (status === null && this.assessment?.value === value) this.assessment = null; + }); + return value; + }; + + private assess = async (): Promise => { try { const settings = await this.settingsService.getDBSettings(); if (!settings.egressCheckEnabled) { @@ -124,9 +151,8 @@ export class EgressService implements IEgressService { const state = await this.egressStateRepository.findSingleton(); if (state.status === "degraded") { - // Recovery detection belongs to the poll loop. Make sure one is running in this process in case - // the worker that entered the degraded state is no longer around. - this.startRecoveryPolling(); + // Recovery is detected by the scheduled job. Make sure it exists in case its row was lost. + await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds); return "degraded"; } @@ -147,7 +173,7 @@ export class EgressService implements IEgressService { method: "assessAfterFailure", details: { results }, }); - this.startRecoveryPolling(); + await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds); return "degraded"; } catch (error: unknown) { // The egress check must never break check production; treat an internal failure as "unknown" and carry on. @@ -162,129 +188,58 @@ export class EgressService implements IEgressService { }; // **************************** - // Recovery poll loop + // Recovery job // **************************** - startRecoveryPolling = () => { - if (this.polling) { - return; - } - this.polling = true; - this.logger.info({ message: "Starting egress recovery polling", service: SERVICE_NAME, method: "startRecoveryPolling" }); - void this.scheduleNextTick(); + // upsertJob only sets nextScheduledAt on insert, so re-arming while degraded never pushes a pending run back. + private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined) => { + const intervalMs = this.toIntervalMs(pollIntervalSeconds); + const seed: JobSeed = { + id: RECOVERY_JOB_ID, + type: "egress", + refId: null, + isActive: true, + nextScheduledAt: Date.now() + intervalMs, + intervalMs, + }; + await this.jobsRepository.upsertJob(seed); }; - stop = () => { - this.polling = false; - if (this.pollTimer) { - clearTimeout(this.pollTimer); - this.pollTimer = null; - } + private removeRecoveryJob = async () => { + await this.jobsRepository.deleteByIdAndType(null, "egress"); }; - private readPollIntervalMs = async () => { - try { - const settings = await this.settingsService.getDBSettings(); - const seconds = settings.egressPollIntervalSeconds; - return (Number.isFinite(seconds) && seconds > 0 ? seconds : DEFAULT_EGRESS_POLL_INTERVAL_SECONDS) * 1000; - } catch (error: unknown) { - this.logger.warn({ - message: `Could not read egress poll interval, using default: ${error instanceof Error ? error.message : String(error)}`, - service: SERVICE_NAME, - method: "readPollIntervalMs", - }); - return DEFAULT_EGRESS_POLL_INTERVAL_SECONDS * 1000; - } - }; - - private scheduleNextTick = async () => { - if (!this.polling) { - return; - } - const intervalMs = await this.readPollIntervalMs(); - if (!this.polling) { - return; - } - this.pollTimer = setTimeout(() => void this.tick(), intervalMs); - // Never hold the process open for this loop; the state is persisted and init() resumes it on restart. - this.pollTimer.unref(); - }; - - private tick = async () => { - this.pollTimer = null; - if (!this.polling) { + // Runs on the queue at the configured interval while degraded. Errors propagate so the queue records + // the failure and retries with its usual backoff. + checkRecovery = async (): Promise => { + const state = await this.egressStateRepository.findSingleton(); + if (state.status !== "degraded") { + // Another worker recorded the recovery, or the state was reset; nothing left to poll for. + await this.removeRecoveryJob(); return; } - try { - const state = await this.egressStateRepository.findSingleton(); - if (state.status !== "degraded") { - this.logger.info({ message: "Egress no longer degraded, stopping recovery polling", service: SERVICE_NAME, method: "tick" }); - this.stop(); - return; - } + const settings = await this.settingsService.getDBSettings(); + const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); + const now = new Date(); - const settings = await this.settingsService.getDBSettings(); - const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); - const now = new Date(); - - if (!results.some((result) => result.reachable)) { - await this.egressStateRepository.recordProbe(results, now); - return; - } - - const recovered = await this.egressStateRepository.markRecovered(results, now); - this.stop(); - if (!recovered) { - // Another process performed the transition. - return; - } - - this.logger.info({ - message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`, - service: SERVICE_NAME, - method: "tick", - details: { results }, - }); - } catch (error: unknown) { - this.logger.error({ - message: error instanceof Error ? error.message : String(error), - service: SERVICE_NAME, - method: "tick", - stack: error instanceof Error ? error.stack : undefined, - }); - } finally { - // No-op once stop() has been called; otherwise keep the loop alive through errors. - await this.scheduleNextTick(); + if (!results.some((result) => result.reachable)) { + await this.egressStateRepository.recordProbe(results, now); + return; } - }; - // **************************** - // Lifecycle - // **************************** - - init = async () => { - try { - const settings = await this.settingsService.getDBSettings(); - if (!settings.egressCheckEnabled) { - return; - } - const state = await this.egressStateRepository.findSingleton(); - if (state.status === "degraded") { - this.logger.warn({ - message: `Instance egress was degraded at startup (since ${state.degradedSince ?? "unknown"}), resuming recovery polling`, - service: SERVICE_NAME, - method: "init", - }); - this.startRecoveryPolling(); - } - } catch (error: unknown) { - this.logger.error({ - message: error instanceof Error ? error.message : String(error), - service: SERVICE_NAME, - method: "init", - stack: error instanceof Error ? error.stack : undefined, - }); + const recovered = await this.egressStateRepository.markRecovered(results, now); + await this.removeRecoveryJob(); + if (!recovered) { + // Another process performed the transition. + return; } + + this.logger.info({ + message: `Instance egress recovered (degraded since ${recovered.degradedSince ?? "unknown"})`, + service: SERVICE_NAME, + method: "checkRecovery", + details: { results }, + }); }; } diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index d253523fa..a352c0888 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -58,8 +58,8 @@ export interface IJobsRepository { // Delete, drop all rows deleteById(refId: string): Promise; - // Delete a single job of type - deleteByIdAndType(refId: string, type: JobType): Promise; + // Delete a single job of type. refId null addresses a global row + deleteByIdAndType(refId: string | null, type: JobType): Promise; // Delete jobs that reference monitors that no longer exist deleteByMonitorIdsNotIn(monitorIds: string[]): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index 60bd4cd77..ce4143b2d 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -273,8 +273,8 @@ class MongoJobsRepository implements IJobsRepository { return res.deletedCount > 0; }; - deleteByIdAndType = async (refId: string, type: JobType) => { - const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row + deleteByIdAndType = async (refId: string | null, type: JobType) => { + const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row or the global egress row return res.deletedCount > 0; }; diff --git a/server/src/domain/jobs/job.type.ts b/server/src/domain/jobs/job.type.ts index 70d635f81..ae8fbddc3 100644 --- a/server/src/domain/jobs/job.type.ts +++ b/server/src/domain/jobs/job.type.ts @@ -1,10 +1,10 @@ -export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const; +export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"] as const; export type JobType = (typeof JOB_TYPES)[number]; export const LOCK_MS = 60_000; export const BACKOFF_MS = 5_000; export const PARKED = Number.MAX_SAFE_INTEGER; -// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type. +// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type. export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`); export type Job = { diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 6f4f31353..49f630b01 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -10,6 +10,7 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IWorkerHelper } from "@/worker/worker.helper.js"; +import { IEgressService } from "@/domain/egress/egress.service.js"; import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; @@ -28,6 +29,7 @@ const CONCURRENCY: Record = { evaluate: 20, "cleanup-orphaned": 1, "cleanup-retention": 1, + egress: 1, }; export interface DBQueueWorkerDependencies { @@ -43,6 +45,7 @@ export interface DBQueueWorkerDependencies { geoCheckPipeline: ICheckPipeline; dispatcher: IReactorDispatcher; helper: IWorkerHelper; + egressService: IEgressService; queueWorkersRepository: IQueueWorkersRepository; // forwarded queueMode: QueueMode; // forwarded queuePrimaryProcesses: boolean; // consumed immediately @@ -61,6 +64,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { evaluate: 0, "cleanup-orphaned": 0, "cleanup-retention": 0, + egress: 0, }; private checksRepository: IChecksRepository; @@ -70,6 +74,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { private dispatcher: IReactorDispatcher; private geoCheckPipeline: ICheckPipeline; private helper: IWorkerHelper; + private egressService: IEgressService; private bufferService: IBufferService; private isDbConnected: () => boolean; @@ -90,6 +95,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { this.dispatcher = dependencies.dispatcher; this.geoCheckPipeline = dependencies.geoCheckPipeline; this.helper = dependencies.helper; + this.egressService = dependencies.egressService; this.bufferService = dependencies.bufferService; this.isDbConnected = dependencies.isDbConnected; } @@ -175,6 +181,9 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { case "cleanup-retention": await this.helper.getCleanupRetentionJob()(); // Get job and execute break; + case "egress": + await this.egressService.checkRecovery(); // Re-probe while degraded; removes its own row on recovery + break; } if (job.intervalMs === null) { diff --git a/server/src/worker/worker.job-scheduler.ts b/server/src/worker/worker.job-scheduler.ts index aae8a489b..557a2f8e2 100644 --- a/server/src/worker/worker.job-scheduler.ts +++ b/server/src/worker/worker.job-scheduler.ts @@ -41,6 +41,7 @@ export class JobScheduler implements IJobScheduler { evaluate: POLL_MS, "cleanup-orphaned": POLL_MS, "cleanup-retention": POLL_MS, + egress: POLL_MS, }; // Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd. protected toCheckJob = (monitor: MonitorScheduleFields, now: number, immediate = false): JobSeed => ({ diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index 8b13287d4..75acaa063 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -9,7 +9,7 @@ import { createMockLogger } from "../../helpers/createMockLogger.ts"; // ── Notes ────────────────────────────────────────────────────────────────────── // // DBQueueWorker is the orchestrator: it drives the polling loops, claims batches -// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup) and +// sized to free capacity, runs jobs (check / evaluate / geo-check / cleanup / egress) and // renews their locks, and registers itself in the worker heartbeat. The repository // claim/lock atomicity is covered in integration/jobsRepository.test.ts; here we // test only the orchestration around it, with the repository fully mocked. @@ -114,6 +114,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), }; + const egressService = { checkRecovery: jest.fn().mockResolvedValue(undefined) }; const queueWorkersRepository = { upsert: jest.fn().mockResolvedValue(undefined), deleteById: jest.fn().mockResolvedValue(undefined), @@ -134,6 +135,7 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses geoCheckPipeline, dispatcher, helper, + egressService, queueWorkersRepository, logger, ...overrides?.mocks, @@ -226,7 +228,7 @@ describe("DBQueueWorker", () => { const { mocks } = await start({ queueMode: "worker" }); const claimedTypes = mocks.jobsRepository.claimDueBatch.mock.calls.map((c: any[]) => c[0]); - expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"])); + expect(claimedTypes).toEqual(expect.arrayContaining(["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention", "egress"])); expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("check", 500, expect.any(Number)); expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("evaluate", 20, expect.any(Number)); expect(mocks.jobsRepository.claimDueBatch).toHaveBeenCalledWith("cleanup-orphaned", 1, expect.any(Number)); @@ -440,6 +442,14 @@ describe("DBQueueWorker", () => { expect(retentionFn).toHaveBeenCalled(); }); + it("egress job runs the egress recovery check", async () => { + const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 }); + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); + + expect(mocks.egressService.checkRecovery).toHaveBeenCalledTimes(1); + expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); + }); + it("geo-check job runs the geo pipeline (no evaluate handoff)", async () => { const job = makeJob({ id: "geo-check:m1", type: "geo-check", refId: "m1", intervalMs: 300000 }); const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); diff --git a/server/test/unit/services/egressService.test.ts b/server/test/unit/services/egressService.test.ts index 8c4fbf183..ffe124495 100644 --- a/server/test/unit/services/egressService.test.ts +++ b/server/test/unit/services/egressService.test.ts @@ -46,6 +46,10 @@ const createService = (overrides?: Record) => { markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })), }, + jobsRepository: { + upsertJob: jest.fn().mockResolvedValue(true), + deleteByIdAndType: jest.fn().mockResolvedValue(true), + }, networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) }, logger: createMockLogger(), ...overrides, @@ -53,17 +57,20 @@ const createService = (overrides?: Record) => { const service = new EgressService( defaults.settingsService as any, defaults.egressStateRepository as any, + defaults.jobsRepository as any, defaults.networkService as any, defaults.logger as any ); return { service, defaults }; }; -// Lets the setTimeout chain in startRecoveryPolling settle (settings read, then timer armed) before advancing time. -const runOneTick = async (intervalMs: number) => { - await jest.advanceTimersByTimeAsync(0); - await jest.advanceTimersByTimeAsync(intervalMs); -}; +const degradedRepository = (overrides?: Record) => ({ + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn().mockResolvedValue(null), + markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })), + ...overrides, +}); // ── Tests ──────────────────────────────────────────────────────────────────── @@ -149,8 +156,7 @@ describe("EgressService", () => { expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); }); - it("marks degraded, warns and returns degraded when every target is unreachable", async () => { - jest.useFakeTimers(); + it("marks degraded, warns, inserts the recovery job and returns degraded when every target is unreachable", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } }); const result = await service.assessAfterFailure(); @@ -162,20 +168,18 @@ describe("EgressService", () => { ); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") })); - expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); - service.stop(); + expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith({ + id: "egress", + type: "egress", + refId: null, + isActive: true, + nextScheduledAt: expect.any(Number), + intervalMs: 30_000, + }); }); - it("returns degraded without probing when the persisted state is already degraded", async () => { - jest.useFakeTimers(); - const { service, defaults } = createService({ - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - recordProbe: jest.fn(), - markDegraded: jest.fn(), - markRecovered: jest.fn(), - }, - }); + it("returns degraded without probing when the persisted state is already degraded, re-arming the recovery job", async () => { + const { service, defaults } = createService({ egressStateRepository: degradedRepository() }); const result = await service.assessAfterFailure(); @@ -183,7 +187,52 @@ describe("EgressService", () => { expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); - service.stop(); + expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 })); + }); + + it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: undefined })) }, + networkService: { requestStatus: statusFor([]) }, + }); + + await service.assessAfterFailure(); + + expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 })); + }); + + it("shares one probe between concurrent failures", async () => { + const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); + + const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]); + + expect(results).toEqual(["ok", "ok", "ok"]); + expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1); + expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + }); + + it("reuses a settled assessment within the cache window and probes again after it", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); + + await service.assessAfterFailure(); + await jest.advanceTimersByTimeAsync(1000); + await service.assessAfterFailure(); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + + await jest.advanceTimersByTimeAsync(5000); + await service.assessAfterFailure(); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); + }); + + it("does not cache an internal failure", async () => { + const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings()); + const { service, defaults } = createService({ settingsService: { getDBSettings } }); + + expect(await service.assessAfterFailure()).toBeNull(); + expect(await service.assessAfterFailure()).toBe("ok"); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); }); it("falls back to the default targets when the configured list is empty", async () => { @@ -209,196 +258,68 @@ describe("EgressService", () => { }); }); - // ── recovery polling ────────────────────────────────────────────────────── + // ── checkRecovery (the "egress" job) ────────────────────────────────────── - describe("recovery polling", () => { - it("marks recovered once and stops polling when a target becomes reachable", async () => { - jest.useFakeTimers(); - const recovered = makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" }); + describe("checkRecovery", () => { + it("marks recovered, removes the job and logs when a target becomes reachable", async () => { const { service, defaults } = createService({ - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - recordProbe: jest.fn(), - markDegraded: jest.fn(), - markRecovered: jest.fn().mockResolvedValue(recovered), - }, + egressStateRepository: degradedRepository(), networkService: { requestStatus: statusFor(["1.1.1.1"]) }, }); - service.startRecoveryPolling(); - await runOneTick(30_000); + await service.checkRecovery(); - expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith( + [expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })], + expect.any(Date) + ); + expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })); - - // Loop has stopped: a further interval produces no more work - await runOneTick(30_000); - expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); }); - it("does not log a recovery when another process already recorded it", async () => { - jest.useFakeTimers(); + it("removes the job but does not log a recovery when another process already recorded it", async () => { const { service, defaults } = createService({ - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - recordProbe: jest.fn(), - markDegraded: jest.fn(), - markRecovered: jest.fn().mockResolvedValue(null), - }, + egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }), networkService: { requestStatus: statusFor(["1.1.1.1"]) }, }); - service.startRecoveryPolling(); - await runOneTick(30_000); + await service.checkRecovery(); - expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledTimes(1); + expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); expect(defaults.logger.info).not.toHaveBeenCalledWith( expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }) ); }); - it("records the probe and keeps polling while every target stays unreachable", async () => { - jest.useFakeTimers(); + it("records the probe and keeps the job while every target stays unreachable", async () => { const { service, defaults } = createService({ - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - markDegraded: jest.fn(), - markRecovered: jest.fn(), - }, + egressStateRepository: degradedRepository(), networkService: { requestStatus: statusFor([]) }, }); - service.startRecoveryPolling(); - await runOneTick(30_000); - await runOneTick(30_000); + await service.checkRecovery(); - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); - service.stop(); + expect(defaults.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled(); }); - it("stops polling when the persisted state is no longer degraded", async () => { - jest.useFakeTimers(); - const { service, defaults } = createService({ - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "ok" })), - recordProbe: jest.fn(), - markDegraded: jest.fn(), - markRecovered: jest.fn(), - }, - }); - - service.startRecoveryPolling(); - await runOneTick(30_000); - - expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); - await runOneTick(30_000); - expect(defaults.egressStateRepository.findSingleton).toHaveBeenCalledTimes(1); - }); - - it("survives a failing tick and reschedules", async () => { - jest.useFakeTimers(); - const findSingleton = jest - .fn() - .mockRejectedValueOnce(new Error("db blip")) - .mockResolvedValue(makeState({ status: "degraded" })); - const { service, defaults } = createService({ - egressStateRepository: { - findSingleton, - recordProbe: jest.fn().mockResolvedValue(makeState()), - markDegraded: jest.fn(), - markRecovered: jest.fn(), - }, - networkService: { requestStatus: statusFor([]) }, - }); - - service.startRecoveryPolling(); - await runOneTick(30_000); - expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "db blip", method: "tick" })); - - await runOneTick(30_000); - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); - service.stop(); - }); - - it("re-reads the poll interval from settings on each tick", async () => { - jest.useFakeTimers(); - const { service, defaults } = createService({ - settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressPollIntervalSeconds: 5 })) }, - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - markDegraded: jest.fn(), - markRecovered: jest.fn(), - }, - networkService: { requestStatus: statusFor([]) }, - }); - - service.startRecoveryPolling(); - await runOneTick(5_000); - - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); - service.stop(); - }); - - it("is idempotent: a second start does not arm a second loop", async () => { - jest.useFakeTimers(); - const { service, defaults } = createService({ - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - markDegraded: jest.fn(), - markRecovered: jest.fn(), - }, - networkService: { requestStatus: statusFor([]) }, - }); - - service.startRecoveryPolling(); - service.startRecoveryPolling(); - await runOneTick(30_000); - - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); - service.stop(); - }); - }); - - // ── init ────────────────────────────────────────────────────────────────── - - describe("init", () => { - it("resumes polling when the feature is enabled and the persisted state is degraded", async () => { - jest.useFakeTimers(); - const { service, defaults } = createService({ - egressStateRepository: { - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - recordProbe: jest.fn(), - markDegraded: jest.fn(), - markRecovered: jest.fn().mockResolvedValue(makeState()), - }, - }); - - await service.init(); - - expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: "Starting egress recovery polling" })); - service.stop(); - }); - - it("does nothing when the persisted state is ok", async () => { + it("removes the job without probing when the persisted state is no longer degraded", async () => { const { service, defaults } = createService(); - await service.init(); + await service.checkRecovery(); - expect(defaults.logger.info).not.toHaveBeenCalled(); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); + expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); }); - it("does nothing when the feature is disabled", async () => { - const { service, defaults } = createService({ - settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + it("lets errors propagate so the queue records the failure and retries", async () => { + const { service } = createService({ + egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }), }); - await service.init(); - - expect(defaults.egressStateRepository.findSingleton).not.toHaveBeenCalled(); + await expect(service.checkRecovery()).rejects.toThrow("db down"); }); }); }); From 08f104e283d45ab393ae67c1fe97aa095bc206a7 Mon Sep 17 00:00:00 2001 From: Craig Lawson Date: Tue, 15 Sep 2026 08:42:09 +0100 Subject: [PATCH 31/35] Address review on the egress recovery job (#3937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The egress job runs through WorkerHelper.getEgressRecoveryJob(), like the cleanup jobs, instead of a direct service call from the queue - Toggling egressCheckEnabled resets the egress state and removes any pending recovery job - Probes resolve and pass the proxy, so http targets honour the global proxy as monitors do - PingProvider keeps IPv6 addresses intact instead of truncating at the first colon - Egress targets are parsed by one function shared by the validator and the probe; malformed addresses such as unbalanced brackets are rejected - Notification id pattern drops the /i flag, which produced an invalid OpenAPI pattern - EgressState findSingleton reads before it writes, so a read no longer bumps updatedAt - "disabled" is a distinct assessment result and is cached like any other - Concurrent failures share an in-flight assessment for as long as it runs, then reuse the settled result for 5 s - The recovery job row is only removed if nobody rescheduled it since this run claimed it, so a concurrent new episode keeps its job - Deleting a notification removes it from egressNotifications 🔧 --- server/openapi.json | 5 +- .../src/api/controllers/settingsController.ts | 17 +- .../src/api/validation/settingsValidation.ts | 6 +- server/src/config/controllers.ts | 7 +- server/src/config/services.api.ts | 2 +- server/src/config/services.worker.ts | 6 +- .../egress-state.repository.interface.ts | 3 +- .../egress/egress-state.repository.mongo.ts | 12 + .../src/domain/egress/egress-state.service.ts | 17 +- server/src/domain/egress/egress.service.ts | 154 +++++++----- server/src/domain/egress/egress.type.ts | 2 + .../domain/jobs/job.repository.interface.ts | 3 + .../src/domain/jobs/job.repository.mongo.ts | 5 + .../notifications/notification.service.ts | 11 + server/src/service/network/PingProvider.ts | 12 +- server/src/utils/egressTarget.ts | 48 ++++ server/src/worker/worker.db-queue.ts | 6 +- server/src/worker/worker.helper.ts | 16 +- .../providers/network/pingProvider.test.ts | 18 ++ .../test/unit/services/dbQueueWorker.test.ts | 15 +- .../test/unit/services/egressService.test.ts | 231 +++++++++++++----- .../unit/services/egressStateService.test.ts | 34 ++- .../services/notificationsService.test.ts | 24 +- .../test/unit/services/workerHelper.test.ts | 25 +- server/test/unit/utils/egressTarget.test.ts | 49 ++++ .../validation/settingsValidation.test.ts | 29 ++- 26 files changed, 600 insertions(+), 157 deletions(-) create mode 100644 server/src/utils/egressTarget.ts create mode 100644 server/test/unit/utils/egressTarget.test.ts diff --git a/server/openapi.json b/server/openapi.json index d0ae81f26..32f5013a1 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -13303,8 +13303,7 @@ "items": { "type": "string", "minLength": 1, - "maxLength": 253, - "pattern": "^(https?:\\/\\/\\S+|[A-Za-z0-9.\\-:[\\]]+)$" + "maxLength": 253 }, "maxItems": 10 }, @@ -13317,7 +13316,7 @@ "type": "array", "items": { "type": "string", - "pattern": "^[a-f\\d]{24}$/i" + "pattern": "^[0-9a-fA-F]{24}$" } } } diff --git a/server/src/api/controllers/settingsController.ts b/server/src/api/controllers/settingsController.ts index d02c3886a..cd36467e2 100644 --- a/server/src/api/controllers/settingsController.ts +++ b/server/src/api/controllers/settingsController.ts @@ -6,6 +6,7 @@ import { AppError } from "@/utils/AppError.js"; import { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; import { IEmailService } from "@/service/emailService.js"; import { IProxiesService } from "@/domain/proxies/proxy.service.js"; +import { IEgressStateService } from "@/domain/egress/egress-state.service.js"; import { Settings } from "@/domain/app-settings/app-settings.type.js"; export interface ISettingsController { @@ -18,10 +19,17 @@ class SettingsController implements ISettingsController { private settingsService: ISettingsService; private emailService: IEmailService; private proxiesService: IProxiesService; - constructor(settingsService: ISettingsService, emailService: IEmailService, proxiesService: IProxiesService) { + private egressStateService: IEgressStateService; + constructor( + settingsService: ISettingsService, + emailService: IEmailService, + proxiesService: IProxiesService, + egressStateService: IEgressStateService + ) { this.settingsService = settingsService; this.emailService = emailService; this.proxiesService = proxiesService; + this.egressStateService = egressStateService; } buildAppSettings = async (dbSettings: Settings) => { @@ -70,7 +78,14 @@ class SettingsController implements ISettingsController { } } + const previousSettings = await this.settingsService.getDBSettings(); const updatedSettings = await this.settingsService.updateDbSettings(validatedBody); + + // Switching the egress check on or off starts from a clean state: no degraded episode, no pending recovery job + if (validatedBody.egressCheckEnabled !== undefined && validatedBody.egressCheckEnabled !== previousSettings.egressCheckEnabled) { + await this.egressStateService.reset(); + } + const returnSettings = await this.buildAppSettings(updatedSettings); return res.status(200).json({ success: true, diff --git a/server/src/api/validation/settingsValidation.ts b/server/src/api/validation/settingsValidation.ts index 33fe433fc..5bf635054 100644 --- a/server/src/api/validation/settingsValidation.ts +++ b/server/src/api/validation/settingsValidation.ts @@ -1,14 +1,16 @@ import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js"; import { MAX_EGRESS_POLL_INTERVAL_SECONDS, MAX_EGRESS_TARGETS, MIN_EGRESS_POLL_INTERVAL_SECONDS } from "@/domain/egress/egress.type.js"; +import { parseEgressTarget } from "@/utils/egressTarget.js"; import { z } from "zod"; // A reliability target is a bare host/IP (ICMP ping), host:port (TCP connect) or an http(s) URL. +// Parsed with the same function the probe uses, so nothing the validator accepts can fail to probe. const egressTargetValidation = z .string() .trim() .min(1) .max(253) - .regex(/^(https?:\/\/\S+|[A-Za-z0-9.\-:[\]]+)$/, "Invalid egress target"); + .refine((target) => parseEgressTarget(target) !== null, "Enter a hostname or IP address, host:port, or http(s) URL"); //**************************************** // Settings Validations @@ -55,7 +57,7 @@ export const updateAppSettingsBodyValidation = z egressCheckEnabled: z.boolean().optional(), egressCheckTargets: z.array(egressTargetValidation).max(MAX_EGRESS_TARGETS).optional(), // empty list falls back to DEFAULT_EGRESS_TARGETS egressPollIntervalSeconds: z.number().int().min(MIN_EGRESS_POLL_INTERVAL_SECONDS).max(MAX_EGRESS_POLL_INTERVAL_SECONDS).optional(), - egressNotifications: z.array(z.string().regex(/^[a-f\d]{24}$/i, "Invalid notification id")).optional(), + egressNotifications: z.array(z.string().regex(/^[0-9a-fA-F]{24}$/, "Invalid notification id")).optional(), }) .strip() .superRefine((body, ctx) => { diff --git a/server/src/config/controllers.ts b/server/src/config/controllers.ts index 7787a389e..ba98030bb 100644 --- a/server/src/config/controllers.ts +++ b/server/src/config/controllers.ts @@ -36,7 +36,12 @@ export const initializeControllers = (apiServices: ApiServices): InitializedCont return { authController: new AuthController(apiServices.userService), monitorController: new MonitorController(apiServices.monitorService, apiServices.notificationsService), - settingsController: new SettingsController(apiServices.settingsService, apiServices.emailService, apiServices.proxiesService), + settingsController: new SettingsController( + apiServices.settingsService, + apiServices.emailService, + apiServices.proxiesService, + apiServices.egressStateService + ), checkController: new CheckController(apiServices.checkService), geoCheckController: new GeoCheckController(apiServices.geoChecksService), inviteController: new InviteController(apiServices.inviteService), diff --git a/server/src/config/services.api.ts b/server/src/config/services.api.ts index e929fbfe8..68b722f0a 100644 --- a/server/src/config/services.api.ts +++ b/server/src/config/services.api.ts @@ -103,7 +103,7 @@ export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): A const tagsService = new TagsService(tagsRepository, monitorsRepository); const diagnosticService = new DiagnosticService(db); const proxiesService = new ProxiesService(proxiesRepository, monitorsRepository, settingsService); - const egressStateService = new EgressStateService(egressStateRepository); + const egressStateService = new EgressStateService(egressStateRepository, jobsRepository); return { ...shared, worker: jobScheduler, diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index b97bb56f3..e15e1cbf5 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -106,7 +106,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig const bufferService = new BufferService(logger, checkService, geoChecksService, dockerLogsService, settingsService, jobsRepository); const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository); const monitorStatusPolicy = new MonitorStatusPolicy(); - const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, logger); + const egressService = new EgressService(settingsService, egressStateRepository, jobsRepository, networkService, proxyResolver, logger); // *********************** // Reactors and dispatcher @@ -149,7 +149,8 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig checksRepository, incidentsRepository, geoChecksRepository, - dockerLogsRepository + dockerLogsRepository, + egressService ); const worker = await DBQueueWorker.create({ @@ -165,7 +166,6 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig geoCheckPipeline, dispatcher: reactorDispatcher, helper: workerHelper, - egressService, queueWorkersRepository, queueMode: envSettings.queueMode, queuePrimaryProcesses: envSettings.queuePrimaryProcesses, diff --git a/server/src/domain/egress/egress-state.repository.interface.ts b/server/src/domain/egress/egress-state.repository.interface.ts index 533aad656..c718d4078 100644 --- a/server/src/domain/egress/egress-state.repository.interface.ts +++ b/server/src/domain/egress/egress-state.repository.interface.ts @@ -2,9 +2,10 @@ import type { EgressProbeResult, EgressState } from "@/domain/egress/egress.type export interface IEgressStateRepository { // read - findSingleton(): Promise; // creates the "ok" singleton on first call + findSingleton(): Promise; // creates the "ok" singleton on first call, a plain read after that // update recordProbe(results: EgressProbeResult[], now: Date): Promise; + reset(): Promise; // back to "ok" with no episode; used when the feature is toggled // Atomic transitions. Each returns the updated state when this caller performed the transition, // or null when the state was already on the target side (another worker got there first). markDegraded(results: EgressProbeResult[], now: Date): Promise; diff --git a/server/src/domain/egress/egress-state.repository.mongo.ts b/server/src/domain/egress/egress-state.repository.mongo.ts index 606551e5a..a5bb110c3 100644 --- a/server/src/domain/egress/egress-state.repository.mongo.ts +++ b/server/src/domain/egress/egress-state.repository.mongo.ts @@ -27,6 +27,9 @@ class MongoEgressStateRepository implements IEgressStateRepository { }; findSingleton = async () => { + // Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write. + const existing = await EgressStateModel.findOne({ singleton: true }).lean(); + if (existing) return this.toEntity(existing); const doc = await EgressStateModel.findOneAndUpdate( { singleton: true }, { $setOnInsert: { singleton: true, status: "ok" } }, @@ -35,6 +38,15 @@ class MongoEgressStateRepository implements IEgressStateRepository { return this.toEntity(doc); }; + reset = async () => { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $set: { status: "ok", degradedSince: null, lastProbeAt: null, lastProbeResults: [] }, $setOnInsert: { singleton: true } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + }; + recordProbe = async (results: EgressProbeResult[], now: Date) => { const doc = await EgressStateModel.findOneAndUpdate( { singleton: true }, diff --git a/server/src/domain/egress/egress-state.service.ts b/server/src/domain/egress/egress-state.service.ts index 35800c7ac..c4701abe9 100644 --- a/server/src/domain/egress/egress-state.service.ts +++ b/server/src/domain/egress/egress-state.service.ts @@ -1,20 +1,31 @@ import { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; +import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import type { EgressState } from "@/domain/egress/egress.type.js"; export interface IEgressStateService { getState(): Promise; + reset(): Promise; } const SERVICE_NAME = "EgressStateService"; -// Read-only view of the instance-global egress state for the API process. Transitions are -// owned by the worker (egress.service.ts); this exists so the controller never touches the repository. +// API-process view of the instance-global egress state. Probe-driven transitions are owned by the +// worker (egress.service.ts); this exists so the controller never touches the repositories. export class EgressStateService implements IEgressStateService { static SERVICE_NAME = SERVICE_NAME; - constructor(private egressStateRepository: IEgressStateRepository) {} + constructor( + private egressStateRepository: IEgressStateRepository, + private jobsRepository: IJobsRepository + ) {} getState = async (): Promise => { return this.egressStateRepository.findSingleton(); }; + + // Called when the feature is switched on or off: drop any pending recovery job and start from "ok". + reset = async (): Promise => { + await this.jobsRepository.deleteByIdAndType(null, "egress"); + return this.egressStateRepository.reset(); + }; } diff --git a/server/src/domain/egress/egress.service.ts b/server/src/domain/egress/egress.service.ts index 9ca98f4b2..40a4d4cd2 100644 --- a/server/src/domain/egress/egress.service.ts +++ b/server/src/domain/egress/egress.service.ts @@ -2,50 +2,57 @@ import type { Monitor } from "@/domain/monitors/monitor.type.js"; import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js"; import type { IEgressStateRepository } from "@/domain/egress/egress-state.repository.interface.js"; import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; -import { jobId, type JobSeed } from "@/domain/jobs/job.type.js"; +import { jobId, type Job, type JobSeed } from "@/domain/jobs/job.type.js"; import type { INetworkService } from "@/service/networkService.js"; +import type { IProxyResolver } from "@/service/network/ProxyResolver.js"; import type { ILogger } from "@/utils/logger.js"; import { DEFAULT_EGRESS_POLL_INTERVAL_SECONDS, DEFAULT_EGRESS_TARGETS, + type EgressAssessment, type EgressProbeResult, type EgressStatus, } from "@/domain/egress/egress.type.js"; import { timeRequest } from "@/service/network/utils.js"; +import { parseEgressTarget, type EgressTarget } from "@/utils/egressTarget.js"; const SERVICE_NAME = "EgressService"; const PROBE_TIMEOUT_MS = 5000; -// When egress is down every monitor fails at once, so concurrent failures share one assessment -// rather than each probing. Same in-flight cache shape as ProxyResolver. -const ASSESSMENT_TTL_MS = PROBE_TIMEOUT_MS; +// How long a settled assessment answers for later failures. An assessment still in flight is shared +// for as long as it takes, so this only bounds how stale a reused answer can be. +const ASSESSMENT_REUSE_MS = 5000; // Global job row (refId null) that re-probes while degraded. Inserted when egress degrades, removed on recovery. -const RECOVERY_JOB_ID = jobId("egress", null); +const RECOVERY_JOB_TYPE = "egress" as const; +const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null); // Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs. const PROBE_MONITOR_ID = "egress-probe"; const PROBE_TEAM_ID = "system"; -// "[host]:port" or "host:port" where host has no colons (so a bare IPv6 address is not mistaken for host:port). -const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s]+)):(\d{1,5})$/; - export interface IEgressService { probeTargets(targets: string[]): Promise; // Called by the check producer when a check fails. Null when the feature is off or the assessment itself failed. assessAfterFailure(): Promise; // Handler for the "egress" job: re-probes while degraded and removes the job once egress is back. - checkRecovery(): Promise; + checkRecovery(job: Job): Promise; } +type CachedAssessment = { + value: Promise; + expiresAt: number | null; // null while the assessment is still in flight +}; + export class EgressService implements IEgressService { static SERVICE_NAME = SERVICE_NAME; - private assessment: { value: Promise; expiresAt: number } | null = null; + private assessment: CachedAssessment | null = null; constructor( private settingsService: ISettingsService, private egressStateRepository: IEgressStateRepository, private jobsRepository: IJobsRepository, private networkService: INetworkService, + private proxyResolver: IProxyResolver, private logger: ILogger ) {} @@ -54,46 +61,52 @@ export class EgressService implements IEgressService { // **************************** // Reuses the existing ping/port/http providers by dressing each target up as a minimal monitor. - private toProbeMonitor = (target: string): Monitor => { + // proxyMode "inherit" lets the resolver apply the global proxy to http probes, as it does for monitors. + private toProbeMonitor = (target: EgressTarget, raw: string): Monitor => { const base = { id: PROBE_MONITOR_ID, teamId: PROBE_TEAM_ID, - name: `Egress probe: ${target}`, + name: `Egress probe: ${raw}`, method: "GET", + proxyMode: "inherit", useAdvancedMatching: false, ignoreTlsErrors: false, customUpCodes: [], }; - if (/^https?:\/\//i.test(target)) { - return { ...base, type: "http", url: target } as unknown as Monitor; + switch (target.kind) { + case "http": + return { ...base, type: "http", url: target.url } as unknown as Monitor; + case "port": + return { ...base, type: "port", url: target.host, port: target.port } as unknown as Monitor; + case "ping": + return { ...base, type: "ping", url: target.host } as unknown as Monitor; } - - const hostPort = HOST_PORT_PATTERN.exec(target); - if (hostPort) { - const host = hostPort[1] ?? hostPort[2]; - return { ...base, type: "port", url: host, port: Number(hostPort[3]) } as unknown as Monitor; - } - - return { ...base, type: "ping", url: target } as unknown as Monitor; }; - private probeTarget = async (target: string): Promise => { - const monitor = this.toProbeMonitor(target); + private probeTarget = async (raw: string): Promise => { + const target = parseEgressTarget(raw); + if (!target) { + // Validation rejects these; a stored value can still be stale, and it must not count as reachable. + return { target: raw, reachable: false, responseTime: 0, message: "Invalid egress target" }; + } + + const monitor = this.toProbeMonitor(target, raw); let timer: NodeJS.Timeout | undefined; - const { response, responseTime, error } = await timeRequest(() => - Promise.race([ - this.networkService.requestStatus(monitor), + const { response, responseTime, error } = await timeRequest(async () => { + const proxyUrl = await this.proxyResolver.resolve(monitor); + return Promise.race([ + this.networkService.requestStatus(monitor, { proxyUrl }), new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`Egress probe timed out after ${PROBE_TIMEOUT_MS}ms`)), PROBE_TIMEOUT_MS); }), - ]).finally(() => clearTimeout(timer)) - ); + ]).finally(() => clearTimeout(timer)); + }); if (error || !response) { return { - target, + target: raw, reachable: false, responseTime, message: error instanceof Error ? error.message : error ? String(error) : "No response", @@ -101,7 +114,7 @@ export class EgressService implements IEgressService { } return { - target, + target: raw, reachable: response.status === true, responseTime: response.responseTime ?? responseTime, message: response.message, @@ -127,32 +140,43 @@ export class EgressService implements IEgressService { // Event-triggered assessment // **************************** - assessAfterFailure = (): Promise => { - const now = Date.now(); - if (this.assessment && this.assessment.expiresAt > now) { - return this.assessment.value; - } - // Cache the in-flight promise so concurrent failures share one probe - const value = this.assess(); - this.assessment = { value, expiresAt: now + ASSESSMENT_TTL_MS }; - // Don't cache an internal failure, so the next failing check retries - void value.then((status) => { - if (status === null && this.assessment?.value === value) this.assessment = null; - }); - return value; + assessAfterFailure = async (): Promise => { + const assessment = await this.sharedAssessment(); + return assessment === "disabled" ? null : assessment; }; - private assess = async (): Promise => { + // When egress is down every monitor fails at once. Concurrent failures share the in-flight assessment + // however long it takes, and a settled one answers for ASSESSMENT_REUSE_MS. Same shape as ProxyResolver's cache. + private sharedAssessment = (): Promise => { + const cached = this.assessment; + if (cached && (cached.expiresAt === null || cached.expiresAt > Date.now())) { + return cached.value; + } + + const entry: CachedAssessment = { value: this.assess(), expiresAt: null }; + this.assessment = entry; + void entry.value.then((result) => { + if (this.assessment !== entry) return; + if (result === null) { + this.assessment = null; // an internal failure is not cached, so the next failing check retries + } else { + entry.expiresAt = Date.now() + ASSESSMENT_REUSE_MS; + } + }); + return entry.value; + }; + + private assess = async (): Promise => { try { const settings = await this.settingsService.getDBSettings(); if (!settings.egressCheckEnabled) { - return null; + return "disabled"; } const state = await this.egressStateRepository.findSingleton(); if (state.status === "degraded") { - // Recovery is detected by the scheduled job. Make sure it exists in case its row was lost. - await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds); + // Recovery is detected by the scheduled job. Make sure the row exists without touching its schedule. + await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, false); return "degraded"; } @@ -173,7 +197,8 @@ export class EgressService implements IEgressService { method: "assessAfterFailure", details: { results }, }); - await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds); + // The worker that performed the transition owns the job's schedule; anyone else only makes sure the row exists. + await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, degraded !== null); return "degraded"; } catch (error: unknown) { // The egress check must never break check production; treat an internal failure as "unknown" and carry on. @@ -191,35 +216,42 @@ export class EgressService implements IEgressService { // Recovery job // **************************** - // upsertJob only sets nextScheduledAt on insert, so re-arming while degraded never pushes a pending run back. - private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined) => { + // reschedule=true sets nextScheduledAt (start of an episode); false only inserts the row if it is missing, + // so re-arming while degraded never pushes a pending run back. + private scheduleRecoveryCheck = async (pollIntervalSeconds: number | undefined, reschedule: boolean) => { const intervalMs = this.toIntervalMs(pollIntervalSeconds); const seed: JobSeed = { id: RECOVERY_JOB_ID, - type: "egress", + type: RECOVERY_JOB_TYPE, refId: null, isActive: true, nextScheduledAt: Date.now() + intervalMs, intervalMs, }; - await this.jobsRepository.upsertJob(seed); + if (reschedule) { + await this.jobsRepository.upsertCleanupJob(seed); // global-row upsert that also sets the schedule + } else { + await this.jobsRepository.upsertJob(seed); + } }; - private removeRecoveryJob = async () => { - await this.jobsRepository.deleteByIdAndType(null, "egress"); + // Removes the row only if nobody has rescheduled it since this run claimed it. Another worker may have seen + // egress fail again in the meantime and started a new episode; its schedule must survive. + private releaseRecoveryJob = async (job: Job) => { + await this.jobsRepository.deleteGlobalJobIfUnchanged(RECOVERY_JOB_TYPE, job.nextScheduledAt); }; // Runs on the queue at the configured interval while degraded. Errors propagate so the queue records // the failure and retries with its usual backoff. - checkRecovery = async (): Promise => { + checkRecovery = async (job: Job): Promise => { + const settings = await this.settingsService.getDBSettings(); const state = await this.egressStateRepository.findSingleton(); - if (state.status !== "degraded") { - // Another worker recorded the recovery, or the state was reset; nothing left to poll for. - await this.removeRecoveryJob(); + if (!settings.egressCheckEnabled || state.status !== "degraded") { + // Feature switched off, recovery recorded by another worker, or state reset: nothing left to poll for. + await this.releaseRecoveryJob(job); return; } - const settings = await this.settingsService.getDBSettings(); const results = await this.probeTargets(this.resolveTargets(settings.egressCheckTargets)); const now = new Date(); @@ -229,7 +261,7 @@ export class EgressService implements IEgressService { } const recovered = await this.egressStateRepository.markRecovered(results, now); - await this.removeRecoveryJob(); + await this.releaseRecoveryJob(job); if (!recovered) { // Another process performed the transition. return; diff --git a/server/src/domain/egress/egress.type.ts b/server/src/domain/egress/egress.type.ts index 386116ef7..49c8d462f 100644 --- a/server/src/domain/egress/egress.type.ts +++ b/server/src/domain/egress/egress.type.ts @@ -3,6 +3,8 @@ export const EgressStatuses = ["ok", "degraded"] as const; export type EgressStatus = (typeof EgressStatuses)[number]; +// Outcome of one assessment: a status, or "disabled" when the feature is switched off (distinct from an internal failure). +export type EgressAssessment = EgressStatus | "disabled"; // Sensible defaults for a fresh install. Bare hosts are ICMP-pinged, host:port is a TCP connect, // http(s):// URLs are fetched. Any one target being reachable means egress is fine. diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index a352c0888..6fe6b23b9 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -61,6 +61,9 @@ export interface IJobsRepository { // Delete a single job of type. refId null addresses a global row deleteByIdAndType(refId: string | null, type: JobType): Promise; + // Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged) + deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise; + // Delete jobs that reference monitors that no longer exist deleteByMonitorIdsNotIn(monitorIds: string[]): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index ce4143b2d..d545b0241 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -278,6 +278,11 @@ class MongoJobsRepository implements IJobsRepository { return res.deletedCount > 0; }; + deleteGlobalJobIfUnchanged = async (type: JobType, nextScheduledAt: number) => { + const res = await JobModel.deleteOne({ _id: jobId(type, null), nextScheduledAt }); + return res.deletedCount > 0; + }; + deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise => { // remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs) const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } }); diff --git a/server/src/domain/notifications/notification.service.ts b/server/src/domain/notifications/notification.service.ts index 61d8d09e3..e270e76d4 100644 --- a/server/src/domain/notifications/notification.service.ts +++ b/server/src/domain/notifications/notification.service.ts @@ -178,7 +178,18 @@ export class NotificationsService implements INotificationsService { deleteById = async (id: string, teamId: string): Promise => { await this.monitorsRepository.removeNotificationFromMonitors(id); + await this.removeFromEgressNotifications(id); const deleted = await this.notificationsRepository.deleteById(id, teamId); return deleted; }; + + // App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles + private removeFromEgressNotifications = async (id: string) => { + const settings = await this.settingsService.getDBSettings(); + const current = settings.egressNotifications ?? []; + const remaining = current.filter((notificationId) => notificationId !== id); + if (remaining.length !== current.length) { + await this.settingsService.updateDbSettings({ egressNotifications: remaining }); + } + }; } diff --git a/server/src/service/network/PingProvider.ts b/server/src/service/network/PingProvider.ts index 53f6aad0d..9194ed9d2 100644 --- a/server/src/service/network/PingProvider.ts +++ b/server/src/service/network/PingProvider.ts @@ -4,6 +4,7 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js"; import { MonitorStatusResponse } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import ping from "ping"; +import net from "net"; import { timeRequest } from "@/service/network/utils.js"; const SERVICE_NAME = "PingProvider"; @@ -19,10 +20,13 @@ export class PingProvider implements IStatusProvider { } private sanitizeHost(url: string): string { - return url - .replace(/^https?:\/\//, "") - .replace(/\/.*$/, "") - .replace(/:.*/, ""); + const host = url.replace(/^https?:\/\//, "").replace(/\/.*$/, ""); + // "[2001:db8::1]" or "[2001:db8::1]:53": the address is inside the brackets + const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host); + if (bracketed) return bracketed[1]!; + // A bare IPv6 address contains colons that are not a port separator + if (net.isIPv6(host)) return host; + return host.replace(/:.*/, ""); } async handle(monitor: Monitor): Promise> { diff --git a/server/src/utils/egressTarget.ts b/server/src/utils/egressTarget.ts new file mode 100644 index 000000000..240ad6c04 --- /dev/null +++ b/server/src/utils/egressTarget.ts @@ -0,0 +1,48 @@ +import { z } from "zod"; +import { dnsHostnameRegex } from "@/api/validation/shared.js"; + +// A reliability target for the egress self-check, in one of three shapes. Parsed here so the +// settings validator and the probe agree on exactly what is accepted. +export type EgressTarget = + | { kind: "http"; url: string } // http(s):// URL, fetched + | { kind: "port"; host: string; port: number } // host:port or [ipv6]:port, TCP connect + | { kind: "ping"; host: string }; // bare hostname or IP address, ICMP + +// "[ipv6]:port" or "host:port" where host has no colons, so a bare IPv6 address is never read as host:port. +const HOST_PORT_PATTERN = /^(?:\[([^\]]+)\]|([^:/\s[\]]+)):(\d{1,5})$/; +const BRACKETED_PATTERN = /^\[([^\]]+)\]$/; + +const isIpv6 = (value: string): boolean => z.ipv6().safeParse(value).success; +const isHostOrIpv4 = (value: string): boolean => value === "localhost" || z.ipv4().safeParse(value).success || dnsHostnameRegex.test(value); + +const parseHttp = (target: string): EgressTarget | null => { + try { + const { protocol, hostname } = new URL(target); + return (protocol === "http:" || protocol === "https:") && hostname.length > 0 ? { kind: "http", url: target } : null; + } catch { + return null; + } +}; + +export const parseEgressTarget = (raw: string): EgressTarget | null => { + const target = raw.trim(); + if (target.length === 0) return null; + + if (/^https?:\/\//i.test(target)) return parseHttp(target); + + const hostPort = HOST_PORT_PATTERN.exec(target); + if (hostPort) { + const port = Number(hostPort[3]); + if (port < 1 || port > 65535) return null; + const bracketedHost = hostPort[1]; + if (bracketedHost !== undefined) return isIpv6(bracketedHost) ? { kind: "port", host: bracketedHost, port } : null; + const host = hostPort[2]!; + return isHostOrIpv4(host) ? { kind: "port", host, port } : null; + } + + const bracketed = BRACKETED_PATTERN.exec(target); + if (bracketed) return isIpv6(bracketed[1]!) ? { kind: "ping", host: bracketed[1]! } : null; + + if (isIpv6(target) || isHostOrIpv4(target)) return { kind: "ping", host: target }; + return null; +}; diff --git a/server/src/worker/worker.db-queue.ts b/server/src/worker/worker.db-queue.ts index 49f630b01..c076151ef 100644 --- a/server/src/worker/worker.db-queue.ts +++ b/server/src/worker/worker.db-queue.ts @@ -10,7 +10,6 @@ import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js"; import { ICheckPipeline } from "@/worker/worker.check-pipeline.js"; import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js"; import { IWorkerHelper } from "@/worker/worker.helper.js"; -import { IEgressService } from "@/domain/egress/egress.service.js"; import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js"; import { WORKER_STALE_MS } from "@/domain/queue-workers/queue-worker.model.js"; import { QueueMode } from "@/domain/app-settings/app-settings.type.js"; @@ -45,7 +44,6 @@ export interface DBQueueWorkerDependencies { geoCheckPipeline: ICheckPipeline; dispatcher: IReactorDispatcher; helper: IWorkerHelper; - egressService: IEgressService; queueWorkersRepository: IQueueWorkersRepository; // forwarded queueMode: QueueMode; // forwarded queuePrimaryProcesses: boolean; // consumed immediately @@ -74,7 +72,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { private dispatcher: IReactorDispatcher; private geoCheckPipeline: ICheckPipeline; private helper: IWorkerHelper; - private egressService: IEgressService; private bufferService: IBufferService; private isDbConnected: () => boolean; @@ -95,7 +92,6 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { this.dispatcher = dependencies.dispatcher; this.geoCheckPipeline = dependencies.geoCheckPipeline; this.helper = dependencies.helper; - this.egressService = dependencies.egressService; this.bufferService = dependencies.bufferService; this.isDbConnected = dependencies.isDbConnected; } @@ -182,7 +178,7 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker { await this.helper.getCleanupRetentionJob()(); // Get job and execute break; case "egress": - await this.egressService.checkRecovery(); // Re-probe while degraded; removes its own row on recovery + await this.helper.getEgressRecoveryJob()(job); // Re-probe while degraded; removes its own row on recovery break; } diff --git a/server/src/worker/worker.helper.ts b/server/src/worker/worker.helper.ts index d8a35e0b2..9e3a6aa8e 100644 --- a/server/src/worker/worker.helper.ts +++ b/server/src/worker/worker.helper.ts @@ -11,10 +11,13 @@ import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"; import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js"; import { ILogger } from "@/utils/logger.js"; import { IDockerLogsRepository } from "@/domain/docker/docker-log.repository.interface.js"; +import { IEgressService } from "@/domain/egress/egress.service.js"; +import type { Job } from "@/domain/jobs/job.type.js"; export interface IWorkerHelper { getCleanupOrphanedJob(): () => Promise; getCleanupRetentionJob(): () => Promise; + getEgressRecoveryJob(): (job: Job) => Promise; } export interface MonitorActionDecision { @@ -45,6 +48,7 @@ export class WorkerHelper implements IWorkerHelper { private incidentsRepository: IIncidentsRepository; private geoChecksRepository: IGeoChecksRepository; private dockerLogsRepository: IDockerLogsRepository; + private egressService: IEgressService; constructor( logger: ILogger, @@ -57,7 +61,8 @@ export class WorkerHelper implements IWorkerHelper { checksRepository: IChecksRepository, incidentsRepository: IIncidentsRepository, geoChecksRepository: IGeoChecksRepository, - dockerLogsRepository: IDockerLogsRepository + dockerLogsRepository: IDockerLogsRepository, + egressService: IEgressService ) { this.logger = logger; this.checkService = checkService; @@ -70,8 +75,17 @@ export class WorkerHelper implements IWorkerHelper { this.incidentsRepository = incidentsRepository; this.geoChecksRepository = geoChecksRepository; this.dockerLogsRepository = dockerLogsRepository; + this.egressService = egressService; } + // Runs while instance egress is degraded; the service removes the row once egress is back. + // Errors propagate so the queue records the failure and retries with its usual backoff. + getEgressRecoveryJob = () => { + return async (job: Job) => { + await this.egressService.checkRecovery(job); + }; + }; + getCleanupOrphanedJob = () => { return async () => { try { diff --git a/server/test/unit/providers/network/pingProvider.test.ts b/server/test/unit/providers/network/pingProvider.test.ts index 04e4ad74e..6b4129850 100644 --- a/server/test/unit/providers/network/pingProvider.test.ts +++ b/server/test/unit/providers/network/pingProvider.test.ts @@ -110,6 +110,24 @@ describe("PingProvider", () => { expect(mockPing.promise.probe).toHaveBeenCalledWith("example.com"); }); + it("keeps a bare IPv6 address intact", async () => { + const mockPing = createMockPing(); + const provider = new PingProvider(mockPing as any); + + await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" })); + + expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111"); + }); + + it("unwraps a bracketed IPv6 address and drops its port", async () => { + const mockPing = createMockPing(); + const provider = new PingProvider(mockPing as any); + + await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" })); + + expect(mockPing.promise.probe).toHaveBeenCalledWith("2606:4700:4700::1111"); + }); + it("throws AppError when url is missing", async () => { const provider = new PingProvider(createMockPing() as any); diff --git a/server/test/unit/services/dbQueueWorker.test.ts b/server/test/unit/services/dbQueueWorker.test.ts index 75acaa063..a2d532c16 100644 --- a/server/test/unit/services/dbQueueWorker.test.ts +++ b/server/test/unit/services/dbQueueWorker.test.ts @@ -113,8 +113,8 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses const helper = { getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + getEgressRecoveryJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), }; - const egressService = { checkRecovery: jest.fn().mockResolvedValue(undefined) }; const queueWorkersRepository = { upsert: jest.fn().mockResolvedValue(undefined), deleteById: jest.fn().mockResolvedValue(undefined), @@ -135,7 +135,6 @@ const createWorker = (overrides?: { queueMode?: QueueMode; queuePrimaryProcesses geoCheckPipeline, dispatcher, helper, - egressService, queueWorkersRepository, logger, ...overrides?.mocks, @@ -442,11 +441,17 @@ describe("DBQueueWorker", () => { expect(retentionFn).toHaveBeenCalled(); }); - it("egress job runs the egress recovery check", async () => { + it("egress job runs the helper's egress recovery function with the claimed job", async () => { const job = makeJob({ id: "egress", type: "egress", refId: null, intervalMs: 30000 }); - const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) } } }); + const egressFn = jest.fn().mockResolvedValue(undefined); + const helper = { + getCleanupOrphanedJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + getCleanupRetentionJob: jest.fn().mockReturnValue(jest.fn().mockResolvedValue(undefined)), + getEgressRecoveryJob: jest.fn().mockReturnValue(egressFn), + }; + const { mocks } = await start({ mocks: { jobsRepository: { ...createWorker().mocks.jobsRepository, claimDueBatch: claimOnce(job) }, helper } }); - expect(mocks.egressService.checkRecovery).toHaveBeenCalledTimes(1); + expect(egressFn).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", type: "egress" })); expect(mocks.jobsRepository.recordSuccess).toHaveBeenCalledWith(job.id, job.nextScheduledAt, job.intervalMs, expect.any(Number)); }); diff --git a/server/test/unit/services/egressService.test.ts b/server/test/unit/services/egressService.test.ts index ffe124495..8aee753d7 100644 --- a/server/test/unit/services/egressService.test.ts +++ b/server/test/unit/services/egressService.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, jest } from "@jest/globals"; import { EgressService } from "../../../src/domain/egress/egress.service.ts"; import type { EgressState } from "../../../src/domain/egress/egress.type.ts"; +import type { Job } from "../../../src/domain/jobs/job.type.ts"; import { createMockLogger } from "../../helpers/createMockLogger.ts"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -25,6 +26,23 @@ const makeSettings = (overrides?: Record) => ({ ...overrides, }); +// The claimed "egress" job row as the queue hands it to the handler +const makeJob = (overrides?: Partial): Job => ({ + id: "egress", + type: "egress", + refId: null, + isActive: true, + nextScheduledAt: 1_000, + intervalMs: 30_000, + lockedBy: "worker-1", + lockedUntil: 61_000, + runCount: 0, + failCount: 0, + lastFinishedAt: null, + lastFailReason: null, + ...overrides, +}); + // requestStatus mock that answers per target url const statusFor = (reachableTargets: string[]) => jest.fn().mockImplementation(async (monitor: any) => ({ @@ -37,6 +55,15 @@ const statusFor = (reachableTargets: string[]) => responseTime: 12, })); +const degradedRepository = (overrides?: Record) => ({ + findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), + recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), + markDegraded: jest.fn().mockResolvedValue(null), + markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })), + reset: jest.fn(), + ...overrides, +}); + const createService = (overrides?: Record) => { const defaults = { settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings()) }, @@ -45,12 +72,15 @@ const createService = (overrides?: Record) => { recordProbe: jest.fn().mockResolvedValue(makeState()), markDegraded: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z" })), + reset: jest.fn(), }, jobsRepository: { upsertJob: jest.fn().mockResolvedValue(true), - deleteByIdAndType: jest.fn().mockResolvedValue(true), + upsertCleanupJob: jest.fn().mockResolvedValue(true), + deleteGlobalJobIfUnchanged: jest.fn().mockResolvedValue(true), }, networkService: { requestStatus: statusFor(["1.1.1.1", "8.8.8.8"]) }, + proxyResolver: { resolve: jest.fn().mockResolvedValue(undefined) }, logger: createMockLogger(), ...overrides, }; @@ -59,18 +89,14 @@ const createService = (overrides?: Record) => { defaults.egressStateRepository as any, defaults.jobsRepository as any, defaults.networkService as any, + defaults.proxyResolver as any, defaults.logger as any ); return { service, defaults }; }; -const degradedRepository = (overrides?: Record) => ({ - findSingleton: jest.fn().mockResolvedValue(makeState({ status: "degraded", degradedSince: "2026-01-01T10:00:00.000Z" })), - recordProbe: jest.fn().mockResolvedValue(makeState({ status: "degraded" })), - markDegraded: jest.fn().mockResolvedValue(null), - markRecovered: jest.fn().mockResolvedValue(makeState({ lastRecoveredAt: "2026-01-01T10:05:00.000Z", degradedSince: "2026-01-01T10:00:00.000Z" })), - ...overrides, -}); +const probedUrls = (defaults: { networkService: { requestStatus: unknown } }) => + (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url); // ── Tests ──────────────────────────────────────────────────────────────────── @@ -85,7 +111,7 @@ describe("EgressService", () => { it("maps a bare host to a ping monitor, host:port to a port monitor and a URL to an http monitor", async () => { const { service, defaults } = createService(); - await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443"]); + await service.probeTargets(["1.1.1.1", "8.8.8.8:53", "https://example.com/health", "[2606:4700::1111]:443", "2606:4700::1111"]); const monitors = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => call[0] as any); expect(monitors).toEqual([ @@ -93,9 +119,22 @@ describe("EgressService", () => { expect.objectContaining({ type: "port", url: "8.8.8.8", port: 53 }), expect.objectContaining({ type: "http", url: "https://example.com/health", method: "GET", useAdvancedMatching: false }), expect.objectContaining({ type: "port", url: "2606:4700::1111", port: 443 }), + expect.objectContaining({ type: "ping", url: "2606:4700::1111" }), ]); }); + it("passes the resolved proxy to the provider so http probes honour the global proxy", async () => { + const resolve = jest.fn().mockResolvedValue("http://proxy.internal:3128"); + const { service, defaults } = createService({ proxyResolver: { resolve } }); + + await service.probeTargets(["https://example.com/health"]); + + expect(resolve).toHaveBeenCalledWith(expect.objectContaining({ type: "http", proxyMode: "inherit" })); + expect(defaults.networkService.requestStatus).toHaveBeenCalledWith(expect.objectContaining({ type: "http" }), { + proxyUrl: "http://proxy.internal:3128", + }); + }); + it("reports reachable only when the provider says status is true", async () => { const { service } = createService({ networkService: { requestStatus: statusFor(["1.1.1.1"]) } }); @@ -107,6 +146,15 @@ describe("EgressService", () => { ]); }); + it("counts an unparseable stored target as unreachable without probing", async () => { + const { service, defaults } = createService(); + + const results = await service.probeTargets(["[1.1.1.1"]); + + expect(results).toEqual([expect.objectContaining({ target: "[1.1.1.1", reachable: false, message: "Invalid egress target" })]); + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + }); + it("counts a thrown provider error as unreachable", async () => { const { service } = createService({ networkService: { requestStatus: jest.fn().mockRejectedValue(new Error("ENETUNREACH")) } }); @@ -154,9 +202,11 @@ describe("EgressService", () => { expect.any(Date) ); expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled(); + expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled(); }); - it("marks degraded, warns, inserts the recovery job and returns degraded when every target is unreachable", async () => { + it("marks degraded, warns, schedules the recovery job and returns degraded when every target is unreachable", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) } }); const result = await service.assessAfterFailure(); @@ -168,7 +218,8 @@ describe("EgressService", () => { ); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") })); - expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith({ + // The worker that performed the transition sets the schedule + expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({ id: "egress", type: "egress", refId: null, @@ -176,9 +227,26 @@ describe("EgressService", () => { nextScheduledAt: expect.any(Number), intervalMs: 30_000, }); + expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled(); }); - it("returns degraded without probing when the persisted state is already degraded, re-arming the recovery job", async () => { + it("only makes sure the job row exists when another worker performed the transition", async () => { + const { service, defaults } = createService({ + networkService: { requestStatus: statusFor([]) }, + egressStateRepository: degradedRepository({ + findSingleton: jest.fn().mockResolvedValue(makeState()), + markDegraded: jest.fn().mockResolvedValue(null), + }), + }); + + const result = await service.assessAfterFailure(); + + expect(result).toBe("degraded"); + expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 })); + expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled(); + }); + + it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => { const { service, defaults } = createService({ egressStateRepository: degradedRepository() }); const result = await service.assessAfterFailure(); @@ -188,6 +256,7 @@ describe("EgressService", () => { expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 })); + expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled(); }); it("uses the default poll interval for the recovery job when the setting is missing or invalid", async () => { @@ -198,41 +267,7 @@ describe("EgressService", () => { await service.assessAfterFailure(); - expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 })); - }); - - it("shares one probe between concurrent failures", async () => { - const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); - - const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]); - - expect(results).toEqual(["ok", "ok", "ok"]); - expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1); - expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); - }); - - it("reuses a settled assessment within the cache window and probes again after it", async () => { - jest.useFakeTimers(); - const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); - - await service.assessAfterFailure(); - await jest.advanceTimersByTimeAsync(1000); - await service.assessAfterFailure(); - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); - - await jest.advanceTimersByTimeAsync(5000); - await service.assessAfterFailure(); - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); - }); - - it("does not cache an internal failure", async () => { - const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings()); - const { service, defaults } = createService({ settingsService: { getDBSettings } }); - - expect(await service.assessAfterFailure()).toBeNull(); - expect(await service.assessAfterFailure()).toBe("ok"); - expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ intervalMs: 30_000 })); }); it("falls back to the default targets when the configured list is empty", async () => { @@ -242,8 +277,7 @@ describe("EgressService", () => { await service.assessAfterFailure(); - const urls = (defaults.networkService.requestStatus as jest.Mock).mock.calls.map((call) => (call[0] as any).url); - expect(urls).toEqual(["1.1.1.1", "8.8.8.8"]); + expect(probedUrls(defaults)).toEqual(["1.1.1.1", "8.8.8.8"]); }); it("returns null and logs when the assessment itself fails", async () => { @@ -258,34 +292,97 @@ describe("EgressService", () => { }); }); + // ── shared assessment ───────────────────────────────────────────────────── + + describe("shared assessment", () => { + it("shares one probe between concurrent failures", async () => { + const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); + + const results = await Promise.all([service.assessAfterFailure(), service.assessAfterFailure(), service.assessAfterFailure()]); + + expect(results).toEqual(["ok", "ok", "ok"]); + expect(defaults.settingsService.getDBSettings).toHaveBeenCalledTimes(1); + expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one call per target, once + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + }); + + it("keeps sharing an in-flight assessment that outlives the reuse window", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ networkService: { requestStatus: jest.fn().mockReturnValue(new Promise(() => {})) } }); + + const first = service.assessAfterFailure(); + await jest.advanceTimersByTimeAsync(4_900); // probes still hanging, past the 5 s reuse window from the first call + const second = service.assessAfterFailure(); + await jest.advanceTimersByTimeAsync(200); // probe timeout fires, both settle + + expect(await Promise.all([first, second])).toEqual(["degraded", "degraded"]); + expect(defaults.networkService.requestStatus).toHaveBeenCalledTimes(2); // one round of two targets, not two rounds + }); + + it("reuses a settled assessment within the reuse window and probes again after it", async () => { + jest.useFakeTimers(); + const { service, defaults } = createService({ networkService: { requestStatus: statusFor(["8.8.8.8"]) } }); + + await service.assessAfterFailure(); + await jest.advanceTimersByTimeAsync(1_000); + await service.assessAfterFailure(); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + + await jest.advanceTimersByTimeAsync(5_000); + await service.assessAfterFailure(); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(2); + }); + + it("reuses a disabled result instead of re-reading settings for every failure", async () => { + const getDBSettings = jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })); + const { service } = createService({ settingsService: { getDBSettings } }); + + expect(await service.assessAfterFailure()).toBeNull(); + expect(await service.assessAfterFailure()).toBeNull(); + + expect(getDBSettings).toHaveBeenCalledTimes(1); + }); + + it("does not cache an internal failure", async () => { + const getDBSettings = jest.fn().mockRejectedValueOnce(new Error("db down")).mockResolvedValue(makeSettings()); + const { service, defaults } = createService({ settingsService: { getDBSettings } }); + + expect(await service.assessAfterFailure()).toBeNull(); + expect(await service.assessAfterFailure()).toBe("ok"); + expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); + }); + }); + // ── checkRecovery (the "egress" job) ────────────────────────────────────── describe("checkRecovery", () => { - it("marks recovered, removes the job and logs when a target becomes reachable", async () => { + it("marks recovered, releases the job and logs when a target becomes reachable", async () => { const { service, defaults } = createService({ egressStateRepository: degradedRepository(), networkService: { requestStatus: statusFor(["1.1.1.1"]) }, }); + const job = makeJob({ nextScheduledAt: 123_456 }); - await service.checkRecovery(); + await service.checkRecovery(job); expect(defaults.egressStateRepository.markRecovered).toHaveBeenCalledWith( [expect.objectContaining({ target: "1.1.1.1", reachable: true }), expect.objectContaining({ target: "8.8.8.8", reachable: false })], expect.any(Date) ); - expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); + // Conditional on the schedule this run claimed, so a concurrent new episode keeps its job + expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 123_456); expect(defaults.logger.info).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") })); }); - it("removes the job but does not log a recovery when another process already recorded it", async () => { + it("releases the job but does not log a recovery when another process already recorded it", async () => { const { service, defaults } = createService({ egressStateRepository: degradedRepository({ markRecovered: jest.fn().mockResolvedValue(null) }), networkService: { requestStatus: statusFor(["1.1.1.1"]) }, }); - await service.checkRecovery(); + await service.checkRecovery(makeJob()); - expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); + expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000); expect(defaults.logger.info).not.toHaveBeenCalledWith( expect.objectContaining({ message: expect.stringContaining("Instance egress recovered") }) ); @@ -297,21 +394,33 @@ describe("EgressService", () => { networkService: { requestStatus: statusFor([]) }, }); - await service.checkRecovery(); + await service.checkRecovery(makeJob()); expect(defaults.egressStateRepository.recordProbe).toHaveBeenCalledTimes(1); expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); - expect(defaults.jobsRepository.deleteByIdAndType).not.toHaveBeenCalled(); + expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).not.toHaveBeenCalled(); }); - it("removes the job without probing when the persisted state is no longer degraded", async () => { + it("releases the job without probing when the persisted state is no longer degraded", async () => { const { service, defaults } = createService(); - await service.checkRecovery(); + await service.checkRecovery(makeJob()); expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); expect(defaults.egressStateRepository.markRecovered).not.toHaveBeenCalled(); - expect(defaults.jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); + expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000); + }); + + it("releases the job without probing when the feature has been disabled mid-episode", async () => { + const { service, defaults } = createService({ + settingsService: { getDBSettings: jest.fn().mockResolvedValue(makeSettings({ egressCheckEnabled: false })) }, + egressStateRepository: degradedRepository(), + }); + + await service.checkRecovery(makeJob()); + + expect(defaults.networkService.requestStatus).not.toHaveBeenCalled(); + expect(defaults.jobsRepository.deleteGlobalJobIfUnchanged).toHaveBeenCalledWith("egress", 1_000); }); it("lets errors propagate so the queue records the failure and retries", async () => { @@ -319,7 +428,7 @@ describe("EgressService", () => { egressStateRepository: degradedRepository({ findSingleton: jest.fn().mockRejectedValue(new Error("db down")) }), }); - await expect(service.checkRecovery()).rejects.toThrow("db down"); + await expect(service.checkRecovery(makeJob())).rejects.toThrow("db down"); }); }); }); diff --git a/server/test/unit/services/egressStateService.test.ts b/server/test/unit/services/egressStateService.test.ts index 85c32f32d..a75c0de1f 100644 --- a/server/test/unit/services/egressStateService.test.ts +++ b/server/test/unit/services/egressStateService.test.ts @@ -9,12 +9,18 @@ const createEgressStateRepo = () => ({ recordProbe: jest.fn(), markDegraded: jest.fn(), markRecovered: jest.fn(), + reset: jest.fn(), +}); + +const createJobsRepo = () => ({ + deleteByIdAndType: jest.fn().mockResolvedValue(true), }); const createService = () => { const egressStateRepository = createEgressStateRepo(); - const service = new EgressStateService(egressStateRepository as any); - return { service, egressStateRepository }; + const jobsRepository = createJobsRepo(); + const service = new EgressStateService(egressStateRepository as any, jobsRepository as any); + return { service, egressStateRepository, jobsRepository }; }; const makeState = (overrides?: Partial): EgressState => ({ @@ -67,4 +73,28 @@ describe("EgressStateService", () => { await expect(service.getState()).rejects.toThrow("db down"); }); }); + + describe("reset", () => { + it("removes any pending recovery job and returns the state to ok", async () => { + const { service, egressStateRepository, jobsRepository } = createService(); + const reset = makeState(); + (egressStateRepository.reset as jest.Mock).mockResolvedValue(reset); + + const result = await service.reset(); + + expect(jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); + expect(egressStateRepository.reset).toHaveBeenCalledTimes(1); + expect(result).toEqual(reset); + }); + + it("does not touch the probe-driven transitions", async () => { + const { service, egressStateRepository } = createService(); + (egressStateRepository.reset as jest.Mock).mockResolvedValue(makeState()); + + await service.reset(); + + expect(egressStateRepository.markDegraded).not.toHaveBeenCalled(); + expect(egressStateRepository.markRecovered).not.toHaveBeenCalled(); + }); + }); }); diff --git a/server/test/unit/services/notificationsService.test.ts b/server/test/unit/services/notificationsService.test.ts index 8e1633803..f2462866b 100644 --- a/server/test/unit/services/notificationsService.test.ts +++ b/server/test/unit/services/notificationsService.test.ts @@ -26,8 +26,10 @@ const createMonitorsRepo = () => ({ removeNotificationFromMonitors: jest.fn(), }); -const createSettingsService = (clientHost = "https://app.example.com") => ({ +const createSettingsService = (clientHost = "https://app.example.com", egressNotifications: string[] = []) => ({ getSettings: jest.fn().mockReturnValue({ clientHost }), + getDBSettings: jest.fn().mockResolvedValue({ egressNotifications }), + updateDbSettings: jest.fn().mockResolvedValue({ egressNotifications: [] }), }); const createMessageBuilder = () => ({ @@ -371,5 +373,25 @@ describe("NotificationsService", () => { expect(result).toBe(deleted); expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1"); }); + + it("removes the notification from the egress recovered recipients in app settings", async () => { + const settingsService = createSettingsService("https://app.example.com", ["notif-0", "notif-1"]); + const { service, notificationsRepository } = createService({ settingsService }); + (notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification()); + + await service.deleteById("notif-1", "team-1"); + + expect(settingsService.updateDbSettings).toHaveBeenCalledWith({ egressNotifications: ["notif-0"] }); + }); + + it("leaves app settings alone when the notification is not an egress recipient", async () => { + const settingsService = createSettingsService("https://app.example.com", ["notif-0"]); + const { service, notificationsRepository } = createService({ settingsService }); + (notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification()); + + await service.deleteById("notif-1", "team-1"); + + expect(settingsService.updateDbSettings).not.toHaveBeenCalled(); + }); }); }); diff --git a/server/test/unit/services/workerHelper.test.ts b/server/test/unit/services/workerHelper.test.ts index 749804768..93e224491 100644 --- a/server/test/unit/services/workerHelper.test.ts +++ b/server/test/unit/services/workerHelper.test.ts @@ -41,6 +41,9 @@ const createHelper = (overrides?: Record) => { const checkService = { deleteOlderThan: jest.fn().mockResolvedValue(0), }; + const egressService = { + checkRecovery: jest.fn().mockResolvedValue(undefined), + }; const defaults = { logger: createMockLogger(), @@ -54,6 +57,7 @@ const createHelper = (overrides?: Record) => { incidentsRepository, geoChecksRepository, dockerLogsRepository, + egressService, ...overrides, }; @@ -68,7 +72,8 @@ const createHelper = (overrides?: Record) => { defaults.checksRepository as any, defaults.incidentsRepository as any, defaults.geoChecksRepository as any, - defaults.dockerLogsRepository as any + defaults.dockerLogsRepository as any, + defaults.egressService as any ); return { helper, defaults }; }; @@ -184,4 +189,22 @@ describe("WorkerHelper", () => { expect(defaults.logger.error).toHaveBeenCalledWith(expect.objectContaining({ message: "Unknown error", stack: undefined })); }); }); + + describe("getEgressRecoveryJob", () => { + it("returns a function that hands the claimed job to the egress service", async () => { + const { helper, defaults } = createHelper(); + const job = { id: "egress", type: "egress", refId: null, nextScheduledAt: 1000 } as any; + + await helper.getEgressRecoveryJob()(job); + + expect(defaults.egressService.checkRecovery).toHaveBeenCalledWith(job); + }); + + it("lets the egress service's error propagate so the queue records the failure", async () => { + const { helper, defaults } = createHelper(); + (defaults.egressService.checkRecovery as jest.Mock).mockRejectedValue(new Error("db down")); + + await expect(helper.getEgressRecoveryJob()({ id: "egress" } as any)).rejects.toThrow("db down"); + }); + }); }); diff --git a/server/test/unit/utils/egressTarget.test.ts b/server/test/unit/utils/egressTarget.test.ts new file mode 100644 index 000000000..22c567bae --- /dev/null +++ b/server/test/unit/utils/egressTarget.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "@jest/globals"; +import { parseEgressTarget } from "../../../src/utils/egressTarget.ts"; + +describe("parseEgressTarget", () => { + it("parses http(s) URLs", () => { + expect(parseEgressTarget("https://example.com/health")).toEqual({ kind: "http", url: "https://example.com/health" }); + expect(parseEgressTarget("HTTP://example.com")).toEqual({ kind: "http", url: "HTTP://example.com" }); + expect(parseEgressTarget("http://[2606:4700::1111]:8080/")).toEqual({ kind: "http", url: "http://[2606:4700::1111]:8080/" }); + }); + + it("parses host:port with a hostname, IPv4 address, localhost or bracketed IPv6 address", () => { + expect(parseEgressTarget("dns.google:53")).toEqual({ kind: "port", host: "dns.google", port: 53 }); + expect(parseEgressTarget("8.8.8.8:53")).toEqual({ kind: "port", host: "8.8.8.8", port: 53 }); + expect(parseEgressTarget("localhost:8080")).toEqual({ kind: "port", host: "localhost", port: 8080 }); + expect(parseEgressTarget("[2606:4700::1111]:443")).toEqual({ kind: "port", host: "2606:4700::1111", port: 443 }); + }); + + it("parses bare hosts and addresses as ping targets", () => { + expect(parseEgressTarget("1.1.1.1")).toEqual({ kind: "ping", host: "1.1.1.1" }); + expect(parseEgressTarget("example.com")).toEqual({ kind: "ping", host: "example.com" }); + expect(parseEgressTarget("2606:4700:4700::1111")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" }); + expect(parseEgressTarget("[2606:4700:4700::1111]")).toEqual({ kind: "ping", host: "2606:4700:4700::1111" }); + expect(parseEgressTarget(" 1.1.1.1 ")).toEqual({ kind: "ping", host: "1.1.1.1" }); + }); + + it("rejects anything the probe could not reach", () => { + for (const target of [ + "", + "ftp://example.com", + "http://", + "bad host", + "1.1.1.1;rm", + "[1.1.1.1", + "1.1.1.1]", + "[::1", + "::1]", + "[[::1]]", + "[not-an-address]:53", + "[1.1.1.1]:53", + "999.1.1.1", + "1.1.1.1:0", + "1.1.1.1:70000", + "1.1.1.1:", + "host-without-tld", + ]) { + expect(parseEgressTarget(target)).toBeNull(); + } + }); +}); diff --git a/server/test/unit/validation/settingsValidation.test.ts b/server/test/unit/validation/settingsValidation.test.ts index 36e25c3b2..745cbd81e 100644 --- a/server/test/unit/validation/settingsValidation.test.ts +++ b/server/test/unit/validation/settingsValidation.test.ts @@ -34,8 +34,30 @@ describe("settingsValidation", () => { expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets })).toThrow(); }); + it("accepts IPv6 targets, bare or bracketed with a port", () => { + const targets = ["2606:4700:4700::1111", "[2606:4700:4700::1111]", "[2606:4700:4700::1111]:53", "localhost", "localhost:8080"]; + expect(updateAppSettingsBodyValidation.parse({ egressCheckTargets: targets }).egressCheckTargets).toEqual(targets); + }); + it("rejects targets that are not a host, host:port or http(s) URL", () => { - for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm"]) { + for (const target of ["", " ", "ftp://example.com", "bad host", "1.1.1.1;rm", "http://"]) { + expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow(); + } + }); + + it("rejects malformed addresses that the probe could never reach", () => { + for (const target of [ + "[1.1.1.1", + "1.1.1.1]", + "[::1", + "::1]", + "[[::1]]", + "[not-an-address]:53", + "999.1.1.1", + "1.1.1.1:0", + "1.1.1.1:70000", + "[::1]:", + ]) { expect(() => updateAppSettingsBodyValidation.parse({ egressCheckTargets: [target] })).toThrow(); } }); @@ -59,6 +81,11 @@ describe("settingsValidation", () => { expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [] }).egressNotifications).toEqual([]); }); + it("accepts upper-case hex notification ids", () => { + const id = "64B7F0C2A1D2E3F4A5B6C7D8"; + expect(updateAppSettingsBodyValidation.parse({ egressNotifications: [id] }).egressNotifications).toEqual([id]); + }); + it("rejects notification ids that are not 24-character hex", () => { for (const id of ["not-an-id", "64b7f0c2a1d2e3f4a5b6c7d", "zzzzzzzzzzzzzzzzzzzzzzzz", 123]) { expect(() => updateAppSettingsBodyValidation.parse({ egressNotifications: [id] })).toThrow(); From b68367b04f4ac1921dc2b8e70179857e940f584f Mon Sep 17 00:00:00 2001 From: Craig Lawson Date: Tue, 15 Sep 2026 08:59:13 +0100 Subject: [PATCH 32/35] Harden the egress recovery job after a second review pass (#3937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Schedule the recovery job before recording the degraded transition, so a crash between the two leaves a stray row (removed on the job's first run) rather than a degraded state nothing is polling - Count any HTTP response, including 4xx/5xx, as reachable for http targets; only a transport failure counts against egress - Cascade a deleted notification out of egressNotifications with an atomic $pull in the settings repository, after the team-scoped delete - Handle two processes racing the first egress state insert - Address global job rows by one key (deleteGlobalJob on _id) and restore the deleteByIdAndType signature - Inject net into PingProvider like PortProvider - Integration tests for MongoEgressStateRepository and the global-row delete methods on the jobs repository 🔧 --- server/src/config/services.worker.ts | 2 +- .../app-settings-repository.interface.ts | 1 + .../app-settings.repository.mongo.ts | 4 + .../app-settings/app-settings.service.ts | 5 + .../egress/egress-state.repository.mongo.ts | 21 ++- .../src/domain/egress/egress-state.service.ts | 2 +- server/src/domain/egress/egress.service.ts | 12 +- .../domain/jobs/job.repository.interface.ts | 6 +- .../src/domain/jobs/job.repository.mongo.ts | 9 +- .../notifications/notification.service.ts | 14 +- server/src/service/network/PingProvider.ts | 10 +- .../integration/egressStateRepository.test.ts | 130 ++++++++++++++++++ .../test/integration/jobsRepository.test.ts | 43 ++++++ .../providers/network/pingProvider.test.ts | 29 ++-- .../test/unit/services/egressService.test.ts | 46 ++++++- .../unit/services/egressStateService.test.ts | 4 +- .../services/notificationsService.test.ts | 26 ++-- .../unit/services/settingsService.test.ts | 17 +++ 18 files changed, 319 insertions(+), 62 deletions(-) create mode 100644 server/test/integration/egressStateRepository.test.ts diff --git a/server/src/config/services.worker.ts b/server/src/config/services.worker.ts index e15e1cbf5..802ad4b0c 100644 --- a/server/src/config/services.worker.ts +++ b/server/src/config/services.worker.ts @@ -78,7 +78,7 @@ export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig // *********************** // Network providers // *********************** - const pingProvider = new PingProvider(ping); + const pingProvider = new PingProvider(ping, net); const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath)); const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger); const hardwareProvider = new HardwareProvider(httpProvider); diff --git a/server/src/domain/app-settings/app-settings-repository.interface.ts b/server/src/domain/app-settings/app-settings-repository.interface.ts index 401d867f7..c8e5a37b3 100644 --- a/server/src/domain/app-settings/app-settings-repository.interface.ts +++ b/server/src/domain/app-settings/app-settings-repository.interface.ts @@ -6,6 +6,7 @@ export interface ISettingsRepository { findSingleton(): Promise; // update update(settings: SettingsUpdate): Promise; + removeEgressNotification(notificationId: string): Promise; // $pull, so it cannot race a concurrent settings update // delete deleteLegacy: () => Promise; } diff --git a/server/src/domain/app-settings/app-settings.repository.mongo.ts b/server/src/domain/app-settings/app-settings.repository.mongo.ts index f2ef3351c..795a2db35 100644 --- a/server/src/domain/app-settings/app-settings.repository.mongo.ts +++ b/server/src/domain/app-settings/app-settings.repository.mongo.ts @@ -84,6 +84,10 @@ class MongoSettingsRepository implements ISettingsRepository { return this.toEntity(updatedSettings); }; + removeEgressNotification = async (notificationId: string) => { + await AppSettingsModel.updateMany({ egressNotifications: notificationId }, { $pull: { egressNotifications: notificationId } }); + }; + deleteLegacy = async () => { const res = await AppSettingsModel.deleteMany({ version: { $exists: false } }); return res.deletedCount > 0; diff --git a/server/src/domain/app-settings/app-settings.service.ts b/server/src/domain/app-settings/app-settings.service.ts index 2df2b7512..c0a0a184a 100755 --- a/server/src/domain/app-settings/app-settings.service.ts +++ b/server/src/domain/app-settings/app-settings.service.ts @@ -24,6 +24,7 @@ export interface ISettingsService { areStatusPageThemesEnabled(): boolean; getDBSettings(): Promise; updateDbSettings(newSettings: SettingsUpdate): Promise; + removeEgressNotification(notificationId: string): Promise; } export class SettingsService implements ISettingsService { @@ -78,6 +79,10 @@ export class SettingsService implements ISettingsService { return await this.getRepository().update(newSettings); }; + removeEgressNotification = async (notificationId: string) => { + await this.getRepository().removeEgressNotification(notificationId); + }; + getDBSettings = async () => { const repo = this.getRepository(); // Remove any old settings diff --git a/server/src/domain/egress/egress-state.repository.mongo.ts b/server/src/domain/egress/egress-state.repository.mongo.ts index a5bb110c3..6f2715d1d 100644 --- a/server/src/domain/egress/egress-state.repository.mongo.ts +++ b/server/src/domain/egress/egress-state.repository.mongo.ts @@ -30,12 +30,21 @@ class MongoEgressStateRepository implements IEgressStateRepository { // Read first: with timestamps on, every findOneAndUpdate would $set updatedAt, so a read would write. const existing = await EgressStateModel.findOne({ singleton: true }).lean(); if (existing) return this.toEntity(existing); - const doc = await EgressStateModel.findOneAndUpdate( - { singleton: true }, - { $setOnInsert: { singleton: true, status: "ok" } }, - { upsert: true, new: true, setDefaultsOnInsert: true } - ).lean(); - return this.toEntity(doc); + try { + const doc = await EgressStateModel.findOneAndUpdate( + { singleton: true }, + { $setOnInsert: { singleton: true, status: "ok" } }, + { upsert: true, new: true, setDefaultsOnInsert: true } + ).lean(); + return this.toEntity(doc); + } catch (error: unknown) { + // Two processes can race the very first insert; the unique index rejects the loser, who reads the winner's row. + if ((error as { code?: number }).code === 11000) { + const doc = await EgressStateModel.findOne({ singleton: true }).lean(); + if (doc) return this.toEntity(doc); + } + throw error; + } }; reset = async () => { diff --git a/server/src/domain/egress/egress-state.service.ts b/server/src/domain/egress/egress-state.service.ts index c4701abe9..9d605156f 100644 --- a/server/src/domain/egress/egress-state.service.ts +++ b/server/src/domain/egress/egress-state.service.ts @@ -25,7 +25,7 @@ export class EgressStateService implements IEgressStateService { // Called when the feature is switched on or off: drop any pending recovery job and start from "ok". reset = async (): Promise => { - await this.jobsRepository.deleteByIdAndType(null, "egress"); + await this.jobsRepository.deleteGlobalJob("egress"); return this.egressStateRepository.reset(); }; } diff --git a/server/src/domain/egress/egress.service.ts b/server/src/domain/egress/egress.service.ts index 40a4d4cd2..eeec82e21 100644 --- a/server/src/domain/egress/egress.service.ts +++ b/server/src/domain/egress/egress.service.ts @@ -25,6 +25,9 @@ const ASSESSMENT_REUSE_MS = 5000; const RECOVERY_JOB_TYPE = "egress" as const; const RECOVERY_JOB_ID = jobId(RECOVERY_JOB_TYPE, null); +// HttpProvider reports transport failures with NETWORK_ERROR (outside the HTTP range) and everything else with the real status code. +const isHttpStatusCode = (code: unknown): boolean => typeof code === "number" && code >= 100 && code <= 599; + // Synthetic identity stamped on the probe monitors so provider responses are recognisable in logs. const PROBE_MONITOR_ID = "egress-probe"; const PROBE_TEAM_ID = "system"; @@ -115,7 +118,9 @@ export class EgressService implements IEgressService { return { target: raw, - reachable: response.status === true, + // Any HTTP response, including 4xx/5xx, proves the instance can reach the target; only a transport + // failure counts against egress. Ping and port providers have no such distinction. + reachable: response.status === true || (target.kind === "http" && isHttpStatusCode(response.code)), responseTime: response.responseTime ?? responseTime, message: response.message, }; @@ -188,6 +193,9 @@ export class EgressService implements IEgressService { return "ok"; } + // Schedule the recovery job before recording the transition: a crash between the two then leaves a stray row + // while ok, which the job's first run removes, rather than a degraded state that nothing is polling. + await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, true); const degraded = await this.egressStateRepository.markDegraded(results, now); this.logger.warn({ message: degraded @@ -197,8 +205,6 @@ export class EgressService implements IEgressService { method: "assessAfterFailure", details: { results }, }); - // The worker that performed the transition owns the job's schedule; anyone else only makes sure the row exists. - await this.scheduleRecoveryCheck(settings.egressPollIntervalSeconds, degraded !== null); return "degraded"; } catch (error: unknown) { // The egress check must never break check production; treat an internal failure as "unknown" and carry on. diff --git a/server/src/domain/jobs/job.repository.interface.ts b/server/src/domain/jobs/job.repository.interface.ts index 6fe6b23b9..435e49118 100644 --- a/server/src/domain/jobs/job.repository.interface.ts +++ b/server/src/domain/jobs/job.repository.interface.ts @@ -58,9 +58,11 @@ export interface IJobsRepository { // Delete, drop all rows deleteById(refId: string): Promise; - // Delete a single job of type. refId null addresses a global row - deleteByIdAndType(refId: string | null, type: JobType): Promise; + // Delete a single job of type + deleteByIdAndType(refId: string, type: JobType): Promise; + // Global rows (refId null) are addressed by their canonical _id, see jobId + deleteGlobalJob(type: JobType): Promise; // Delete a global row only if nobody has rescheduled it since it was claimed (nextScheduledAt unchanged) deleteGlobalJobIfUnchanged(type: JobType, nextScheduledAt: number): Promise; diff --git a/server/src/domain/jobs/job.repository.mongo.ts b/server/src/domain/jobs/job.repository.mongo.ts index d545b0241..d1aff472e 100644 --- a/server/src/domain/jobs/job.repository.mongo.ts +++ b/server/src/domain/jobs/job.repository.mongo.ts @@ -273,8 +273,13 @@ class MongoJobsRepository implements IJobsRepository { return res.deletedCount > 0; }; - deleteByIdAndType = async (refId: string | null, type: JobType) => { - const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row or the global egress row + deleteByIdAndType = async (refId: string, type: JobType) => { + const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row + return res.deletedCount > 0; + }; + + deleteGlobalJob = async (type: JobType) => { + const res = await JobModel.deleteOne({ _id: jobId(type, null) }); return res.deletedCount > 0; }; diff --git a/server/src/domain/notifications/notification.service.ts b/server/src/domain/notifications/notification.service.ts index e270e76d4..eb0e333d4 100644 --- a/server/src/domain/notifications/notification.service.ts +++ b/server/src/domain/notifications/notification.service.ts @@ -178,18 +178,10 @@ export class NotificationsService implements INotificationsService { deleteById = async (id: string, teamId: string): Promise => { await this.monitorsRepository.removeNotificationFromMonitors(id); - await this.removeFromEgressNotifications(id); const deleted = await this.notificationsRepository.deleteById(id, teamId); + // App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles. + // After the team-scoped delete, so a request for another team's id (404 above) cannot touch instance settings. + await this.settingsService.removeEgressNotification(id); return deleted; }; - - // App settings hold notification ids for the egress recovered alert; drop the deleted one so it never dangles - private removeFromEgressNotifications = async (id: string) => { - const settings = await this.settingsService.getDBSettings(); - const current = settings.egressNotifications ?? []; - const remaining = current.filter((notificationId) => notificationId !== id); - if (remaining.length !== current.length) { - await this.settingsService.updateDbSettings({ egressNotifications: remaining }); - } - }; } diff --git a/server/src/service/network/PingProvider.ts b/server/src/service/network/PingProvider.ts index 9194ed9d2..7bcde8c7e 100644 --- a/server/src/service/network/PingProvider.ts +++ b/server/src/service/network/PingProvider.ts @@ -4,16 +4,20 @@ import { MonitorType, Monitor } from "@/domain/monitors/monitor.type.js"; import { MonitorStatusResponse } from "@/types/network.js"; import { AppError } from "@/utils/AppError.js"; import ping from "ping"; -import net from "net"; +import * as net from "net"; import { timeRequest } from "@/service/network/utils.js"; const SERVICE_NAME = "PingProvider"; type Ping = typeof ping; +type NetType = typeof net; export class PingProvider implements IStatusProvider { readonly type = "ping"; - constructor(private ping: Ping) {} + constructor( + private ping: Ping, + private net: NetType + ) {} supports(type: MonitorType): boolean { return type === "ping"; @@ -25,7 +29,7 @@ export class PingProvider implements IStatusProvider { const bracketed = /^\[([^\]]+)\](?::\d+)?$/.exec(host); if (bracketed) return bracketed[1]!; // A bare IPv6 address contains colons that are not a port separator - if (net.isIPv6(host)) return host; + if (this.net.isIPv6(host)) return host; return host.replace(/:.*/, ""); } diff --git a/server/test/integration/egressStateRepository.test.ts b/server/test/integration/egressStateRepository.test.ts new file mode 100644 index 000000000..c5090ab84 --- /dev/null +++ b/server/test/integration/egressStateRepository.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it, beforeAll, afterAll, beforeEach } from "@jest/globals"; +import mongoose from "mongoose"; +import { MongoMemoryServer } from "mongodb-memory-server"; +import MongoEgressStateRepository from "../../src/domain/egress/egress-state.repository.mongo.ts"; +import { EgressStateModel } from "../../src/domain/egress/egress-state.model.ts"; + +// ── Real-Mongo harness ───────────────────────────────────────────────────────── +// The transitions here are the guarantees several worker processes rely on: exactly one +// caller flips ok -> degraded (and degraded -> ok), and a plain read never writes. Both are +// MongoDB filter/update semantics, so they need a live engine. + +let mongod: MongoMemoryServer; + +beforeAll(async () => { + mongod = await MongoMemoryServer.create(); + await mongoose.connect(mongod.getUri()); + await EgressStateModel.init(); // unique index on singleton +}, 120_000); + +afterAll(async () => { + await mongoose.disconnect(); + await mongod.stop(); +}); + +beforeEach(async () => { + await EgressStateModel.deleteMany({}); +}); + +const probe = (target: string, reachable: boolean) => ({ target, reachable, responseTime: 5, message: reachable ? "Success" : "timeout" }); +const NOW = new Date("2026-01-01T10:00:00.000Z"); +const LATER = new Date("2026-01-01T10:05:00.000Z"); + +describe("MongoEgressStateRepository", () => { + const repo = new MongoEgressStateRepository(); + + describe("findSingleton", () => { + it("creates the ok singleton on first call and returns the same document afterwards", async () => { + const first = await repo.findSingleton(); + const second = await repo.findSingleton(); + + expect(first.status).toBe("ok"); + expect(second.id).toBe(first.id); + expect(await EgressStateModel.countDocuments()).toBe(1); + }); + + it("does not bump updatedAt on a read", async () => { + const created = await repo.findSingleton(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + const read = await repo.findSingleton(); + + expect(read.updatedAt).toBe(created.updatedAt); + }); + + it("survives two processes racing the first insert", async () => { + const [a, b] = await Promise.all([repo.findSingleton(), repo.findSingleton()]); + + expect(a.id).toBe(b.id); + expect(await EgressStateModel.countDocuments()).toBe(1); + }); + }); + + describe("markDegraded / markRecovered", () => { + it("markDegraded does nothing when the singleton has never been created (callers read it first)", async () => { + expect(await repo.markDegraded([probe("1.1.1.1", false)], NOW)).toBeNull(); + expect(await EgressStateModel.countDocuments()).toBe(0); + }); + + it("lets exactly one caller perform each transition", async () => { + await repo.findSingleton(); + + const [d1, d2] = await Promise.all([repo.markDegraded([probe("1.1.1.1", false)], NOW), repo.markDegraded([probe("1.1.1.1", false)], NOW)]); + expect([d1, d2].filter((result) => result !== null)).toHaveLength(1); + expect((await repo.findSingleton()).status).toBe("degraded"); + + const [r1, r2] = await Promise.all([repo.markRecovered([probe("1.1.1.1", true)], LATER), repo.markRecovered([probe("1.1.1.1", true)], LATER)]); + expect([r1, r2].filter((result) => result !== null)).toHaveLength(1); + const state = await repo.findSingleton(); + expect(state.status).toBe("ok"); + expect(state.degradedSince).toBe(NOW.toISOString()); + expect(state.lastRecoveredAt).toBe(LATER.toISOString()); + }); + + it("markRecovered is a no-op while ok and markDegraded is a no-op while degraded", async () => { + await repo.findSingleton(); + expect(await repo.markRecovered([probe("1.1.1.1", true)], NOW)).toBeNull(); + + await repo.markDegraded([probe("1.1.1.1", false)], NOW); + expect(await repo.markDegraded([probe("1.1.1.1", false)], LATER)).toBeNull(); + expect((await repo.findSingleton()).degradedSince).toBe(NOW.toISOString()); + }); + }); + + describe("recordProbe", () => { + it("stores the probe without changing the status", async () => { + await repo.findSingleton(); + await repo.markDegraded([probe("1.1.1.1", false)], NOW); + + const state = await repo.recordProbe([probe("1.1.1.1", false), probe("8.8.8.8", false)], LATER); + + expect(state.status).toBe("degraded"); + expect(state.lastProbeAt).toBe(LATER.toISOString()); + expect(state.lastProbeResults.map((result) => result.target)).toEqual(["1.1.1.1", "8.8.8.8"]); + }); + }); + + describe("reset", () => { + it("returns a degraded singleton to ok with no episode and keeps the recovery history", async () => { + await repo.findSingleton(); + await repo.markDegraded([probe("1.1.1.1", false)], NOW); + await repo.markRecovered([probe("1.1.1.1", true)], LATER); + await repo.markDegraded([probe("1.1.1.1", false)], LATER); + + const state = await repo.reset(); + + expect(state.status).toBe("ok"); + expect(state.degradedSince).toBeNull(); + expect(state.lastProbeAt).toBeNull(); + expect(state.lastProbeResults).toEqual([]); + expect(state.lastRecoveredAt).toBe(LATER.toISOString()); + }); + + it("creates the singleton when none exists", async () => { + const state = await repo.reset(); + + expect(state.status).toBe("ok"); + expect(await EgressStateModel.countDocuments()).toBe(1); + }); + }); +}); diff --git a/server/test/integration/jobsRepository.test.ts b/server/test/integration/jobsRepository.test.ts index 78ac17b64..e2b426ce4 100644 --- a/server/test/integration/jobsRepository.test.ts +++ b/server/test/integration/jobsRepository.test.ts @@ -533,4 +533,47 @@ describe("MongoJobsRepository", () => { expect(await repo.countDueBacklog(NOW)).toBe(0); }); }); + + // ── global rows: egress recovery job ────────────────────────────────────── + + describe("global rows", () => { + const repo = new MongoJobsRepository(WORKER_ID); + const egressRow = (nextScheduledAt: number) => + seedJob({ _id: "egress", type: "egress" as JobType, refId: null, nextScheduledAt, intervalMs: 30_000 }); + + it("deleteGlobalJob removes the row by its canonical _id and leaves other global rows alone", async () => { + await egressRow(NOW); + await seedJob({ _id: "cleanup-orphaned", type: "cleanup-orphaned" as JobType, refId: null }); + + expect(await repo.deleteGlobalJob("egress")).toBe(true); + + expect(await readRow("egress")).toBeNull(); + expect(await readRow("cleanup-orphaned")).not.toBeNull(); + expect(await repo.deleteGlobalJob("egress")).toBe(false); + }); + + it("deleteGlobalJobIfUnchanged removes the row only while nextScheduledAt matches the claimed value", async () => { + await egressRow(NOW); + + // Another worker started a new episode and rescheduled the row after this run claimed it + await JobModel.updateOne({ _id: "egress" }, { $set: { nextScheduledAt: NOW + 30_000 } }); + expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW)).toBe(false); + expect(await readRow("egress")).not.toBeNull(); + + // Nobody touched it: the row goes + expect(await repo.deleteGlobalJobIfUnchanged("egress", NOW + 30_000)).toBe(true); + expect(await readRow("egress")).toBeNull(); + }); + + it("recordSuccess after a refused conditional delete is a no-op once the lock has moved on", async () => { + await egressRow(NOW); + const [claimed] = await repo.claimDueBatch("egress", 1, NOW); + expect(claimed?.id).toBe("egress"); + + // Row deleted by the handler (recovered), then recordSuccess runs from the queue: nothing to update + expect(await repo.deleteGlobalJobIfUnchanged("egress", claimed!.nextScheduledAt)).toBe(true); + expect(await repo.recordSuccess("egress", claimed!.nextScheduledAt, 30_000, NOW + 1_000)).toBe(false); + expect(await readRow("egress")).toBeNull(); + }); + }); }); diff --git a/server/test/unit/providers/network/pingProvider.test.ts b/server/test/unit/providers/network/pingProvider.test.ts index 6b4129850..150472925 100644 --- a/server/test/unit/providers/network/pingProvider.test.ts +++ b/server/test/unit/providers/network/pingProvider.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, jest } from "@jest/globals"; import { PingProvider } from "../../../../src/service/network/PingProvider.ts"; import { testStatusProviderContract } from "../../../helpers/statusProviderContract.ts"; import type { Monitor } from "../../../../src/domain/monitors/monitor.type.ts"; +import * as net from "net"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -27,7 +28,7 @@ const createMockPing = (result?: Partial<{ alive: boolean; time: number | string // ── Contract ───────────────────────────────────────────────────────────────── testStatusProviderContract("PingProvider", { - create: () => new PingProvider(createMockPing() as any), + create: () => new PingProvider(createMockPing() as any, net), supportedType: "ping", unsupportedType: "http", makeMonitor: () => makeMonitor(), @@ -37,7 +38,7 @@ testStatusProviderContract("PingProvider", { describe("PingProvider", () => { it("returns success when host is alive", async () => { - const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any); + const provider = new PingProvider(createMockPing({ alive: true, time: 30 }) as any, net); const result = await provider.handle(makeMonitor()); @@ -55,7 +56,7 @@ describe("PingProvider", () => { }); it("returns failure when host is not alive", async () => { - const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any); + const provider = new PingProvider(createMockPing({ alive: false, time: 0 }) as any, net); const result = await provider.handle(makeMonitor()); @@ -69,7 +70,7 @@ describe("PingProvider", () => { }); it("parses string time values", async () => { - const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any); + const provider = new PingProvider(createMockPing({ alive: true, time: "42.5" }) as any, net); const result = await provider.handle(makeMonitor()); @@ -77,7 +78,7 @@ describe("PingProvider", () => { }); it("defaults responseTime to 0 for non-numeric time", async () => { - const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any); + const provider = new PingProvider(createMockPing({ alive: true, time: "unknown" }) as any, net); const result = await provider.handle(makeMonitor()); @@ -85,7 +86,7 @@ describe("PingProvider", () => { }); it("defaults status to false when alive is undefined", async () => { - const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any); + const provider = new PingProvider(createMockPing({ alive: undefined as any }) as any, net); const result = await provider.handle(makeMonitor()); @@ -94,7 +95,7 @@ describe("PingProvider", () => { it("sanitizes URL by stripping protocol and path", async () => { const mockPing = createMockPing(); - const provider = new PingProvider(mockPing as any); + const provider = new PingProvider(mockPing as any, net); await provider.handle(makeMonitor({ url: "https://example.com/path?query=1" })); @@ -103,7 +104,7 @@ describe("PingProvider", () => { it("sanitizes URL by stripping port", async () => { const mockPing = createMockPing(); - const provider = new PingProvider(mockPing as any); + const provider = new PingProvider(mockPing as any, net); await provider.handle(makeMonitor({ url: "example.com:8080" })); @@ -112,7 +113,7 @@ describe("PingProvider", () => { it("keeps a bare IPv6 address intact", async () => { const mockPing = createMockPing(); - const provider = new PingProvider(mockPing as any); + const provider = new PingProvider(mockPing as any, net); await provider.handle(makeMonitor({ url: "2606:4700:4700::1111" })); @@ -121,7 +122,7 @@ describe("PingProvider", () => { it("unwraps a bracketed IPv6 address and drops its port", async () => { const mockPing = createMockPing(); - const provider = new PingProvider(mockPing as any); + const provider = new PingProvider(mockPing as any, net); await provider.handle(makeMonitor({ url: "[2606:4700:4700::1111]:53" })); @@ -129,28 +130,28 @@ describe("PingProvider", () => { }); it("throws AppError when url is missing", async () => { - const provider = new PingProvider(createMockPing() as any); + const provider = new PingProvider(createMockPing() as any, net); await expect(provider.handle(makeMonitor({ url: "" }))).rejects.toThrow("URL is required for ping monitor"); }); it("throws AppError when ping.probe rejects", async () => { const mockPing = { promise: { probe: jest.fn().mockRejectedValue(new Error("Host unreachable")) } }; - const provider = new PingProvider(mockPing as any); + const provider = new PingProvider(mockPing as any, net); await expect(provider.handle(makeMonitor())).rejects.toThrow("Host unreachable"); }); it("throws AppError when probe returns no response", async () => { const mockPing = { promise: { probe: jest.fn().mockResolvedValue(null) } }; - const provider = new PingProvider(mockPing as any); + const provider = new PingProvider(mockPing as any, net); await expect(provider.handle(makeMonitor())).rejects.toThrow("No response from ping"); }); it("throws AppError with stringified message for non-Error throws", async () => { const mockPing = { promise: { probe: jest.fn().mockRejectedValue(42) } }; - const provider = new PingProvider(mockPing as any); + const provider = new PingProvider(mockPing as any, net); await expect(provider.handle(makeMonitor())).rejects.toThrow("42"); }); diff --git a/server/test/unit/services/egressService.test.ts b/server/test/unit/services/egressService.test.ts index 8aee753d7..72f76ac02 100644 --- a/server/test/unit/services/egressService.test.ts +++ b/server/test/unit/services/egressService.test.ts @@ -146,6 +146,27 @@ describe("EgressService", () => { ]); }); + it("treats any HTTP response, even 4xx/5xx, as reachable but a transport failure as unreachable", async () => { + const requestStatus = jest.fn().mockImplementation(async (monitor: any) => ({ + monitorId: monitor.id, + teamId: monitor.teamId, + type: monitor.type, + status: false, + code: monitor.url.includes("maintenance") ? 503 : 5000, // 5000 is HttpProvider's NETWORK_ERROR + message: monitor.url.includes("maintenance") ? "Service Unavailable" : "ECONNREFUSED", + responseTime: 3, + })); + const { service } = createService({ networkService: { requestStatus } }); + + const results = await service.probeTargets(["https://example.com/maintenance", "https://example.com/refused", "8.8.8.8:53"]); + + expect(results).toEqual([ + expect.objectContaining({ target: "https://example.com/maintenance", reachable: true }), + expect.objectContaining({ target: "https://example.com/refused", reachable: false }), + expect.objectContaining({ target: "8.8.8.8:53", reachable: false }), // non-http: only status true counts + ]); + }); + it("counts an unparseable stored target as unreachable without probing", async () => { const { service, defaults } = createService(); @@ -218,7 +239,6 @@ describe("EgressService", () => { ); expect(defaults.egressStateRepository.recordProbe).not.toHaveBeenCalled(); expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("egress degraded") })); - // The worker that performed the transition sets the schedule expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith({ id: "egress", type: "egress", @@ -228,9 +248,13 @@ describe("EgressService", () => { intervalMs: 30_000, }); expect(defaults.jobsRepository.upsertJob).not.toHaveBeenCalled(); + // Job first, transition second: a crash in between leaves a stray row, never an unpolled degraded state + const scheduleOrder = (defaults.jobsRepository.upsertCleanupJob as jest.Mock).mock.invocationCallOrder[0]!; + const transitionOrder = (defaults.egressStateRepository.markDegraded as jest.Mock).mock.invocationCallOrder[0]!; + expect(scheduleOrder).toBeLessThan(transitionOrder); }); - it("only makes sure the job row exists when another worker performed the transition", async () => { + it("still returns degraded and leaves the job scheduled when another worker performed the transition first", async () => { const { service, defaults } = createService({ networkService: { requestStatus: statusFor([]) }, egressStateRepository: degradedRepository({ @@ -242,8 +266,22 @@ describe("EgressService", () => { const result = await service.assessAfterFailure(); expect(result).toBe("degraded"); - expect(defaults.jobsRepository.upsertJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 })); - expect(defaults.jobsRepository.upsertCleanupJob).not.toHaveBeenCalled(); + expect(defaults.jobsRepository.upsertCleanupJob).toHaveBeenCalledWith(expect.objectContaining({ id: "egress", intervalMs: 30_000 })); + expect(defaults.logger.warn).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringContaining("already recorded") })); + }); + + it("does not mark degraded when the recovery job cannot be scheduled", async () => { + const { service, defaults } = createService({ + networkService: { requestStatus: statusFor([]) }, + jobsRepository: { + upsertJob: jest.fn(), + upsertCleanupJob: jest.fn().mockRejectedValue(new Error("db down")), + deleteGlobalJobIfUnchanged: jest.fn(), + }, + }); + + expect(await service.assessAfterFailure()).toBeNull(); + expect(defaults.egressStateRepository.markDegraded).not.toHaveBeenCalled(); }); it("returns degraded without probing when the persisted state is already degraded, without touching the job's schedule", async () => { diff --git a/server/test/unit/services/egressStateService.test.ts b/server/test/unit/services/egressStateService.test.ts index a75c0de1f..1701098d2 100644 --- a/server/test/unit/services/egressStateService.test.ts +++ b/server/test/unit/services/egressStateService.test.ts @@ -13,7 +13,7 @@ const createEgressStateRepo = () => ({ }); const createJobsRepo = () => ({ - deleteByIdAndType: jest.fn().mockResolvedValue(true), + deleteGlobalJob: jest.fn().mockResolvedValue(true), }); const createService = () => { @@ -82,7 +82,7 @@ describe("EgressStateService", () => { const result = await service.reset(); - expect(jobsRepository.deleteByIdAndType).toHaveBeenCalledWith(null, "egress"); + expect(jobsRepository.deleteGlobalJob).toHaveBeenCalledWith("egress"); expect(egressStateRepository.reset).toHaveBeenCalledTimes(1); expect(result).toEqual(reset); }); diff --git a/server/test/unit/services/notificationsService.test.ts b/server/test/unit/services/notificationsService.test.ts index f2462866b..d0fdd61b2 100644 --- a/server/test/unit/services/notificationsService.test.ts +++ b/server/test/unit/services/notificationsService.test.ts @@ -26,10 +26,9 @@ const createMonitorsRepo = () => ({ removeNotificationFromMonitors: jest.fn(), }); -const createSettingsService = (clientHost = "https://app.example.com", egressNotifications: string[] = []) => ({ +const createSettingsService = (clientHost = "https://app.example.com") => ({ getSettings: jest.fn().mockReturnValue({ clientHost }), - getDBSettings: jest.fn().mockResolvedValue({ egressNotifications }), - updateDbSettings: jest.fn().mockResolvedValue({ egressNotifications: [] }), + removeEgressNotification: jest.fn().mockResolvedValue(undefined), }); const createMessageBuilder = () => ({ @@ -374,24 +373,25 @@ describe("NotificationsService", () => { expect(monitorsRepository.removeNotificationFromMonitors).toHaveBeenCalledWith("notif-1"); }); - it("removes the notification from the egress recovered recipients in app settings", async () => { - const settingsService = createSettingsService("https://app.example.com", ["notif-0", "notif-1"]); - const { service, notificationsRepository } = createService({ settingsService }); + it("removes the notification from the egress recovered recipients after the team-scoped delete succeeds", async () => { + const { service, notificationsRepository, settingsService } = createService(); (notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification()); await service.deleteById("notif-1", "team-1"); - expect(settingsService.updateDbSettings).toHaveBeenCalledWith({ egressNotifications: ["notif-0"] }); + expect(settingsService.removeEgressNotification).toHaveBeenCalledWith("notif-1"); + const deleteOrder = (notificationsRepository.deleteById as jest.Mock).mock.invocationCallOrder[0]!; + const cascadeOrder = (settingsService.removeEgressNotification as jest.Mock).mock.invocationCallOrder[0]!; + expect(deleteOrder).toBeLessThan(cascadeOrder); }); - it("leaves app settings alone when the notification is not an egress recipient", async () => { - const settingsService = createSettingsService("https://app.example.com", ["notif-0"]); - const { service, notificationsRepository } = createService({ settingsService }); - (notificationsRepository.deleteById as jest.Mock).mockResolvedValue(makeNotification()); + it("does not touch app settings when the delete is refused", async () => { + const { service, notificationsRepository, settingsService } = createService(); + (notificationsRepository.deleteById as jest.Mock).mockRejectedValue(new Error("Notification not found")); - await service.deleteById("notif-1", "team-1"); + await expect(service.deleteById("notif-1", "other-team")).rejects.toThrow("Notification not found"); - expect(settingsService.updateDbSettings).not.toHaveBeenCalled(); + expect(settingsService.removeEgressNotification).not.toHaveBeenCalled(); }); }); }); diff --git a/server/test/unit/services/settingsService.test.ts b/server/test/unit/services/settingsService.test.ts index 92a31a89b..9e611c247 100644 --- a/server/test/unit/services/settingsService.test.ts +++ b/server/test/unit/services/settingsService.test.ts @@ -43,6 +43,7 @@ const createSettingsRepo = () => create: jest.fn().mockResolvedValue(makeSettings()), findSingleton: jest.fn().mockResolvedValue(makeSettings()), update: jest.fn().mockResolvedValue(makeSettings()), + removeEgressNotification: jest.fn().mockResolvedValue(undefined), deleteLegacy: jest.fn().mockResolvedValue(true), }) as unknown as jest.Mocked; @@ -192,4 +193,20 @@ describe("SettingsService", () => { await expect(service.updateDbSettings({ checkTTL: 60 })).rejects.toThrow("Settings repository not initialized"); }); }); + + describe("removeEgressNotification", () => { + it("delegates the pull to the repository", async () => { + const { service, settingsRepository } = createService(); + + await service.removeEgressNotification("notif-1"); + + expect(settingsRepository.removeEgressNotification).toHaveBeenCalledWith("notif-1"); + }); + + it("throws when repository is not set", async () => { + const service = new SettingsService(makeEnv()); + + await expect(service.removeEgressNotification("notif-1")).rejects.toThrow("Settings repository not initialized"); + }); + }); }); From 3857a71e6263b4b3e2d8257962979a7008a2de00 Mon Sep 17 00:00:00 2001 From: Ahson Shaikh <59800198+Ahson-Shaikh@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:58:35 +0500 Subject: [PATCH 33/35] Add Easypanel to 1-click installation options --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b5b74dc9..c9d027d50 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ JWT_SECRET="$(openssl rand -hex 32)" docker compose up -d Then open http://localhost:52345. If the app is reached at another origin (domain or LAN IP), set `CLIENT_HOST` accordingly. To build the image yourself, run `docker build -f docker/Dockerfile -t checkmate .` from a checkout. For TLS, put any reverse proxy (Caddy, Traefik, nginx) in front of port 52345. There are also 1-click installation options like [Repocloud](https://repocloud.io/details/Checkmate), -[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). Note that the Helm chart has not yet been migrated to the all-in-one image: it still deploys the legacy `checkmate-client`, `checkmate-backend` and `checkmate-mongo` images, pinned at v3.8.1. +[Pikapods](https://www.pikapods.com/), [Coolify](https://coolify.io/), [Elestio](https://elest.io/open-source/checkmate), [Easypanel](https://easypanel.io/templates/checkmate), [K8s](./charts/helm/checkmate/INSTALLATION.md), [Sive Host](https://sive.host) or [Cloudzy](https://cloudzy.com/marketplace/checkmate). Note that the Helm chart has not yet been migrated to the all-in-one image: it still deploys the legacy `checkmate-client`, `checkmate-backend` and `checkmate-mongo` images, pinned at v3.8.1. ### Configuration From 360b4b80353d133302c0941984fca6181d1bdfa1 Mon Sep 17 00:00:00 2001 From: Ahson Shaikh <59800198+Ahson-Shaikh@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:11:38 +0500 Subject: [PATCH 34/35] Add Easypanel deployment screenshot --- .github/easypanel-deploy-screenshot.png | Bin 0 -> 124673 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/easypanel-deploy-screenshot.png diff --git a/.github/easypanel-deploy-screenshot.png b/.github/easypanel-deploy-screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..27127328916f5fab32dd7f43b677e825c7fa7266 GIT binary patch literal 124673 zcmagG1yEeu5-v;#CxIk53Bhe}cZc8(!QEkScMk;D;O-vW2A3egAuzZG*TG%?Irj$6 z`RmMK1&D*5q`F}0h(DF!@y8P$46I53Cd#xwF|>523MMzxe9n2OzhG~c5(-(IlEiYu?$okqtC9x9jIPh)qk(my&#Utx{NxzUEJ0b%XQi*)2 zp5*qfwwrCJNAIXqdisp?UXZ96UiU52CSEr`;T*b?Ji#N3g_eq6-R6feD9#+lY7#&h z85k<)Jv_{dATt;^=-mtG2OIi)!IxV9ajFt7$A4?GMiC8t6g$Z7@2LVomj}Nu+L6}~+CG+J8qn1d| zoPsHZ+YXru3r!H;qoTa{-xr4}>{KIu(VFllxc_|-k`ww$O#8wIfB9cmhZixS?+)Jz z_y;i|?Ek#bk{l8L_jkX*FOb8cB^{vi;nDo>??QiDV)}p13myD_PaTQ!zbEcUhOeE6 zXp{lf6Ug)X?SX+!)3F$0Or@dgpIrX=*RNA~1Z!v4>NbleMO z9cfF(*I4OqU+gbRq&rg>YbmzTxpdv;rbGzo$$$i}9_qW&LH zUDCf;Nc3Fp9eZVlK^e8o$ftZ3`X+eP#z`4h3YK6uuYtqyLrU6q{3Z;{Da83y7TZxPU!is` z^*?sZ$jcYC$P*}9mK1Q#(SpIQbk4+aX+cwB6$lfg+eEbQBlSe=soBDQHLh$B`I@=* z$|n3TSeno&fRREU1YS@sScR?&pe2nU@;0FnV?IZqGn_-FnS?T+erNLn^Q~I zHp`NJm-jHnr9{`#+gbX5%xEG4v#~9kOv#U?UUap7*?awZHux5B5Pq5p>|M{`l3cy} z2$oeK39lxUju`6a5A4#ej`xvHO>FM1D{N7Ia_t!yxFd}mln+5PetU@6=&Rx%3U&64 z3ML(fIs^nI4Y_4Y0k*LjnNDRC)x=;n&uN`7KM-oX)JTJIf%>n0E`ix_$&2xUYP`G% z`(e?Nfs=ZcHpACagX$h-J07BFYjZ+CyKb5@%$6dJvJV>XcxLmIN|c4MRMc{O-ZLRm zCxxc(n}qnFfR_;GQ1pm4QAt(d242Jtsg!2+%fiDD>>FtY@kh$u3o%C;w0bp;Q6jTN-VA9Gu*MT(KzY#isW!!i0*#*B#gUP^Wr(dlM59OpUag zDSgefkH6bJ`Lgc|z-RbR*R4Q0g6Ol~R|6q<*KK#y&TB*d=&`eJr6LDq)TL#?6m~SR z^Sc02ER*HgkZ&-|{gb5B{hw1#rq%%&Yl^7<$rCdl;bf^9jzdX>pwY=aE}L%S(DdYz z%{h1LDs0h_aZW2URaeW2Y8b(0zk{~^aHfs66kGV~brskws9*!X$#~hw(0<^%Y~hAZ z!WoU>>^p8{?mpeAb^Z~|pSy+5DU)Ei$M{iF?K5`79zzaPkLB^$JeKI#h|1UlYz4bH`A4xo`lFb5?&B*k-v ztTGVx`0HdN@3vuQCi4O||aN~C_l}GCvv~Mj6-lwYQrOG;$ zvar`EoW#x+e-*g3?XP1$+{d+$S>SqHpS!5aP1m-r!PRJX(V0QYRr%Q6)?5oxDVe~w zUQ9aV>TR0O8Bp6?bNtF7C_pEm(Wz+y1ip&ixR|$DZge0|d8>i8TuY(sTUot{nklpXh1Wbeiiv!J+pH-WK^TQ)=08=jy;1o#0}Jw<9jgTQ-D9 z<{JNHL2}kEp4YKhrz?9CZeydZXDOKv$)&S*BdeY^EvN~D5@ioj754;D>V0=Q7Y?Hj zAz=h(zYiDOJ>v2NVzj+B6m`yO=u_sHRKHIqM$xhip3f<$Y461iOqN>^rhI;5LMn`Y z7hkD=x;cD%d1yioD=Gh?6`>v9uX9E?QXh|mG1d5?D;;U7IkVWN+r{f{K3+5!K3!@)bL(@@rfTwP|WwFHK!l@MOb3Bq#0V zi_HSgbBCOOHE(J1HiL^)o>aKZUniI(4wJUrk;_#r<@%TJsF^JJLL}-If-;dcl=w&M zt;=OmCUyVgk-msBgw^30hO~*LB&@j*rm+C|(D14Z`4Dh^ZV>&))T=P|mJ64My-_Q! z8SHtN(y9i>bBBzjgEDDI$|4T^!;IWx!&ZvQ$CjNri_F`*t+vzL!^f(`mZOd7hnpW4 zt4sIw>YBZYWZ&z{XbFR}jELeY6|QzraD()e)-Obl4A9*IhP`etvWc~AfBa?2h}(J6 z@CK-k(-DGTQ+j>v#^qOtrhuC5Hw--8@_E=wOiV;gy82*K(CTq1CnHnky+6Z5|* z+)pWH-Y6NY+8XEZiP_N*Pq+)IEn}?iS+&k#KC^OP2f}a5Hk*y&VM2BqRBWg96&MNk zIO9BtXxNIq7i*Hm)%SkpJrowZ*Eihdt#^J-&y(pu}Q}( z@-NHfu}P7}!d`{C-rUy$GNm-2n0C)g-PQ-zcGfzNRd(;k$!Q))^7Nf9J;!Gq2FY3B zlGYRuPI}~`#zQRs6+!01(QEHB_0b34O!pCTzQ-P2@8dAopficKF%9P&Jx}xM=7#Xf zlR);nl_35*{X0jLm>Iw{+KQeyd7Z{a9n{HAGao%;)O;>#G8d~1U&nw&Ftzl?8axYOl&eUWu&Qi2qE*@S5)wec6k_?BB^J;*x z`Nag1CYLzfUX(;Dmk1E^FSqpIKc&%cK2$>G>_Gq~ymZ13IWm5)J29@#8CinM^biaV?|k07@5oKp3Xsi{rux?Faj%X*#j&`lxy<3;+%TkeMw z2y*M0LA}QHcp={!+?a6dl)BpYYtjkgyc`>!+xczbh84p1`gYOw+ea(8Q639xwky}U z{5K>RAD9qP>4Gwm4y2UyLcVvB9Xnn<*j{<ZC zpO#O-!D0t5?=OtBq{x4;cL^=ks0Hwzi7SH^+!0Hs*bWOCibKanOw3x7cLKIu3>FeI zNDU!&e#nwh>(rEQhW|$a^vgfXCiJ1 zd-p0q=xfxV4cUOKK|{#ce!h+S3Vk9%@>`FK0Pcv%_@jcaBUQ?|CUo(ghCv?)oMuUX zu1wMM9B#dt}u7lg!3Krv^^F6|nW2Ms-B;cj@m|t*LtP^V9Qxw zy4z$YF>5{aIu!Elj1BX<{LraFvc)?;04-}0l6If#A4?cmQ-NNgIa$a%7hhEN z!IrIcZQYM@EWJ2u=E0< z4%C3lp6eIKWz5{p%J5grj$ zz>ogS%*?9VPVaqfTn`B22ovviG9M_f15r$W*|i-7(BBO&-$61TuQT~C8%IT)DOnO< z!~Jz2;^5%0E+~j(m{VWjxj~&5&~!0pq5F6KRX6#}V-5dpjnVC9UiYu)fL(T4p@2R$5o9PhkUgoN=5x z&Ev5$W6)Cp(6_%WyLGl&wp(moK5x6!arseh6^MnIITRj(CVsqJ50A#Nq%@Y5@RYxZ z5N6bOQ8h9iO$+Ysq6LZ}$<(A4{Z(yaTo>^hhNr_P1Jb9qi_Mf_)-K@9zqnPWR8(WH z)Fx8By7S-0=G&xNyslSz?ApPdXS~p6pV<<&F!ssD9HIxAd$($oqhbV_pO)Eph0#c1 z7;MZ4QCZ-ofUFlo`}~51v}P=HPO+=iD_n3F7T*!0p%Ej!BJ^d9#bZLx5dL_L(|(M@ zfpve%|HwQ$;B!CilMax?=-fCcC ziDN@!a4aqpK=R&Z&{o%d-;iKX*uMRQx?PaUacJ!m4Ug5@mR%uF(g=w(E9q0c#()7Q zzBP_l#J(ued;fTsM_q4VPUWSU#F{aNC9GW+L!tBeQ%o*w?PNFQV ztA{`n6Eoje{8vAab114JQ^&XXA1mlRW(RdHo2Fm9Hrq+DDF31P#-r_`u7Ve0CPzIY z9*$2Z3uR-avhGOnqzvgVS@4LTfURUiI2vFa7*ePkqW)p;RH{&%FS@Dj%sHx)B5Gl= zqVX+YQ6ZpPI{nadZ?B-gl!X#2y`gy|pOl%z4lH3(6wxouH#2vfr?;KWo0Fk9p8^wObOw@-W`Ke7{X_AAPCoj<-Qc>xRer$ z^q%|?Um-F6hkiMn%`?{ai@-WUke_S8&g1;FP2&Z`CXFJs>^e6ByI#SS;dq2ihFL4C z^kK@bq6jfe0uTKUje|d$Hz6%)o4h!EC+%$B7_vdS`Q74?|M94P_HN)2O7`g===mLQ zU-P-`MSEF!6EgcKnynT9%Jvt4xZIZe3(6^}ZaYAb<1@bK|!k-qYs^PIp?NkKKR!y+qhol#g7fA&SqTMbZN4oG__se;&qF1g# z<|dQ=c0={yI794y7@GG-c~3cS2`udH|1MwC7cb} z0bFJ5{MXLD)gq6^yth|$W7zCy;v#tn?RO`G{TKHGs%V0|=$GjH z$LN2j#QSVyui%>8Zo3b8q(cpfh1^@_O@o{!j$EIgRTBz?-^~+gX(_rpbWl~#a`9oG z;4!4c=L+k*zh9#U^w_b)-hPaz`>3{+yk5dY3;i^uzF3` zaJDi}=e=Fef47p3sncR_Nm(-S65Mo_oyfa>ELY@C3QY0XG^D%vt=qE0y?l*igVEpq z8L+tS9Gl92z#4>>9RA1Ge3{WdDKnkA9xvb};Y!EZ_-s(yxbJ21+;WckY%RBMMLkYG z&NW$?6ygwb)>ZeJsc-}BP0je6%l01Ik-ZNiTSt91Su3vMb#E5k*D1Cdp|HWlE#E_3Gf(4?wvL>ERlN)j0vSy2c_zyhdzVrXY))``~_`(K5wFL4LNCOFa6iiYt{ zc0YXhpy6(ST=(V(T52Ch5&h`Gt6#qPj``T~gQk5R6x0f3D#^VMzek1h7v+na?|-C9 z$+~P!Ro&~}?mGG0fxbQ-57T1uZFk&_y{%|i#TxY(kwjrFsVif4z+|t%`LPae9RF%rhWOsvzzRUBoZoA z;Yv3 z_AP9#OeIX_B6c#BL%^D!qE@00KT@_g5|2dYy@HZm=y?HDE^kFB@p_iae-L-vAI@C4 zCTT!adW&U~tXq{*Ez7X#A?c4%Hh8YvB=}8My%Wg1Yhw84c_w%qO)Pvh#>@KSq)OD_ z;a>wkO~`1cZOR;hK#-Q*B(k>MIdTQ3b1&$U=ML)726Dd+tTRe|4g!%KqwpvWE7(tA2s z+7DEwn>!h}Rk5}@6(k16DjE)9Ok24~dDk&=cCoOV^=HnvT6vbvY3O|CR__9Nuf1=d z;d4eY3t^8~_^TT3g^S06u)PhtD~D0^6EZLVMkbqj<%jSW+{aO?jCq%%Z<@|y;eHH7 zQ&a*oC9aWsYGLWJt1|)#T%^F^A{H_-Kj+l-Uz3u>I%WQ5EiKix)pPcqHrD23yH-OB z0nmd4MxF&#TTI^^|3kFY+AoS|ky4h`Mqg%x`E!uG!4xw4fVTi0+0x3)bfw=a>-eHK zZC(6AeXR4S%gN)M0VYU-qg6SHv2S5pFF+wDvwq`t&tX|>RPhIrG(KTEpBC)ZzJ*LShQeU%)I%4l*c8v|^09%W>= zL?SDv63+41`60aFy`9<>0Y@i=fwVpa3q5>9jA75Fj)qVB%<)}5xx!YLit2MA2QJim z4u#IJ5htjg(I652f+1}|{Z`(kPN&+Ov}V&pWv0AQbdh;4vfgofTiD?);czkLXR$>i zpu05wZnpU6z_>gKlJ=qV~Mk@j-a1Isc|+o+L} ze<5Je5$_YiFe0f1w8W|febkMJidtjz`rYB%*%A1q4}xi)}$f#}R@hKiH< zQ)5*MJqJDd!W=}TNDO$q(DT*+9#v6-_ZZCP|~Qvd`wm;8o{-)k~91V+3oVrQIR z?Ioqfe`Ux6de|09fsqc(%{=aR^qQRjXWsTBJwYu&BBmi0%yj$7;=}2Q#5D11Z3+=* z^V#J1b?}J#Jp%iQ@bm`(?PUGJz#i+}^QbLyF3j?q_nz*K0?nBgA6( z&X%kuT?ofA`Cx0I(m!1xdU5c-e++BWDE8-rP&t6RIUhx&BD{~?SW7E&cld6$7*-H0a zZ`3It&3Ne~@c8}t#pUndG3RkCB?bU`#N$;)>787hEkUiRXi0%yaK`O_+$XqW7PU_| z^5RgfhJ~%Bqn@cCXwW^ZEa2tZaG7xlt_cGE77zu;&aZ-RN3t<+|?x>}0FzTZch@uG1nb`VnEsc+HM#YC+aUQ%q3DFDc_v>~z`uX2tgRQ?J zpjrWBI|~sG8H}2<4>F)lANYJ&&!VgC6dY4m2y;PGh{4x+;}guJU~z%kmKH}-#b5S= z8-Os>cyYFYA@<0P^+rM(@`1$!`wDN8z0ubunt{V`bfU9m3*NoM$oM7RD_;pDns{rA zs`t)E<|y!fR!BE1X!coE*!%?4J(rwFJ-H`RBTi>i$7S^|;gkMEwPO)Sf&|yY> zPMp!jg`S@gslMM>uWw)_?Q!XCpXKm*^QBMXvvG)?>KD8Vd`Ug*3)Hy#=Wt;nM#$>Q zyA9+xHhH0M9cesQjwO~}S9d)*RaG^f6rTVeFqouQj@hhCzNs@;Ye*uE7?idweeFlW z?=4flmBs1wbrPbgfR;OcrpQDfwRh)^PW?W8p-o7gG+0W;=_;qIa z@ycS*;kZOHLJPb!3SYm2_${={MEtcZ@L8!%Bk}F-{@U+kE5hUvf~s+?#cKV5Tz?@o zlV2fqW~}{@v;uBcTBu@|JkOW#ykG~gaJgCkHCXC{%S^_h6Ic52ELTWJIz`ii*Wq%l z+2Bg5P5J0Bt<{8T`Bt!2P{@Dk-FNISU>mk!h~cZ@0|*4z+%VUen$2}e1@*W@kO$(k zJ6$MsFQ#ua;zF8{C|j>nND+`Y{k5e0JFEPvg+yRmhNkzvSA@()n=#}}W9Vw0a1zAam%(8trjWd{LpvGAf0V+OIr|WXGaPBvfex2t7 z!XXLetoF4Sbr*ygErrOU`Xt0 zm4WU4*qy-^?x>Vf7$wabMc|EYDgy$T$1~0Td&h(e*RBWi`Scq*O61Z(Qh;ZiEnqXT zfu8w2tYWkUrQA+L$AryWV}$FShEJ3HKVE5~SSG#P!OSClQxc#%^Z8N<7W^4GGyJ)< z;P8MY+m$mt4V`cV_mL#W7~hZy*Z>!_IadKvZSt()XsCATrgI@SIK0$|4xChsTHj~; zq-B8~85^!fTC6Cs64)6}pngskF==|5F~^Tp{G)p?0- zfr`$9WWM%`I%OJVN_zrH`5_Rl2+`b_Jln52 z!#nlbi8x7w4iuN#NrsrmZB7dWw<23Se zH95VP4OyD=$*Y@)-*6&ET6R~*Tq47WO*9z!u{u)0GJmaPAC&VX1-f4uwfqxS(HU~B zU3%)$V-9bOr)k{eCCyU9NZ*uRr=eiU_9s?J7LzF-X#^J5m@yhsf=zsR%Rj< zowfZu|N5SNSqZ127r2UZJf?ZT;km~CGyH5k4CH;I%ZKN(X62Bo@&Rdto_2U>=p1%K zk1Hs8jOaO#2}yUPWn|zPoEkShf$#-BjnsEyH$Z;gfx29ktxUnZL49)#G!c=^+lWC9 zQcNvnBUfxTYHHh3m4MB7C4nMV=Wb{Y9H>xEVhL*bt?$B#Fb|3Es(+Y&w3$vbI@YuL zZr#&by_N^@1mQh3c|p54KjOij=wYWze%b~Dn;8ykmdK6Ek8X7vq!sV@#)V&*Lo7^3ttzU3G z7MUj_{~q@CH7ST6# zDx0^Ro3!;#&Fsqu%Dr6xy6NV+s`ua_%ia#f;x)&_&iKu%t4N5f64j-8A`#MQIfNX&`~>s=i}nC?Wv)6RCVO z>N1s*MIA%Z15B;hIS43PopQEad$fvov|N-I%t#HAmmujf@@wd3Q7j33x1l;T!D!IM zx_JCkxB@RBZTDn+&J?VC_`GLCqrdPZiP}v}n*51AAB|y?-zQtA%vqSt4Tn2_bYHa2 zonz_6J`Yea!Ya1Hc2;?lLn!U`-UKUIo4rMHn#QSGVV3Zcef9@)mu)8v{4^V@ltW}y z6Lvf^7!+<;zi^Wu<)asO{QdCRX1Nkjzcr0)cBlQL7m!GIywlE1Ui%jLIrdke`|W|+ zo@1m^N{zfqH6-5JRE26sgF7^qMV6MlEXlV%39qrMAz)~^I9dd55G#7EyLBW~-!X-i zJ4A&hcUnhVO*47fV=eLLBI?hQ@@G`T0e^2>&i&omKPC)k@SUh7_Rj~fNY7smeIabT zc7~n~lt?$_+g;8W(K~B{1uR1m$7yGEfGX0|Wc>;;YSOe@DS@apUD7!+H4!Yq(+S8G zbo-iI?!BwgUXwA}F19ub|LvRmyUW^lj95>#q;71t%Kf=lmthW$i8%!QgjS>=+O>S6 zIoX&@Wv+|a%*r^wwFv|o$0Eo)UkKb&qAk&29JDr4D^S0&4=jt zugec}CRXezXX2OXg>McEU#NyJ1nU~_J{_Ti6SiEwL!;GH;I4di_6YX| z=2?&xP~tUH_2jMQ2R>WUC}oyS)cK*49k24R^URL50;KTdS4gn!bm# z_knKt<3%FhMCt9C_Z$~j=J~39+1N-O)xTpP5wvey)U9JV!m6T&RIS>#U`f# zq)(aW+T!x|@UB`1J#}e95uRXqwL_J)piC4Yw!;s7EKKCAX&fWF=b~jQB-DyAfbzpx(4c!t0j_~M zlDutW3r1`ur6KtZ+5k9SRU5Bci7vTFso0F-(*eM&>>0A~cfN@gqSlEl?=`cu*d=Qgh(bEp`I+w&+*%Z@P_6?LjOGedsYusf zkaMD=q5ImYr>x)FDOB6(z8On6q1@i@V5IjtBYDY8@_EFr@2KpSTHEL!aKt{bfBLsK z1%H`MjJUynv8M5Ezuk)u8z(V_UtlMwovO^9mUzkfQmqkzu?$aH!kl!Z{P z+v-ZbTc(cUv<{(MQK+VL3xqTZk21QX#Wyo<$UC?-q)C+9^hyo^7&|#?1}8I&b#9Wx zOg0q64UJwTw&h7lKkX&GGk9@5>{dqAs6QD;uNL8Vv@jBx!CI*2f}M0+>LEu&k%`r8 z6E`E?{Euu(>O`qlol(1cgE@{~0e7mxz<3na-WB;d?Y&0X#zZ4(`E|C`X8J|N^65lO zXou*o=}VRWUowL$kjHg8SAW_C>7RhJNF!46DcN+hb_T*#ir-~ZRC*ZjJX555E7RVo z=9eC}C;_eyOm4_XIS@K&xplcDfVdWQWFTu!5a_16@IjCL9kkJU9&%B%ru_I%+KhsF z%@~fux!TiAX`c?-{7Z!m4*eaM8pxt0DI?QEifKXG!_tW?Qbr2Zyf~@_-hDXua-2*T zYj?s(Hq13;ESQ;oI+k1q=4h#t1Alqo^RRJRDNyq3{M$Xl{+sP}F@#$!>+U)e zgK457WJX2;^w-^_0fkSc0CeN!iu|3CBkKRjRKLjeh;fp|MXq(HEJrIQQx%it1+{h$ z8ECj4BM~;P92Q7J)}=%bdi<#-k)vxm2@lY;RLf}U=Fw{$6v51Gnv(JBnwZH?Y^h`W zZ{Kk@EXV(oq&oW0^{5&E65=Njxe9S)k!#8phHa|)2ApZJM`pyo6N&_m#)}uL$FW61u=u*-yHFmUP;Nb zL!o@auYSiJhMIJOCK3&Cjkswra}(Vpq?e~jv^f#yTKx@QFu0M&E#-ZyWb_zmxS3O< z+RWRop{;H4*b!TPhCAJV%2NEVKwygc3-Q;#(ZY%TWDdN3&7V85ZV0%Sv&8BtoM~ac z1!1WJ*(3TZS&WTp^lg4t7M4oVR_Am>@|}&i3dBMWha41oW!?q3a`^3pPslL876?I4 zHnZH|lVPox;0D!B;&9vV{SYB4gjAE3)(FdSKjVdE_AHb5X81u>Z#P zFJ(^cZ*NP8j!A!Bdm;IkPlhIDe9T(Qqrrcg484t;oAz_P=?JuiKTr6VW z@xXLy&WAM9?*Ih$9lw@_MaAyLrPuD8SofS&GYqFEtxib2i#oJ*lg#a2vFRvi{lF=; zn2w&>WU-R1v$aF-W7aF=Bk4IUB|d62YnJv?f1!vKOi`x@LIp{av!?s!3#49149cn% zJ@xrQ&BfpdccNO_4}R@wXrlWUB0sSx_UME=YV7P}CUgM)5IJ9#Fw z!J;S`VY7oIqnI!KWNY;nTvs1S?NVaCQ?d$D^)}FED{va=+TZBg(lx0hIck$jNtYz9 z&>Bzg)!nSKT&IoKO>%4;740iYMz(_H4$kI^<4L{R$*Bj+hBqo7PLxXIl1l&mghtS< zggE=9U+bA(fKdtc722AAS*!6+N;H2?yb2vpbV7l`6D&P#NRT;o@Jk|6+ugzZgN^+4=cFqlFNUc!}u^NPNaPTx{o zyid}9mWbVrboo+JN5}G^5cV%Ivg)rvRbz(OjYFZ#HRwjxb#DD%>3U*3sklZFb1E^N zQDYTIoOqxwxFPc?UI%?{^-iF8_xWGsS0DPipxKPr=Zj}nYRfOAlk2Ho*{7XwBA0PJ zFRRGJlD+c#M|a^NZ5;VtbNFV1ff806J(S76o~j-vSaCKN+IxwGA)ZtAC23D56VHY8bDEZr zU_Xng-qP?PsRWsV{NTIgVi9=l6b{kNS(IOaD&JF%s#0` zZ^-xR<>^#yBZ;}MjFCN*S{X&B^5d|!aU-GcNYv1hY&+T0f7e!$0@T37|U=pVl0ShNkwt})7cOb!{H|>Yi`l|8Qs!MINKlq0VPUh@Qi82tMUNM zSg~#GT)u1n!A$W%{f~{2qgbf%eY-{l>y85s5xIfq#s!}1zd$F_(*JhSb-1{7kG|KR zH~)#`92s4~wPL+rha){`(_y-Xtgdlb&j=EdLqKa}AvSYu0T$7UI3;p1S@TY1gc8_r zgYoy(Cb2A(va4CskkT>`SF2F(2Z>vmxr0X9RK@wy$jM+g=Cq0PMbR4UTFph3-n64+ zH@fYfR*tq+3Dw8j#P+rp4jXmr6i4y1?pdjfO!6;WsHm!U%CH6KU3vdFaF+};tJy|n zmj0s;(fSJZ62a5;(1+kk8<`UK$84;7FnujvwpAkLvvzm>StnI@Gh7sL-oPf&G864W1iz$&R1RWPi<%nby#Z2ybAV+85#7Omg6&1CE>EK1AuBok)>N&7fh9}l{qMALga*x# zEr#TS&)f7M62{KnE~cs6ra2?ogBEo|w3@WHcCb8EeJaVA6|HO8qfon&f$@(O6kw2CgU+J1%~|?W zO8)}?js@yOFL`;GWS^+XZ%c%Agou-FQH-$tISC+~K6XzlfXo-|Iubs=W_+TnqQ7=L z&?F`WMpFoCH)j(WZ(YAhppq?m2Dwa<4znU^oqYsGq4)NM(_> zTR=+IkL+k@Xa<5bA;vY_`c6@`O~o5gHgkOKhoIwQ>|;B5mtj!fb$ZaQF@=(=LJy(V+)9pTslw5;n$FyJ|(BGXzFR!)QL=cRIRou$mD!`V~AiL=(bxNkYA_ zGBR!dx%i-fPm|D-lE zrYi!JL@iatZahceLNto;kLhQ6)o9?6ck~^Mly*s!*7m^pzcBf)VqWt)8Vq(d9|P3e zn$uw_$lA4-_8Dqc6^5J6s>3+o=}9`LjwlT}*@4l6o`$po)43F)XjE=%LBupq{|f+r znFvr-Ch_9Qer1okj8a!X*kkYh}I z`Y6eE&@|9OKR0RtvET$UT>3$?qq++8@{;i<1&zaoslCl+CdO`6DnvyirOnQJeM_{4 ztToMd9_`o5Qy(IKgX3NgtenI#m1z+plWF%nKPAqWFEO4rUj&pe|1es;350KJsc_== z`=X6=Mp!R?mNn4*?-y7;qml&TFHR7jm<`#OBa0m_B-g{NvPjNa!@CT zi5_W{OP(5so(c7vcy{$tjy86Kus#a3)zbq5^y?KEn->#eF@27I&t)`*Tx2_IL{N{Y zp=sceK%5Ylmur6^QAupX1o1rNPPdonbFtO*_b}4Q*O}K%n$35w{yS>_pF!n7kZ?LM z)RuXwbV!g^Ql{E{ZK2zp-y?lfm8Jya^DA36G*50>(|LqjzwV#y{X^)Xmzok*cUj+Bc;=6_M+Y3}MWp2t zqbn$-?1L!(EsEJgMX^9n7~>~(F1TX{LCuWr=-d2uLz91+zrBBf;m2&>HH8c_O2bXg z&IL;VInH%4#8r0rD}pa2X5t%UGL;bkqli5HJ3o#AuN3JnF(JuSkbqzQgvj`=2j@me zJ7i*$C&@51aw~qHLh?2!i|$w;m&(5id@1Dd99)I8D%hsRj55RznmXke%o>!>**f(_sg?umM zZ#CHpX&Tf)NCYIfSqu>+5|Kpf@=Sx?F$2vCKkTC6E4Ru9VaWFs(>D7o(XgEwN!uH6 z@#;gK6^@S|{)WalSzi$S5udN?!Aeqchj8U%Z8Aqf}w2F4n@Qq3YPNxFr+3(H&b zAKoCcED}V>wkWNnmYIjJg)~LI?mZV;KLMV^3XOljz-g>?1hHx3b+DdGk}Rh&Gs|GT}?;cnbNFJn<;rqGf|Mj|Fm zbl6ca4)rlKK_$Ns6SWVk3JKM{tcu){hN}xPG|0|-V*MKt9}Lhl#7xkVzU4P9TGwen z&))s{CbcD+1fq5}PU7U;u#2?xPO2jf9i_}H>Lw*)yDPtCDa<>YCh2i%!YU$ZK8p@| z4SxragX7rqXS1R73C$e5W8yw=j@&yrs9;}PjOQ?;Hl8{jQW`dm&>(|Rg6Rn#`_ zm;xn;|7735V5m0p|FQQLPH}$Ax-jkp_dsxW3r^6%-QAtw?gV!m+}+*XJvamiA-KCF zNJ#FRvwz9n=YIFQf55F%^-?ucRL!jE?$z>iuV;}FGX9-=BzGi)dO>g1gA`qS>=jjT z5J>6l$=9nTUJv+uw}RYMjTL77F$*9SI1q|ZT$hx(d1_Si_cT;LsZtyJ^ie?Kdw@+R zkE#hL0|KaOzdNT^zJ~TLwqJo_iAty<6tOrsDx1+chBDR%oP^h`q4$Z(IC<9-=9r8l z=Op0XPk*lTKg0$5TU=~%>?(g&yA=a-r6Ia{{LHw2?dV0;%!H=Uw7?@GVe+15j0-`( zs=YwgGaEeogdV1OjU45!MN*znM%IRbaSsZg`}# zsOgDAduw%QbExHa+hu?ns66Y(Kr4T{cN-j#0tvu5uU;&y zmUw>h0tQ}OHb?F3;T!{pZ6)?T$NaP|JXq_Eoak(3pO z9eqmAid-m;r-cW>{%cF}lJpRxUNjc9y{z8bYkleje&r{ZY2x@K{sxv`pTFk)qH?`R zIqFcY%69+Kz(~_#vXO@Jw$(WU5f$DBF9#N@`yURHRGw)oZn1HM!HA;CYx43!ssM# z-!o9(X>^yaC;qLMs8wg%O1k!}Tx4t$$D(O%=d`nP^aFl_H8ppDJf_bfc{+G(Zf(yb zso&LMyE6BW6#Daj*D>W36kUITsS9Yxon6{-86I|w&{RU{Tw8v!MfVhK^XM9^l5#Jb z)DnM?4=pIXt>+b=2tki3SqIT zV7D^f?@OJ%e@i>`pdd;>@ucArlK)n`EffN`crlK2uZumxaTpaj5aA|RcUEIE)MuvD zUqdZD>CDgtzv`8EaV|r~7G3(D_rngRl<}6_nK5y2qPa#u<+7b;WZLP0F>#fDz>?0x zaFT;fTN1O$&9b-bQn;!0KVls%Li*xjlbCTi&>#OJ_W%9@Uk<9QyN^f?+8q2xR(~1^ z|A34AM!zeR)m@6yzoPqZFO0T+d*%nSq@aIqu)o0PZ%A$Rx8lD&jPw2ftiTh9e{lX0 zX8Hf;1)y;MHeW%Z#=jNae`kY&%LN=>&>M1sKU^yR+cG}^Na8XB6K1tPp@VKWE#MezgR(FbGBq6Vc(f z%0aRbb>;7j6`c?pl6xJsgX6%r4r}_l!{dYW3gM9W6U#GempZl&tw#u7AW1`zi|<^a z?`$6=MREUyYoc&Kx_EoLr=O?&DWe_xFv~xk%F4&H+i`oHfeL7sf_4z|>TFsF zPnT_QG6Kica`h`>QkAX51jbZ6WoF&5KOd~_WTfWoJ7@5fougUORbDgIG$>5H9-Op3 zMVp#YDElA^2`~P(oh)0#!r!07`*a<)-HBPDr@mf^RN^&bf9=F;2v{Q}|J_9W= zV+VJ6fb&E|9;1~U0|SGaWY%uuwoJdLniPHZG4`Rk@GuYZZfGd}icJd-*Dv%L*Dtwg z$(v}g{mB~ zJ;Y&Zl6tTD!^jk>&~TO99#l^5sBraEfeeaFN9(nQSZO7aNZDCcG z1S6h=BCA+Y)-dEG8#L0kMMAGv{EuxY2I*p8m1+)JJ)Fb2%vb;PB!cFDX9t{P`aS=G zp?E-0fF7@Ap}BujRPTNa`h{TM5a&O9%N_qDMJZ5p<2sovd?HO|rW8J=>Av$?gqTE6 z+cGrVWyLyy3SQ+`bEKvox;Xtm-w!r`&zoKq{mOHM3?p;LhA{}3ZiAn?>pvsvM zZ6tlT2NYCRktnKD?@bP_SMXW42rhN8sd^Nm)}0!iu1}h@pV$x?9FCcVSJ8-%cfC z$g6b!ilai7Gj3K*IXT=w0|{J-u*d7*=R-=kGmsnZq&p*aT+Ux7aSO5U)}HYxdR= z#xs|t8BN1cX-A~l6L)_tH#`8T5L1#um<;lYbLIoXZ21{zaO)is|Fw_kQa)i;Xv90e zZ$&j%?f&aRs(J1Y?!Lah-rkvfc8xP@|D`bUIEZk6$ON~}A*`P-Thguzzp6+TQ#SVY zC@qDte%&8Tm@n0*t#s%h(IJ_`G)3o}`UyG;7BR4? zjEorC;qo1OQ|Ns9RuFy=;Vx--gl#l1GXsC(t@UZ@pTFI-|Z9mewk zl+^D84GstS={a2SZ6oGcDyLxdnuLN>lXkC|2+`wr#-*)j?QZI2ZX9|gLPV8}gtN}X z>9%Nd?T{z7nzyXHrc#+WJ%684nT4H*q}etOfRHSE$G7c8qO|v|6hs4#UUcR$X9CPAoBz*-mCXUA|L6@K<+B=FA+H?d@C1U`|_s3 z7|_i_kYlFmq&Ncpi?KvYYT!!@!)p#6x}q>?R)No@aFs%K7z7;%^Y?+;SC)RO&s<6g z=Knfi*&jyZ_#}=-)cg75Pu7{u0ZCG?9AN;kd|Yhy5Y-%>11vWH2hF8`DFwv#b)e1g zPs0abw7c0Fiz7wf=e)%;TdfL}M0>4xP&N0a7_WN^w%4$CJB&!^TSI(-`ZapA$~)xd zuq!8>p`Z7LzwQg8nLB?zod!I&1w3yJe83M}a$CEEW@A*rbp7X@p-s{*yq3<0D0T7Ag_{I=vkOtQAUel(z zFK3p90Ji{-dcj*ifGp%RZ=wuJN;6{SK}*3?Z3nL@UFLS9Lx2qGG~l~O z00_C9@)Rj{bY30f0KYCi;Kwd)F@!I01TUVsX>+$_^Tm6VBlns@_=ZBy|I+u6U-%|5 zo^+24atO=Kp+~Xi+dqYd(GWV%i~d>IaW3B<4{(?UcxrZ~M85;x*~f;w9}~a$-o71E zeLK6_zInA5Z!%bPJZ1Rv=FN|RH@*|H27@HNib(qrIM^_IA8czo4nP5S+Ey?}u&*N; z13N=HMRo<&oDmdTLwyj~R0TS3nJg*qf0N0*@445=*04hA!-8S@V#BGEFj<$3oq?-mlGYC1bH?PdNP+P80sKl!N2)oq@+Zk>UE7lSF zs|j~l!y9&qEKVtT1@UJI5K@?370F|RYs zI_F>q57!e#Dz+ud5?_pED3WUkxX=TAs6H6~+=%|QRW?=D65W$1*DEt16ldpnvM!%4 zcXSYJ8A&@m&NFhe@|`i_HxA>+m52~phT^V@sY=*pM~@=YFs^Ket)&kOlmFy$;^F5- zlI212OGVDbMKWd0%B)sJO@4TLKW!;=+c?F{P@A!%H4{AWT0(w2`wS0tFi6#fPbVMw z2bQFn?*T{N&w$k~G{>%}#`E2>QW|to1bL~1O&<)pf)B7FFe2jU=*renoFcxk;!&{2 z5k_9HBJ~h5SjDpM_z(~hH!-1(d0%|{dT3r-!@C>oTAEx?eKJ|8w#@Z95I)u8Ynz?s z-X)nzVJnx+)I|t+_pe7DhMaXdgiewS9qRX^grMb>it{Y^ePeN1z2dX;6cdvlQ=rev zIVGj7-IPW2Ihx;*m`0z*r^(c>E48*YEvd_wOBIW(>*^_hBhS*8%6Q zY7rP={E+0Ff!u>%`etW0HlPTrPZldxr-LY%WbIaiy3o#;e=87rM&R}?@8wKXKcZlG zml*<4sMJ7ee+MKs^9VV%+ht(4I%QSlgb?`=u*c_Q1-^ZFVV-lefQJuZOicD5Vz<5T zeZOAJ)AUyoq$!`g&s#oe)1Ni~q*kGTPT=IyAfqfNsM7ANjSKNUxy%ZVjYd3(h+)mm zU8&z|9XBw2iYY)3%`?1{?~i2 zv_OeQa3$9XMZACsLHAK%v|Y=x25yPQpn1tT_?%lXa9l9};NEH>KUIQcbOko2x&tDo zqR?aUawAR)bl{S%zAmD!Cqv3e2UmUA%53UD>g4@BxTU$5F{b8o%>q-*`7OCt>%3flZ07phw;Kc+k<64y~fpCIU!lo z-6QI4n2QfNc}JV;h1D0AA~BK`w}dSij5WaOj-ma)hU+oBB(@zFrYS($&x?jKT;Tzk zjGyjRqMEu#V#b3J-{#8ME>3IV(4js=WE_eLy#wn&9ZfJ>TJeC_ieOF5i4Dgr;B~OF zk0Bjz*G@I*$?1NRS{_J_;XR_RoPva#z`L#eX`LO*exBtHnM+*o>u46Tq{jd(-l|@O z%Cv2D16*5n^v8(l8z3ItmZa(cbPQ|YO|wyT!kKzwu@@q?TUybd_k$luzl4&Jm`P>y z3hGCPZ|%5{&l1269!$kB;Om1$+Wf7x265$rM#~a%y-(Z@jj46Yr7a> zC>^sqiaa4N|v4y)QHNoLjQ3p;6Sc2uD+47k$TF~<3&p?H{HCd22QuOBk+Kk;2DYqvcLH`XTycL zQHlN@CVv5=pZ>}!ZqG0Vw{2ZdtDsP}mTc9v3`6bDZ*6_|TW#2uM*8;G)sa+f_ul{> zriy}#>g9V;;|7aT870j@_grQ^C^79a&8*puho?oC7Zy7{l;1P}JS>@a_6f_>B1WIjDd(I%hJ=ljZkz%ot;8<%OXXN?e7y||hmJEj@9;JCi0|m;x zto0owiWmb))DfwtZ~mM(VM;mDb z?%9sQHYOM{s@{_1t496d`{(nvL%A18B75-y4!NetNK%QG$Ac`qp56QpuXbems1qAx z;ItavZh|#*q_sg82J^NtQ&(T4GwO-r670IfR&}LtRw+ial<|3LCZz`MlP2=hv|Ij*D=JcqY%I%N9CuON{1rKwCaGo?7O%UD!n5iw(G zKgNM_xfq*6PnM~(>{QMjOY@I)cUpZXTEBuJ@bfi=d|oMIHZl8ZE+97Cl!HYYmNXcY z!5DOXY|WFGW7Z|q|2B}=-%$pE-r`S`Av21#U$cg^J3h_c4T5Iyj#3wdl!!GK7zj-r zI0LvvFnp{@HGUdsJL~!YM6gejCrp7~m^`G>Qm@N$+}C@)Z$6g^>;LB5*!1}72RKt| zpNiKjC%#D`m8U1n(e9JM<^pWveA(UP=jLat4IUd#W0(HlyPS3nTI+1U>KYB6xCQ$? z8Y{P{M`Oeu8!z9#2HbuO=zZ9UzU;nzrCS~+a5LWcqjg-342xxJZ{XKUZ!#_9o!RYJ(!uP3!0PUJu*w0Jr7Wr0@rp&d9LKZYi2V72t;% zmUbxZQy%ooW~*59Y_7j4>ju6#B_V>Mu7`V`S3**J{&~61U*g);`9}&}^4(F^5-}aT z7di4r*nVg<0j1uB9FLzPpG@L>Lllxks~uIZ8ra-@S6eB*xEms9^4*Vd5O_Z0FddCqb6u1d251O*#$Fo6ejZ@$ zX!jACsCXK_pP|{`g&MBrE>RdHY6~XMc9mF?}A4GQzxM zQ?S6bbrC#`^Ixgc@9c>p2tkf3I7<5;&!~o=m8Rp~nT&bSeXGD#j7XHp2vYY>(b&C6 z>$76uB$f4$mfJ&g(8pBgY+EXW^|TGOBYyv->m#@BT7|_Yu*CD#_K^G9zG?~uvK!nT zo*#G`!=CBjOG);eOdQWnM7-Zw(Gt8H?rJ8mNjg_f;NSSN66xDgv9ibB|A@dCqET=~ zn(e>drD;!A{xL|b9gMHYv5!h}^2yk9Dc7n)?Sw|;~B1F+#A9sre;=m+y|<))<~uI`HnWSzw9 z2)frQ9}(bvE?>MX1*LVp?w3Cmb3Jwl-r)KWVpTR{>ObVv0n^|OB_+&k0?CYY&9|~C=R1i?2f_hnjw@bBLKG)$4NX0l61DIpO&ID%wN2x@PqHK=Q4M9 z_r>Sz`C7?u=(1YV!5RrrT-FQN9Q$~`Vp8_VF(>p6WHYD9!e2Lj-}L#IJ}>IKl*ps0 zWz0K}z$@_0{Twi9IE5B6hyt^P$EB_ix?2V+`91;0)>U{oRKmhs6v_u!+;Sr_gj|nJ zFUdog${T`*647L-?7`Q3< zFT1qSb9C*eW?>1^#Xvz1PzEbmC}9mI%gg{CM)>xk?c8^>o2-;&v^7y>EJ~{s%rnb( z8bj>9x}EJZvj(JqFuTyKs#Ud}?6F``_;%}i_{PiLX9Lge&=d`RNI%SEn$MPeo?Yjy z%Ez~2fz{UJcjl8a&acBrTHLn30UHjyKUZ*@!5+Xb^Z^j==$PFb2A||ph5B8Ya(#Vp zk8>un3d+xj6qi478evI_6iPKV-vsv`3&sI%s*zPd5O2ZncN7krp%v=%>4g9?}NPN!dq{Y2w02092A)u0*mGjW} znW=S6f20P&9ZF(}6MC|0Orff`E0X6X=;?<-*4h5*eR{r*nOiKQ99NF%u14(GpN6Mn zHIVhbGxS@-)nIfIB6H`FrdgW4#}{x(tnBNB_nc;}j|X7Wq_>~#qN&new@fj234Px{ z=A=eN6aL|dD1Tbh7L;9W8pj*9+6Z{_r{h|ZhF!qzv4e+ie%>w4{-q^zo-aE!XK9YdX2;np zim&6kYkuFqAo2kK1P)LbSc2kS^LpPMWPhk1fIC$lcu3oB+;E7hMoK|#y*+}rGroZRY>NeuC-bi5POuxlqckO^F&1y+0duWA?CG*V>W># z=jCb$Z883x7O?w+$a~X?Nz9`BcCg-PDspZZeLw}IWc&gIfbWE-rCINz&<-!XA2F%r zJacR)zRL6sGK6D28m%ryZlcJKqFM!dN@zW4UMvi(5UTwI-%Cw$wliu zk|AZdvL>A@$ zRrh^^O$oa8_aupOw3VtxsWb7Wlh&fxhLUOt_Hcm&|YYaSK9qVFff=dQ8);uXtRUTe)t}wDrI^dBe&e!sQn^^l@SkKVO zdhqA|*m?nsO5%PM$a%#F3&t}hZXDCDjen&q-^;)KVpf#5he6Q-T)`sJi}4MBqD+Yz z$6=zGS|bpsvXowr>)v5Nr>(Ud^olMaD5}AJ5qYu;G1LBjSi5 zBJ?;xyrk!Ie*ZePsQNcKGJ%$O)qa}c%EYM!2t`2Y-S}HLt0wRg(Xl3wSvW5R9x5Q{hYHgE9qFQt~owN&eT0l z_@}4nta;i?7=+nu=E1?ZI2O8}W=at|h3o4(AQ8wB@!3nELH7^XI# zpgj^1qDkJlD;Gn1sYJU6wmlyZ@p^Ew;$`}T5K~Jd+u;ymXUALs>S*Tr!0t*ua~X1< zG9kpzGxF=NxR88ph+$JB+>cArCQLh@POAKHNPld>3)3D0*G@f*WjuB{<|p;uk&DLD zBRax}EN(}aXz2&Mpb9BiFhh=CCV!{<~pv?~hf!5_q6+qkCYK72*BArN$HXLH0R{ zugTs34C(=iipCx9O)(0BoQ-R2!3->sC3=`LAvj%2i1uqrwAi(MKfOnFluHqaDe*f4 z2`7Vb_<28A4aP7ygQ@Reb9BDlS})6k>Srw9>E$12jk>5qC38^O?z-fCmL2HexJgGq zC8x}}l0={1C|v)t_LloKK`=*FHzrMZOkL+myophNL1x!wuWpDM2^6Yns`n>i*eJ^3 z{OoHp@XR#DcI3KB>M19+v;ie1QNCHUR)Z?)hBbj2gh`OWDn4R}fT`mep zLrHOHp@sVGrYD7RAfc#7zRj_Oz!J>eky8pbv>6L=m|ni*t5=%+qT*!ibTBD?-yTIX z@LrHc=SFrDj)5W!X0PkwiGdCV(o}7>BkVwDGy_VrG}oKl6O~Lw49+CKdtQ=Aoo)p~ zWb%*<7%hGmAA)>HZaoDiT_#Grl56Xg5ra%n7I4-bKU(soQ|(&)Fl4E|snV=o(}+Bq zD`E~d*-Cw>Y1=%pZP=Y|1NG8n&1lKEQ|w3um{!<&zX?qtkJLic8cdQBm)fdwRT5+% z797*Zu+2}#rGX&vq=1V=V& z{5SEx4oYOuo^g)da^BtyxOnXQ96Zb_T<`NMU5m#|1Q|oFlJRg)F z=%)fzc4avkoKHmvNQ2yrH35kdGczS66@bgbhCh@M(!*^_4lQfwO5AAL+GeqiyGS@~ z1$CM+)B7{_E^9EwT)L*L40tIKP6wt3O~v;<1KroG4Vrd7EJC=^U#K zb#}sG=H{663g(=QFjN=EW1xeLJ>0=gk5&Va8xZm#?7v!PI`Bx*6Sa&fKk*(_6FVF0 z>F)WZ8#U#0ge3PnI6eJI1|GP}DG11JBArJplxVgV=DG<8NapkJU#e?fWG*^e18KzY z)!af2ua6gs8K0N&wkvefy_y0)B_6CiGDCqdx+w!!SITmW_b*HWlZFPtFTEsSfF2R^ zBkI>VPexnM>p6K^1aT_pr7oxng-J|@4ViIr_;JgI=NRw70+i~?_ARM1I_J6=QKOUC zP8cNMI-KZIF>7E*xUs-yfC|M$IfX}b2vc_4|AbqmYxFe(b0R#O#USl5lR%nPqVn7v zicE&F5p#PXXtju@qv#?id*G#-axghPC@ z`1jYm&WK!tE{}uy)X}r-?gJ4bB|KAyI*qF^JJFi}3}L^RJb%D{7yC#bqDH=yjTLXg zZDxL7tAeInQ5Re@rbO9m@MX=}s%aBb*4JpEI0~W+@$4eI9eMP%lWk#6<;5*-`r1-U zt5$;(=L3CJwb_#3S2OjQ%Ck$`_wRKqGR*DT7g@q$P?5w`D>{sWm3m$3oc~r1-7LSY zCYl!RA4um)5h^tY2BQ$-%YaQPg2~qv_9@@Fr|blg^?DVzyChhS^C9+CHc5qq>r`=3 ziaEVlbz2t~Lxx`fKt2H=TsOly&*Aeh=E zBN6Y=TglCV%6K`!)(l6-5tq3_7p@|Tk`QI~j7dFOLMlIc@&$c3k}SNFmK}yb0ShV_ zgzR%ogpfEwfjUya`@j=(TxBRIZUrDY@m~zoi2RE`kN1b%byz{!Dj$|-VYd}dwft~_)FQ5x|U0}xeW z<#ri+!*g=D6~Wj$rV!#+N>yeREQ7dcop6=3GLvBkCX7i_EPcLxFWp7@J`qcM{Fa7Fwyve z>k{iQ5$p3Z!ln=6XA^U(qV)fG-UQ62>hijhZMXjT|lCtsuh&aSwwSP26&grqoH?=F}C9k(GvGD)wXv z7HFvCP==rhnRHJPqa-7sb=9WCS?zC4u$h!C!yJDvTp-R6Z6G9jDZu}19ZuZZmPCVf zjzh9Z)kfN@_N11+%Syq5 zT>45e@(R~LTa6hr$uvuf#uba^F=CNy?%Nu#L?+NQBwM~pEa+k;Luucoc2;<_}7aD>C^l)@A#o(J*1ou8bz z`#<99+L4K>T5uPb<{K?$Lx1wK_#t&N-#U$4B3Uq4FmxyBTPSw!1PzI5!3OSFTF50=Gn3j8u*%a5p}!q zPZ%SG*-U%4-*^0P>spfPXJ1}OVa|WYQ@=u`u2pEO(CAe5No!1p1JC+hBT7xdVU1fO zju2XEUFvu9{kk*IO0@%l?$s@?%@S=H6bA7UUQfSwZU$^?Z?!*D(lG~ih=c!aR<_C4C{VsYGS9jK+>K;5%1{az6$*l)VcC#0TC7@+}0@t!S@qb zA^6;IvQYU}=u-$V{wp6a|0xvTV!O9ew|Q@y%J*PVdGk#31Ps*7|s57|wp;Kr}sZ5t)^TAHe9tzX=2K^R{>cVoSctMP0dU_aLQ zn?G3%)_iqsQ(Fm{1z?1`VJyWIZ87_TVlCc9d0PJBSO6hkWY=|%WOAF4Ee*75--gpT z7?JB&9|-QtAWRO9(*sv}=$+gvXKID$E->VI9b~vO*>~)xwmwU5D_f8FFDZjCsJXxT zHuv~b$xD{SBs#t2wD88`XchaxUS?mrM`07Vz zew+Z;0DpMZk%lCb8l=lkmE+-t$Wp0}7hrHjUiLK(FDi!ZpGvTzHhkXQ0~jo7`DXF z3ql?7vsiKb|P~5=JjYImb+ewy0Y2N~nRB>a4j0-y|8UO`6Gk5t;7H0C0(+9u$rJ_kI zA42qMYVubs*^FA64ApXQ=C!2PL0{;Jb2A3)u&0E18iC-*W;4)Zn0hbLa}>?je>jBp zE7ph%e-fi)qQ8Y5H*`nAIo2@lu>N&jQ6yoc=EWVImq9W7D@0DqJipR5HMoUQpeTH56@5PG=e>ZV`u9PDp>K*Pp|1=-*$&rbPVbSR>- zg+AN>cMPo&oZuPyvV|TMCo3#!BDyZiRISu<|~@Y4-|GJ)Z`XSob=j`#Tv{gz_-H)V&r-*X&I zzL!)KY`+Km26yt)5GQe%5+d8b0jvLSv%jRv+;X0OgUO`kb6xL~j;-fqUH1)I>nC(? zV1y0jboG-#meuSMFe`GxLu|KcGK*wSDV+RkQrk9Mi42C88&VSh*%_s^{ z$bS@vsSHDnZ0)78a<0+0GP>Xk%)oU5f@+X^^-**P=tVsN@>0R2|Ao8P=&7eDz?Nzx z@*$sj?JWp4;et|os53|9U?<@n84BvXRtQ?A?8I(9_{xIrf?3LO(pqAi1CXXWmsXiRmqk@Fh<$A4h(SvI1|sn^B9OdN~-?B zvE?EU4|ioAx4YiMba&~}GUS<#RzBD~-q>N%1LBpx16LF(+XyW-!0%k|fA!%FHdAB+ z6po||Iu9qUcN7X9y_T%(DKI1?ww@d|IYU5|H)7>H-S7XFf2m(x|`f90Txku z4-Je?@Ec8b{&78q3{ExP?4YNF9i?IH+!8myw6{&ucOSSaNi&$&=;@k8gdD`Wo;qN$ z=f!{~7^x~8k9e&mNEj^(qCJkVf~ww*j#cwGsVI9QL@k@c>Q4mui;-Y)fw1lw>9|Z1 zDuz2YD`c@VfYu(8VuoSyo8)rjo>S2fYMskM@`dxJSs@!GcBfT(Pi*>Ua*ktBDeCb& zqH&mFegcNOs8zVD{)?#!Yn(>@1=l7uiL92YmfX2wq%R`r1((>|upHO&5T zbiX^RPuoMFB`d7p$XR621;!_0p{jAHA zEX|RBHs0WcF>Bo@3e6?9!Gt=)#Dic3)ztoQ0Tk%y=q5)S_Jr*;cUx}Ng6^{u^6X$1 zQ@cxgBk(ve?iaLpFj`4aCn1*S5SHekLd?3)raZs8e^T;*quLMg4h|$s+bvYPG1@YN z;)Zg;(H2<^)AMF|hZ3|$)-ayaLoyHVyC?RYml-v8 zkJUKGQt;uLmJ!|JBw9F>602O^YoZ1MCGujnjHoRVVXtTfrq9W-=}i?v#p6nk}VxX7K@ls;K^%@0BGz79=rR{V=4*=4}}x2i7Y~6?F)WG$y;!|(q$o2f%A^y z!uPs-kDB4Ln^Xo=*nB);ox?a4Yb~g-W;vyce=;s~s7qN=sKsaUD3O1XvhL7#+C;iJ zIuKox`~Zd&ty1N2YCq*{YaML~#+}7oowQ*rsIg;m%Frg$z1I1tqf@0Owma9F?IdPW zuh!bzQrPOuJY~&bEH5KS<6CJ-3U__ ztQspAs*a(?Y}dkEOSbk147;Ud2p%U4D48Au1O`H+QLTqhQE%Z!MP)pZhu@M!@ohzx zTw!lHkI5KXk&fyT#Kon^q6EW#)P?9-=4W4LN3Vx^MTtlv7n=@iR7}1tZpaMQW8;wA zAx48@=Gte56OC*;I*#)rXiV=gucMd=C36<^#ukgzP-`ZFhLka!w3((xqyOsVG#+Tl z&HPF}mT=P-jUkMf*Kqu_VR=@gT|{ql zx`0crk_kx|eNg(2iePG%9XB#rx2KC08;9Lo#dRI4i5jKb`R!{DjFZ}{?G3}i1C;t3 zjZfs)N3{kaM_^&~{lV;8y}$ZwL67KHLoQkUXAZSr4r*muW~BkQ_5!!TMo%2_bh&ZG z+$ci_sKpjn4mIix8rq86Mq7i`7Z+0*9M-DL^vqySb*oHH4M-S^YeLr2NY<*Bq7|ee z%s<-6zZaaIp4_V(49>5zxg04MV^#tJ(^=W(24NOBGHwb`sj zRy{abl%3=NvD+v%1Oe=|P}IDzQjl-VQc*d~v=$3&g+UaqDzP|)wm51SK+|&mH zo_QrH@9mCZy@O8inZP%Ro}5VAI7DE+(DIE(?2|0XU$cdp)5FvddtU^d2MbzLFvza! zsz-l#OHkgFLpVR`bnk#%;>F*&*e_9CPUQGnGw$CSRUixLQ+2B-aWmI+-_xZkDyR)= z81@`wMbqRc}q)GH))z1(AH!3Q$7> zq*6!yzBp=-7)h)`&%&@S9|FN3!|k%mK_ZXrs`uSwdlaSGfsQ8?C72nGM1lbm(r2kF zEc>YQ!RvDl>IY}1SHvG~#tTHRdtkX4HwwYw%{-(+DPHXiLQ%6mBjRTYI6rllw0^@t8m}vMqy-7m$1CYyX4}*0~Y#vu>5tHfy{*JLL*?t2YO`9d;p&nNl31uYL+~ymQ=o~D%E*FqVBy> zl->YE>jr(a2<0?>8d|BZ{$?tjsrE+WQnZq+Q08h(u!~0;>#!uwer(SbC(AA-e$$P5 zIpZi2;7s0gEmiWkep)G_?)YbrffUAu3_YjJ z-@{fd?5y=5z^}9^B+pOotOc(X!E&RaG@BH4v}r7v3sz5;HOf+vO%!QN;vOR5?xpVF zj%Hh(CNelM4K8aDaj4c8G6#X0KM6YD6sdW$|1;exPLI+tT3{nU~Jq%Ky_&f0H8AqXT;n0wnTtA zGiLFX>p5oe=E$_czz9A;om!*Wl>&l|_YPfCQR#FJ|IA2@W~dk>72J{5iNqwhYxMyG ztelQL8b%7iIXo2KCP|4hPzA~_XUe`RWg+g6VY#4g=3LUajL~1W(RvDx!rl$x=@CGJlM|IQ~GU6PXHMhtf1nL1PC~ z!~#iMMdwtxH(3r_lpA24CTiZiJXQ88&T|!^S1pyT^APDDHKCV>zJn>%p&g8|K-xQGfmeJ?N2 zoRpxhZ$9|wyxzGUoEK2#WU)!F+i*Igt}$cNy%0*(+@(RHuG4c17#?gIH5H}Mxn?1? zV+k_11igth!DX#vJd$~$BJ5W8x~0VJLR|<3m-*Y|xTSK-*{sA=h~cXl*-zcpQt#{Q zKZ%1f+oJ-UaF4}xxjj#$N=PJNv2(MpjKm7t46ILQ48%N4_hcA?tP9U-!X(~19zq&; z6cw@K^Fx1g6})a*QZg!mK**DbVU@ezOa*;xVb}UNwZuACxr$9q#Y(d2j$>s$&WKu& z@92S8^uUOvR-WW0Fj%6hjB%U*GoX)hLWT*qiz#f1H_NlT1)d$!O*2|O%_%`f^R_$7>g#G%fz^`>&eMMGDozispx}`Ku z&H%ji# zzZ5lst1I_08_Spcn9Rss1~&=jO{4^7gj_bEX1qMo-_IjcLI#d!RQ>lAzH`Uqjigr| zMi5ORJeH-p9TB1~gIaUjS7EkU(p^6^o*^}&$kJ{ao4d;pKephG22b)*^owKsnPg@C zgf0cQsK=p>$oY@O9=wK%D@**b)2LaCfR#e+o%{pu@SwawHm-T^fyCU44ar5}DzZ5I zoiyjI)x~rQNG93Zty1lOU#HoB;;86Q4rE*T=Ck5dyevqRh#;1U(8LvS*&9{ENu-k! zmI}rUc?YM&aNfXoyt~=iW~EZHK+j`XIrAX!2vhVAINQG=4paskoVt-*e^!bP#jWzl zTc7leytq+QK?iFfSb!{2dL>EU*_G;Z-4PxJ<3U2Ck1qZC2Ljgq_gz{U&Qh)wt=;YLH{Uc= zP;kV8r@K6*mjf~b_t=xAOxZ!y4JiN??yvfo_loHy{Gam?2-D^;H$=@YYavt{e{28? zuz^%se;#8Ud`-W&ePRlKP)d6x{NoCkjI@5-Pt$bAUwpXUJq$_5 z8sbMp2h#?&@-4vA$Y<_(UIOFQR{*%{ zs92QaPfMvS#TH7h*^kA)pX}Hgel@zkM}3ctesBK2*n11Fs=6-hTcoACyFt3UySqcW zyG!Zr?v4Y}C7ptFw{$nsg21iCWjBR`Bn$JWDl1Qd$|pIvXiQXN2rN8s+L z4)&#Y(%3rfVS=WIqon+lk2%1AshsSV)l)57fA*hHmIWSGyYhy$`-)S zQMx$Fx)d}|nI%Wn_T{3p_&zwosxo$5zNHDC+YWakMFjVVjBbXbZ;YoTlZdW$irW@l z-Oebf43~sJhx)BUIoBIA@Y=&@i{3|Pe*Udi=nq9elV;w`*epzWONH-!a*oSxJqPfF z`&B&#ez)@!sahg@>sLK^d4Ld(j07|{3W#p4Pq(3kBmO>kcl@0_F+BOEM0t5GDGss^+KN*pLCmGGGu1?#kBqg5F)b)VgWB}8^ z^FxrxJi4ctE}F@%-q>ZNaGciOKNBZRfHrA7Yq~kCX7oQjIa-LjH4nqL2i+{fib zhs06u>%-;Uc~5J>?Zo$EiwWPsWPsSpf`IeDOpH7KvV&M3C0T>Wd&Zysn0+q*>Ll-@&QN2K*=!d_Dq5Os`=IK_=60zUa&m;-s9Z0GjL6 z0%$#etC`$Xh-M|8S2%hot6M~-&)tw626P|7kVUzkejNxpAQ7`40ICl_Vp^=weHp?y zwKh;R1T;JcwQNa#0o*Nz^;{w#!I;DB+W-VjbJc~`IPK@(qB(@DP^!ise8ro(UENIH1Xd+0rVI!hK35NY7 zdDwxaLnlqG_k_ro#7$y8*#o}R{vE(GqjQ6z_RXGFfdG=sFL=;% z8E2oX1lW>`0RmEVZdlT->57#V9!mg!+!HI`o;uvcN7>QJbzeau^x93RGa2gJ-lS#8 z5UaAjSaZLIU8hr-{>^s&7N8IC_Ht21XoyD^4jlObFa|#$j0pFB%YnsrU1L~8H z0AL!wbT=L`e-1};vBJ5KL=5fJ0S(G^uY>&FEp3Y&ar+=v7I9wCUomtCSjikMFO)X>XO%Z|GLiXe*dcZVTFBg<_8=6&(fR z9IImz-N5NY^N@iKCU`HBsl}>i5(tMGj^*!Z%CZYX!*>*t3+$eO@TBB`6@4QJY-|4; zOFa_zMu)A6QE>-EP|y3E-P2@s(s3LEPG8f>#dKKSIj<`4o_+a!nE=JW;1$r_&8DRa z3l|~Z2L|KO2+)!PCR|HvQxKm0mwUiPcG%QTM)y!p57(~3jkqOiFYF9@zAE4SlSlb1H)!fn973zvx@w;==>pl zq$W}eNXsFZCPW!(1k$3(DDtPxWG>}(Cb3Jx2u?#fbZrsYWFDyux52124<94~9+xNm zOB4IV^GOB{*Sh#PbcV4D8LUd+MkYRiJ%3|3rV-`{8HTUl7^ic_eifsB+U%GQULU{{ z4%-8|`V^uS#i~{>fH-Pu6vhC(5y^KGoFm^#3WM|>MpOZLe$0RsN7S_>G?eEt)+xcE zL!At}PB~;{mA7TX|H-*L>CMPsVmZO-Y5f*-eUoU$j~w!JTUpcRF4tXj?uEZg zlw}Z(unw)Ly)ChYkubI7J!_l=RCBz&X)PQa3RxUMpGzgRO+Nk{H6)j2HW?F?t=AFw z@oxyHs{@d|d5YHQzjOTvBEJY`99aOhJkV`zaR~$j27J)43Hr8kq%KlsBG%vHg@fT~ zVSfa6sa{2tE8n1igutQiEc!A0f~$K~_@BR$eF#r16k?zWOxEOQky&n)m~=)U)Z?bi zwx?i@98g8nOvL!A)g6qHQ{Pe$J;mz9^3+?5qb*CJj1axkEzL$$X&PPHG6euhM zBDnZ81MgwvazRsEk3KY~nD3%RR@T-J&lex*UdnQDPren8GQPU``Q_LJ1=fkiqf_$r zqf00cK>++^^IRld^#h#N#uxd$iFlM=GO9@(#CJ)cA%NjhLTkq4>gWQ0en3ym?(lmm@B6joM2O}ox z_20@U36m^<8;qk?WB-#bs=+9kh7;eZ3=b7hFOwc5Tk6iAauw+*#fHHUCw6i$iBz z!zZMZsE3NvY4(;!IBmfOzJdSQzQMozo~zR*1ViI$61X9!*r3EV@km-V5_Sfcvi}vV z10?eAX#77jk8-Bi4~cW4jtM(EOT;7LayYlHXi$wvIQ#>YLr{Kz`-1j|5RAyc9X?`7 zMihHfRusv@_vrZbQp1@airBrxWA@QgRVcULw|r;WjZ1_ut-*N?QBifRTv%{Fu=5d* zEr?SJgb};jz@Q29AINIwTKihMK??fjcID&V0UPQ9)!Ynnl|zM$+!mEZeSmL1t^?B=o zfJp+o1P!aognU2?kMn-A8l(JxwtVyPTQRBsdC>ixuOSCmr8<7)`tu|uk}vX!SR~5n zDgiC+kl3{70x(mlkI5F7?}%Rz>(KK3@l~1|U;=pM|6_D8&beV5wE!rWO<4)p#f%fY zy`e$CViTKSvy;HFTNO1XoL{gXFNGgOANI{_%iWE8?g#)}Zlo}2-l}eNjZXw5ics)J z%#b&C|1yhCYPJ&HB~*dii(-CN`0BqEk_IRqVKQXMF{oUkmZ|Zj$aTWdQZ_n(nrt98 z+nk=~-oqKtSzkr!-egn&Pg zQr5o9zjNP7fi#E9-G^YzkmM;DA$c!2>UEsVlJltDx#s@VsG2Q@i4;qi#$*=w%?s$# zE(21`X9z-sDCW1xEUHO0e1ESlF@$*vP!`gr!2CBA;Gr_${UqInV7L8ai0JpSDZQ9~ zB~?;Tfovls*oYy6HXk(OblD3%%L25oZQDD``F05ph?GTvT%4p02}R66QCgtzI2k@O z@dr75c;Xd69#|`|IYHb4kCU=g);6-b1w==H2y>jV$PXY$eGdzuchwW>vIP_d3c4Ld z@(aLd;?)MoEQV@oVb>$XcWVJLokTWzA@BK@nC9)Y#d3vYvNU+I`)b?I99vJ`?V`VBb7aCzv}{M`oPFp6UWn;O&Gc!5`l{m(q-~V2R6Ns z+`qb97CkSF$bV|{j~VlvUEB)i>|os8s`7_zQJCqw!01 zp{7L21Xg8|p2zc%GbiX%v+z(1ag|FMl4GzSB!s+k0f^G-mUSoqX(F1gFx^1*`$kQX zAmizs5HgTrFy}&TH-aL2zW1H9#(p@~_Xj9b;1<@K#NxD|HgyP9SUF`(^e3IV<^g*|ln$g687NSVH%SJe-k={NaR(`gRS+9U zrE&wwgcR-R^X7^$Gy$9MgUD61B|b2|5j#cEbYT60Yx`h_BR<;AdyWF2INg+bqKOpi z4LWLce`|zZ`*~Z|7-fW%_jIscUp+M=E~cG$Vyw&~x|!snv4%N_Od;5gSo}P}eV5BI zCo+awwJOt4eb>|ERy3? z#;>fH3`;FW8i|C|Mz%b}oD&nz!m<7p%(RIsC@b+b*{{>9ROjlkcMjvOx%wN4^4 zZrm(7f+ZLmHBN>niUcO2g+*@ld@u%2@%__P+C33dem(DGGy4#ell?cI87mwX;V{&J zh+3ef)z8LOw^h_LqdU0$VN-eW@v6LF+zA;kIbIDH{Q#Fm)trn&Af*X%DEt8n$!DTq zN4pb*((0@{Rqhb_ncn{Gf`(_C2^mugSGi;aU0UdC_EGY`<*0n9x67oO;Kk{GO^9GR zQr>(B5t(iXi&7K*LV^KGz$XtW+eHIuiefs^Ou0%?4WPV(nU7!*;iOrK#FgfBMffQ; zv1H&pOaW3DK=DTKvFlW!Y(m!iCgfgerJ$ZLUYxiuf=QCF#7Oxjf*UpYn<%cgtdKJ0 ziDP+$Y$L;+xKttYk+?21j5Jebcq*8FsHg)wr*#0VmLWR6&vJ6W^nlb?Q9V|nD}~$m|kZJ(No^A}eVmc#KrpE_6(0)m$F1H4sVv(xtyu*f{@d8xY(h2pi98 ziCm0q9TF8OoJAZ3oEfGoW+tR^L{C93S6?E;4#rlHlE_4S2)Re7ZE+~Ya=2chZB#~{ zCmG%kkRZa@(srtrXc357{r0vPO?UW9Nn-LY%?g@dFlWsJ3ZV&q{f`Q^w5ooRTd|(= zK5AN(gz5q5)IuC(N4Dwjx;1Lv6!3LF-kS?}F=5tlDRXGC0e(1sD76sM3)67)a{V(o z>sCSrR(%C<(1c6Z!xFlHi&HDao2V`D%CV15>ZQHd2=Scq<<28|bz%~9PBTw3hJBRT zu%3VyhlH5^OwBphTrg2%j52m+(%?u(vVk+w-*OvF0PgRKgoF|^+`n)A`;Y+`5_x*O zMYu4?S@}QS{A*P7`_1y0-{X$!EA8KL_g`7lKfmW=0jZ%kSkm$T*C(t1HNc7()zAM1 z>_5l;=Y{w09vl`E=${(l|Niv<`vv~*75Kkb;QuFAU}{d*ui~GqXkX*4{;iA6ZiEjZ z$)+0kQxcu7 zcCA-eS9@W~K>e+Kx$ypOK|aA#zWn(E;))IzbKX)eY?~x%-K|lf4kf z?d_Sk71oxwCucuh{W@J7=V0moPOY1Mz+CA~B;-Z#Cku3837D9eC}dz~XTLp~Eo*IE zrp@V5+em+n%u713%Z-icQCpnXZk)cmyK8Q4j*Ugk`@x4m5%hQdZ_N{WSv>eI7>4s- z&aa10uBZOHNN2e$1ZBubZc+7l^WB|Sx=g5mBpNZ(t(|>g@T1#%fk}kiekyg5Q zAl?|O^^nAbKNXms?~1>BjQ_{iLL>tOiLUU;x;i@F zIwzX)28?tRB|UoU@5$yC6)8IYQ)cW0=(jnzbhUpzUnrm4FF8istd)VT*wDpA)+SMS zKlEo7mC5It6MR0;T9-81Rqo2LSv2W{OJ+%Qo?R3~kRPuX)LB{4U0J0&V4i26o811`@3Gi<2HmmlynTh&nIiDYcpoZL zAycI}d1!5i+};`~B`E;Y?HuG@2el_x`?{3~$hoZB@wdB3OIljHMp z8_S!bu-P(2QdV_;*@>)wYi2GazvFqUm}2+OXh}bVSl61Llq2-=2&j+@&1848YKF!O z`{xD#FI;ecAF~oop8MyR2PEXfkWt7n%-F}y&P4FxKmqM8+_TGov@pvvlL!L^j3=FqlS z?XeHX!+x2ngV|jFn_ZaG0biv7!y@~SytYSA>QftAZB=DwU3F(cbyHV$XLTWw8~X&l zw5WxC=f}2_Ox}~scG~+kfrj@?;~{FO&i{57f_vTr(ZvQ=YmED6P`;)>@SK2TF#o?aFzA1w)b~h-jnsaWy~6r{Pz?M$c+^BbWKp= zX#1&{hIOu73{v!+*AVPeR5RKv&a+NFJEYG)J6)5`Msxq29f%?JnSoquzYccN@z2L> z5`?Z??qI8`}zZf-fxPYdjvjV&!4oSbmwm6erUU0v<%?VX+at%`X9o#M+a z4u}+gf=>nZxTR>z>m+(5|9OI zP{}OTNfnGW%-gDZ67p9_oXH`>* z5xFh zyiuc=6(eNg^Ra3tu&-yui-s4A-QAy3V#2cz?8U5&-h)pwl1P_Sr}!LaRUWIjKseG>7g(Hf$6l>-rL&3orO4kot7>uo=x`YY!L6)EMn!FWO0uJfFWM4T< zk5RE$Ddnv#?HM3kO+URRZgAoPEs0;MW>Rh*CuX!Lwbc`TbziYKsSjN(>d|#d6&yADbJ*Yg0@#Ll>_2mg(E`kvK>(<( z(n?75HMaVEp}Lx5Tg*?B!0*Gu)L;mfJub8HH{&QoY>o~yZ=0bni@ZARP&YVpb8?Z# z$#){r`QPH{#SF<4bH&nIqRZAVX6NNIINVllj#$#O3D@V!S_?pRRh@o4`2j)uc{N3& z(_08J5Ld{t~rtAUU`p@Kc^r>-uoT`0O)NS#w9k6Lo0_uB-Ep6 zX;u;9Sy?gk`ngxnS#^;woM=zJe|WB+GdU;&mh6O+%#txkqcq8XJ@!es;Xn$k$kM^e zs&Mb-j7}L2nl>)hd=VK_$F|O?rsFMobbUs-IHlRkhkh98@|i?xNvLwkU8_KJ29F@| zrM>yaMMIA~{T~C0e<#PD4pE?za6O@CQT*SP?v)?H+yQF#W`7^_{MfkY22a>Dph5IH zz@h8F>%2wr*usf;Xq$Pt3ulK`hG0$jWmAL<)4&?^Ra4J?ut#80;HQ$`L3APAG|M5% z$li}Nl*L8)Easdg;T^})+vMRSD`oM;c$OWfOW$i0oo__JNGxH`P6)RMcaDHhi!rS< z(;V^(PPZoG#rpl{dCXU+5~Z0a0Vd~SDH;ET&p*96ZThuce|(LpBmE(lcm9h;gQeln z{=!^f)GdipKCuY`3v6`lr3-B0tlHj}4i8?}e2fXjjB`5>j=?e=ijFVP`m@+uf`&A0UiSg-%{i#tciVk5K-a^HrK!}qLdLEmbgeT7P) z`+0Ja|7El2j6i?l_^A4%9FHsC{%|v)vYOAky=qa*`$+&?$MDNf9vz9j+TxdISUEO0 zl@JjJ5qL21=810!WSY8m?8EGLjsYxA+hX^44joQ_wc6Y-08A)#cHA~9t&W|r14__< z8^JH2$s=Z5HIFD7}NvP75rEi)$sgd_JK20Kzb3&th-?3!u_jsH2(oUCM z#cz5TKU^c94mh7Wv|lSo^2^63vO4bqqz;GI<4tol1C_%)Jg7{Gz;<%>32S}6tE)}k zt}l6*#T#l&aAU4?uoUds(8~>wcXs;%4ao=T*8BS>$|#SszNd)Uva0%pOeBH3!(bmO zRd*@b}>A)?PgbaEqzDtiE8_-D>-jY6gybq&SI4vE-dr7tfI~Yu6{muLx{vFQd!byK5+4Z zM6C9mluNp-7nByCFEC!b1Eop+rcg$)6ql2)w@gE!={^5sj4zOHF|0t`VGAI zQp_|B>;PZ2E}*zgrI;)8)A{fr*=jcH=??GyI9MD@e~wOD!2@6%lNSw%y1tQ;mnDLc zQyl+HGt}{oKvI&M?g`*Tn#OQb*l>n;Wu8-e8zFs2vU(Rb9me|$h zEwJnWOqm|g!9D-->@LmgZ+HeS%h1|aKl;2w^oq^v{#i7gvYW;JvM@>kgnz34I@d6H zdnA}|H-c_KRn_qF_#5H@aH<~wdGhU^W=KSOBJl^CG9t1HLtN%s%)tTh0#%2^|D(_aIIa&~PV;0!9z+`P>C zrROcws>9|dd5-1XTWPeRXdF@0!V!?~aorFUJOS;NB|xXOpDpN~=;&%{TM#G%%LA)N z49@K0kt{v^9ooAtLgIEfUS2$Ccxx-G4cp2pH9i1_LLJi4p_H>Zu|4{%>pEQWfFtPt z3sA@EXl|%!Zt#4#YAg+ieLP=S>;n|b04{iIW&LHRRFIL&w?Na-zs})xWJ`N5>WW!? zgcY(3M((w5!!q+vLf!KbGT>KDz-y${81bLD+dPAa>|<5cKlYNwBnf{Q=3r6PS8ZVR zSZU{ZHu?@sIp2A_zRQ1XYc@0#MC07py=#-s87CMIx)ZpZnmm1mOzj7f1^Z*0*&%*j zlV2_$+wMl%21Aje_3Hhn#oKw>S9#mdGq(U%T3z z!%<^`dc-oFR@hRCD=SSl!?yquXKn((RQeHDfBTkM01N@ZKuIfV0|5mXuD+&%RP8?V zzW%yi)G~>w0?a&rZ|SjFAUY&?Jr++Q`o$rO$89HuQve_a0m`jr&12&kKt)$9i(*zN z70-bKIX)729~K~zw`~T(HNVW24aN~X^#U;mum@TI2iXk3HGr!5Dqa*3R7og#6T;{} zEy1^z12CWfCt5Z~&t;t)<4izLFAC^~C;9K^27ZcdJZ&K%2|O6FEC9VsAnZ=ajk|7B=P+pw#*LiGHE{EHRP>14H0~FJ1PNiPm_y!`iTNb03L*H zl7&JObZ^9I()fi0_~_CkQ36d=Z=m0dqdct9dp*G$_ik_!=vXT)E^zpbBm=(LR)7&Y zK9}XxB499G12{g1dM$kj^ZU^KtYa~T&9dRU@pHoCSE}B9qm?&#tDAeHzV9*x(bI$E zf`P^O^+)IRE4ufq2d{M=gqO~0W_WpP%Nfh-t7n{@&`ilnB5*mMgpkE#;wh2Egpt8g z3iI>Pk2*czVPr^=b%@DHz-D~T$In(zPRAzB$IkD^?8B#t4GRmUJ@0in8yZ*-Iu5$C zE})SnpYYp5u;hBoMx z*1q+uSxhsH6M5+-Wrj%X6AUsKsSrf?8Gq!9toDF}y4L2OlN_eRVp$)c}p=_-V*pS5Mysh%Hm4M+U(3>~lx+jyOpYrbLB`&6T_+;OD z&J#7cI5YFsZ)AQdUV*ZpV2O|Lke#s1=GaM94iOrk6PTv1o-VqeWvB5zzKosB9Z#A|n6G zg0$R^U{tqxLM60OkaM6Y0e4<8{9Ux$`+ER#TRdObo#$~TAE^KB<{_iR zk`Wwn1S-n|cnmBfn!w2G-u8=UfK(4C4Av`-7eu}|zH9qlp%PeW^YT58`L@qv{kEyG z59GRBl2*?bhsgm_x`Yn8loob7)@m5sn7Vw}iEC+Y0Um>{7Y#plHh^Ha7g53p5FxAL z<^0+lr|r5szwWyl&jTzjG$u6``T9VC^Mbh|MdIgK$k#c2kCzYAh2mz~2JPPCUI*De zy~`FmEMem>2a5)V^^PPK4Z^LfCV3ISu1hBuXQIn2CZBlRk8Wh!d^j|kM*5KiJ+B(A zKKlL?57=Od{snhGGx2&KqVb>L`A@+7dV-KLdc4P(f%zL&?(g49+%RleO`)Y)Pm8#P zNT>Pcr(kn=rf#wuYgtNTbU5z}Au^JC^Jp+k&l#sQMeV&aX-xIxRozfzd$L^#z#&ObGRMP zAqhU;0PX;fK#bd>17bzc9FmBhE>*L>M1ZJ9{M-#(5 z?GDZTIexOJ^%Q_Z#psp+w;pzj349AK_<*tF`ypfI^f*igz3_c5xtPVh(KI4}2sz{p zVhEK%K8;D@k}&16Xb~k88AtXb^t(|u4OeHe_n-XA9r^grGHn%C6QqJcAR z-{NO;|3mQ0r7v^Z5Rk0S9d@sbMXMQk-pAhRmRbLvPI^TL6#sew{GM8n0|-Ljz{>HC zX$pt@oqrxCQKrDtIdRfln=Bf_V2{BEV2T1lzJud{04{&Mr8lNUKfGo8~>PkB1wPbMMD>bXKIxYkd)8anJ!(0!l&eBe?I4UVV zR5L=dq^`%r`8x5B;reNcd%B3#y@^5=Nv;Lg=gFvVTML!J7}rCrVa!v=@FG08WvJ4* zz!pQ(^BSB3k-2XWRE(biOw2g@qVLs@q4$p?W9IPTv4Z7Nm&hO7-&b|T;P%BC^#D5n zgl%aCk~bdY`iRtkM%5{^k9f`BIL>#0&g{yy=#R2;**kah3t@x;O< zI}Ryv>4ySx8r+EJ;JzkPSi~vEFsiOJaX2hOr-!Zhp7SqfcXtUzMsc(@w5e|}jzUjr z;=Tgp+Z8p`cb~r8$5@}hpCwoZeD_ZWMkGs4ikCkfaWKPWi2a2p10SK@#`k-Bd%q_{ zN~S56DWE-F4!h&OA$++?>&|8|8v%4KGN=_3#`x4hZTh3|tf}|F3^CBTwzfu@RQLH? z58?(I1f5p(dF#o99POE%eQ5c8-c!Pi)Ze)wyoW*j6HB@~s3r+h^!EY;%ppQlmmvw3 zaAsC$*hxK_u8XL)VzrFDJ`m!CfIXwFdl_~FMltEzcKJY@REpBi`Bdk{XtCS}pJz+- zC=Zds6V*Pe#b81Yj+_Wji*q?)gEwQcma9vMx8wv=MaVz8yLm$-8$jjGM;4H#nMSGi z&`+^%nmBWvAs`?y;c%}_6zY9~h0$HuyVlk_ar3DMxzc2XIt?B*4so~xW;X$8dOwrc zM^gcl;hOs!SID;l;6v7tr;bZ(2s~_`Nr zw_l8>!Dx~aXQbtz&%$?>0IQLXrY5i6Krdidpoeb>{S6svb<}&R5k{*0BPpZMNcKZ! zy~NvDD~@mz$-8eSdX~&MuDenxqo=0p4A5OfL*WBW;8IEBe`f-va@?gP=+P3;P*9ID zt#TMXttrV$Fg|D8g0HsIBYU#=3cA$zMzq$W`OQE8y7dL~$8ztNW99k`k~$i5oH;M; zk3ZM?m<%12VWUW_5S9umd+`S?c)i#DcV8LF0=$ZhwOq z4!_E=yM0fi;p^iJ>!gCf?iBaM$4_VL`{P-Q-}=wzG^LJ_fZYN-h1 z6uqN8v~NTDzwmP)Z+X5*3U=I-8b0H`!#x$be1~vz>(kNIm1_{NQ;@TTF)EDnxi|dB zheHv$L{hAFPWKoh-gDq_aZjuHGj*;KjGc#nUQJCUo{PRm|1H37{{(6z!MIbm&DPVT zU~BG|1SPY^Z2R_>Fi?T_>wc4Pll*aJq&OpIt{KtTWxLm&HdS`&*+;tFB!n?JKRRbE zI(|ud7OneS@UwlVS>jbd_BGpY z0f*Xue~gpE-F1nexyoPt_Ek;#wbz-B$u;m#Cfmaq_lX1<#_w7~o{sWAiwziQtqa=J zvoJdi^tOUDy=c8~ng&M_XRU-KDJ9}LhTZkMTboZN+d@UgPzo&(u>PrLM*^{My*MD;yxv&p?M zb>LQF8c_%mw9@GE-iFwQ$>c@DrgbnOrhSV`FzsmGQEosGKaa7-8xeEQb7&(yZZ1c^)f|OP=A}0pu)A6jCCVLc|FenPJ!>|FDnaP z{#-qdD0|*GSqDeD{p+O?#GoF1NLYA3moF4~qss({W8r;uD1>4Z=-}cAWIL*CBU^7; z=5$w;tSJ*}+t-$8kSMk=wHlas$vMI$KjN`s!ZkN-;j*>+9TbBsF@g)jY*>yOG1j~) zMs&v>HLX~LPb;;VepH8%*Crjp9}Ja_y<0keM*()p%fS z`>cQWgr&kZUTb;&P@WF|h?HU?pHdOo_X*K!m&xmBryupcC85PnsV(uJ$D9WuG-Bx4 z*9W-#qliBbI=>EJ4_}+RTiuQ>Ud#wykL^8p;k*)pCWp@B6Jb46uiIi7PKaL**Y3RU z8@RoB&@zWc>T@gx5{)^Nr(;uP2eP2zj*g<45RuTZN3cC7dLa1D)!0U)>RWmlqqGqt zq$n_qL#xektO-l$%|jQ5x1EIz0(v0oz@?RaTLr=GR8rbcgEh7*gw z)sODbWXL&F262&#aS6u+UcIeC;H~eKhOkK-ItwsIKRW48;o}IU!>pspT{Hgr>GLpo zKH8+QI1Wr?{kwe;2Bd&c3%6NJXi@B7)RJIuxyqCg&+Q)LrmZ(Vb#~rS&kxw8dTLDX z%^P*-%Y_}ekhm}NcVc zuf@cjPjKV@Uc5_1fFvFm?TMIu)dX^F@t|%oS(Nci<2={(V0M1T19S3X~<3ltM*mpE0;Df*RG5ZvF#t zF)WsmCI`eP6Nv9Hx(Mgu_3Rw3QDup*r>lMDF7YOt@Y@8jO*V_bCUv00IhQs)@M)Nl z`JI7j?4`vpigsTCcr98?Z8BZzl&ysTFqsw_4D#SWlV6X~p5i||qZZq84)WxmOAhKV zqqT{)fPK0Q>rG*6iS_XBs_4aqI~$ZNK!TSP$DvItJ5hqut7Tac#|-57Z%AtM%?Zrq zWBQoSdnCQ}+H?ZNZDx^QTLn2Z!-L*0rU9rr4ahRl>#7zuSG*C ze{S!5VZ4P3Qm$0X_ZG5Iilhiq-XNcNb5^uoAB=Cc))@G@>I(f`NFxiwKhGKT*@i#- zC1KB3W13GMZ3y9D#Mr{2n=(x+fISPoLYb-k7op4TEVn~tU_0ppc6#K%NTX~Sk{#^x zUP>Bo8si?MM$)_R(q^5p@SmY`N?F5NSxpP)5TdbNDcN9yMCkB#;jBNDVsG55oR@=t zQTiH)7v-Xa;%d3w#J0WjRZ!Q9(}WRfGQ*9xBd2@!IDLspBrFu5rP@mjpLBY*Gj6`Q z^awi9N})}Rs4c+z%nxa+W6@yZ!NJX#I%0<5jQ7|L4|Qq>9fumNkBupB@r95!EkgV; z*??TEx?V>jOot(-dHIamK(klvdyDkAu=MNYoBOJ$UYxT20LVhx2K(AX=*P!_j>A4_ zgwHSTvS+%hH}0d?Swmuhs9L>Iru&e5ACzYDC=nsa3h_qJ-hFC+x7a@m@fC?~#rLAy zrS>aJ3x27=4knTRZ9y(v&8t!`*s-=BtH4yxeVl_Y@6h9M#-Zwe>pDPMz1c1K326Qj z+GIxjT_it7_GrS%UAB}z%6a=FGceDYQR>?m=d;3O=M%%UrCJPR<}3E*_fsZc`R{}_ zLZ5*;zUO_@FcS}n1CidOF11byTDiOn2^`6{7(EwwgvMfj#Q$;L^wH~fuOi)PnSMTz z{L)q*qk{B6H%~RK@g|)R+sPWS)EFfTB&>#_1kMJjPA4iYbcHy1!Y$cETM#Vdvurv@ zlZMg07K)arMs3b|xI~L-#y4#UwW9zk#&-Kr9C>z`tq`_u7N*>N3ZfRzJz#A9cckSm>>X3Wv@`2J$CEiUQ zrHH1Hg=`0xmLPtPQ#F}y+WRE9G&78Tg4z3ln8-0{En=LX3CxURur$s4@&nbbQr6LmSwthU}Z^9tOMujt0Fp;WCr(qY+%Hm8cZd&8{ug5XC zJiS$^f9+g+#Ww67G;XP#uP^wlh0;S#mWL-Sq6;JUh;?Y=u>&%g zGm~FXm#q%%e^-T57Kj%j=%k26>vanKqyQvwkJ%owqI=z53J5T0%!0N|6BgXd;FQ7R z`pOX3!kU8|ue|%+%5}%ld}SX`9Z7WV`{|^+?I@!>Z=z}Sx_K$(ffqq<6gjxzUs`S4n9`Ju$OFm3SBg2p2PI79J*j_@Z4 z1E}a_nw1`IN2o|_1bq34=rc!KCPVHuh7RI-%H!$a8Bi-S`l*4O?-qTJ(%?v-(uBFK ztsICBk9sLRv^*(s#p#HMsV_;nb28*8m!{&w#e4&VcoG_Q?UvLK12(|&h$zfhaUOz@ zH9}gM;41oHE=A?jnMe|3K3p>9XnOqgT?R*vKc+YNxinC9M{mTR7R>#*oy7Q*gfC|(5i)@S_TTwsX&E-Yr?KH+T{FFDcYL)hTjaV#nNSp-klUI| zRx_9wuSyH19c5s-H>LcH(LqZrp5G!=6({z^O^G9n!B`T{odU<+`e3}5WkSo~iq@k| z=?pNc1s$a=Zq)jYXR1+xmxy(bSvD@&z8@m{sXKqKExM^HzZy7Ih*f(OVJ?)0M@X>> z$P|CQgb`t=tJBiD*Tl8xh-){u>$QrIe4o4N=|}3m?#*^(fCf+LKN&AkId^=BZkQzG zvEChAUk@zIN9{HK8tVfU+d_KKuEOuK(wI#>h#KL&tlL%iiGIW0D_VCd$|69{yob;q zK3W31^|Li{KqF(SlCa7X*3hjFCnGfUW`yDvL=Y$J)QHYk;|3%==I!|5Fap*+E^Qj& z9dQ2l#T?@FU~n5>)=8k}Emw1mv_vC`5WMNxldYMoBs%Q9w>#{qDvuQ0rkdWh{B$6N zm0OLDWFY^$ZlIw6>ju?heYx7-^*jsam*878IKRS3g4UY@N~gkV@O*W|w~$eqK`3}p zP8AvYtz9@lvN|J_;eN-zJPopji{J4*f)6FcUV`Fs@+DWbUS%&FK56PF|z!z3(cA%qqXOYKv zessi2%(eE7R1ehJM4vRCLXpuxYFE(*hiS>@%rW^6vt=kPLad4*E)k8t_flQ3ebC&) z-R*vtMSI;gq$}(@d8=Be0hB6+QgZxMu#&T{^b+pG09L(M<#=y{UTd&`7-AqUODF#e zakPd;rC&|wUF)q!_oVst$%4a@LpL~J(BWb+>5ql{1|R#(-9o^yHnXZ;Z$QGODKAKQ zWS{@#6FG)W7jyH12Mjnm5@>qQx#&#kJ^_rv!aScllEtKfv8eEbK7-5YYb3^(ePBOjk+8Me5B(Cc1i;q766JVIq|%_>DW@Gu0#D-lmaNRo z6`j>bGH{u?!|Ny32r*jBHJc5;v??t`hsE7~#1p(j!K-qz9Ky-L+mp)r&$#eFXY%z| z5K=799y&bUyO1ffg(mZuwKE9^{kBTD?_Q6UfT=-^VIOp@Xi z;qO?^K0dqUWRv50*i2nmsLr)dPbAmA;^biCU}tY>ghBLS$#I4Tzci8OITR|+q}x;R z@VF13F5)Ld8WiSJRh4C3c`jUA6P1YWsVFdy!Qox7ZZN<0VrC@e85x7)a`yUeUPiXO z;ZV1#JZ2Kd@BX{(4Qv;0^Z-B;o7DiRFZguXPv0MA5-&yWe8Vi+kEJ05->z~K76@{+ zJ`HwgYHUFuamD_C~Y+($Q6VqF|?@KCR-N3ArgPZo*w~x|Lc(d#8~P6sz0|rM&A9T!aMS_CFlZ5)CNP&NE zLDxP)+(b2&|E`=YPE2j4t;uE@GF(6?>i^;DEraUVx~}cu?(XgccXtc!?g4^pa0u=e z+%32V2<|Q$4esu40RnuT_c=M|uCJ1Zd)JFlM%OzN`G*bnja-X~YtIgv97pi``jh3wIh+Weho2Xrf%#T^ zFI8)w-vF!%pY_kul#~<-z`zcuA#6H~Ku>KP$ZcL~u|WEqDqfaC4t~Jq5zvZJ1al114R&KV-?28<0PQCeR z9W|5ksR_jlzB^FU#_Rc~Co{lT>|k#%^xC|`gVAKYCzm!JNmWFK8GGxRidwGmK_8#b z^$_3&_y8n<R3zsHYBNcgi8f|ftiSnD@DC;(WK=Qdob z(G>=w+wks)GzpND?C&*4bVTeLKPD>zZn)k{BNechb=94@4e1J)A9$aAmZ zljXtpr9Kq?)lW_De_K_r9K4r9hosJ2QVn(0K8aeL@5-iRv16cXjs089PDmJg{~qU` zZ7tROyF*AA!;pUgEgN$N?*573R>%>+6Qcg61|)-@T*5nHBQOfU&N1ye*Y0|B7@qFz z!-cHcKlFwm`~jVi2NBxm0UReJ_pL$^(n|mpWV=`kmXiAiZ3YB>I&pCu2>3q5`8z*m zBvfA74~uhVBM{YcB-%!3vsnk0&ghFLIIq2&+gVO>viHF?W<=n}r- z4Nu1Rr+f*_!%FUp~e z4ZXT+cA>1(VhoW*6QUDf2l}30hP%73Kv?umC$fY)pX4e_EN_Xue?Ns101a_;Tg`Ce>t_K~p)gzO2K{r9E-w0z|9Y23sH6c$ z{-W9Yf2)H_B#L)ra8-Ii&`5zI5IPd$=C({){{F(ZMB&tB>hOcP%4TLpMGzR-w+I=}|CgSBuDlGY?E$bz4lKC}{}uVic7)EOAR!7__Q-p*=fA&u)Zt-c6z@ zlDDp2+>F7qWnfa`la0V3nNEcYrL%{j1sAy-h>iC`EMr>?eM1})RqbK5Xn+jj89a?H z7ZX?p_%&M{f!0CniwQ{N$SD|61LSLK=gP37IOoS_XCb3E2RilZFIE$k?(5jk&!k1^ zRVh)->9K9KU3x2d_bNL>RT zRpCRbtC=d1_vs+5s*LKu#WX01UA$Ih$cKo1A|yu=>Md-^_9Qj0jCD_usW<^VU^q9r2(%AaG)G!}YYEYSVps_{TXt@T1 zkC2k(gVeX!%=bQI%+&ncrw{cNQym&*rNlM-(<=jfs6^-V6_J`>PY6J~;GL> z9^{>BEx5UE~7T>ATm^w>SoQKE1i&Q{ME9G_;W z7W9Mq?XAdbFGWD2Y6OVVmylavLFl$}jx?fbVuH9Z9iM2sPwrWUOEu;E0FBOtLM(PU z6!E>e%OVzvF2<<6&8O_gRBk?QS6IS0v!nzC%S$>rF%u0e>$J$}Siz?>1m>Le*}0cq zmr1v31K|ccRIeM=X%90}_2FIH_Qz|IS6IEZr8Pqz{M%oIND7OGc6n{?*JNQ9s&vh{ ze?o>2m8IHq<_T2UtzhATaY> z{LVKUiWpw`vCPzYS)gUv9+-WG--qOZ@Ch+7B#OWdsW zV%_^ih{W{Ur8R3NcY)~S$)BN5vep1xb;`5~0b+;l10eItP_NE)$?sWS-MpnTX_ZL} z>Vzp(h0D-AhJ?QAuSEC`?Uy@je9+NGzVrk7KgVTK@D&LWmK*BC5#t8K*TgaVt{pDd zqALBC>$4>XEUYDqrirY(L4c{~_Jp?~p1Ra4nY(E!%fxaiSO*<82DYR0CX!p848jIm zvu-XLRrd%G-*;ufTzpp%JK(eN@<8lMyo}Mt zmFE&d$n2!00vXt)-<+je=b$r%gl8n94*HXKHl*q?m73?;C~Wwm1YIwtJ((18(`&E~#7l%Ne1 zjZ})M5XB{`I-X~D!fTIV{G=PM4yUb0#YXOoMqK8ni@*>+nC@5F2L`sMokJ)XhT&E=h=qKK*@StU(pC<@WX6O0NS&8DO-qZ zWRuRSAr57`E0bLl{|C8`nSL;?X#YH$fGlx>&H@RZZ{Y6M!1lK|YNH0D)L?JS-i?;R zxc1t#4j4=T?IKm885l@SG>CuP{z?CW-xZENCNK0PBl8|%!)I*zr|TN0u(bVjhx-PE zLdiGUx#jUYd{_I!{qp0Hp)DpVFXqt7Xg(@afgp-y>s1z)#3m3ApI$SXl|$fk_z=C(gZoO$u`9i{6u3yh%O z&hO9+ANTVtolhQuLTf|ABRn7JKkW_?;N=02W`rN~zsDUR@Ll8OYksvD`Ihn)!JPfU z;zbciQ;fKm%}6;GTnUM3>aG~3C|h({$L0=VJQ?1nGtOsK-{bdjC%N;JJ82=e-)^)43|lG6GZ>tWIW0aIsp^BYO&~ zx$Oi|&Lw46UQ6-SW2TAZwIltsudqYz#wsdDm({+SBclDBE8j0XZ%(+{AT7XvTJ5X=D=?RQBVwwVYo7&8hd6e?T@5haHoq@>z z)3qfHcoOFDfxuDx@V6B%0{Qv@(%LJ8Nz2DTsULl0U_NEOoYr)MBWx}uLJGprh;%h? zjJ46y_xQ=l;QM21WSMBli{(mgGlU<*Pcu`UQ`sPyU3Asc7nlS3|R>R2&qAlIq&uK@+Mt?ku(n z`sHwZnMHYaY0*_IiyBqhjK^#kLptxVhb4KWa1%ReHgV?jpW?NnsuO*$7WHKXBaWhG z{T~jxipxx3&Bf<9sVfnNc!J*M))c((T4Rg5W`h1bG?0Rmih%#m&=hJwcIKCavNX_q zgF5Oh(c(C2vMm?EWCr;(SQx0K2v*erJE+jon=|$8;pqv=zfB&;+?8%nxe9?Tvmn~h zV%FOryG&L}!)!p=1gbNQDGic%rME0YGoF%=Hqw+W{rHt)EkNTr<=9l`1tF+a-T$yp zlU}(nPiap0DNM^Z{>&rK1GhlA!zr=1;m9z_xYWC+6W8{6@MW9nDe>PIy$A(oJLzw< z0+p)t;je-Qd@czfAw}aUBMv<2a>Fpey<;gOm6eQVdSr*mc|?c~?|VO@a{CD7CXH;e zmCQ!G{Je~8YF0_vbL3(CSl^6}(mr6o!@SR?8TBrP3YfGLAmko!hVxR;QDbG>j;JNW zAG-bs#XmvT5w3AZYcJlSIxxHMIwC2_l&lcAWV~%&w0r-!r*EjjnS$e()0Aewq{W)q zxN=sCJ)4iaSU8myx?HJQp-yWYFy&4Px0Uzf5kHAaT%#h7r#^yRRAENlOcF!8@T)9; z6<3xtE-#mrOUr78CKduz1qfOP24Y^4JSj%@ZabkeIGie~B@nu`fRP+I>wu-XTdy|W zS(Y7N{`)uW@gGm=k%NxZqVL7kOpC;(`;VatZPhEn3x;-j($b)4RIJw$p*H2HgPKIb zZeYKN{ot|ufrB8~W34t9xwg-^d!zkntyz8H$og^P7mchh)?3U5D(HX8+eOGj64=kD z7h&ZTfBVlj7+`l5CCF0a6{^(gfIbs&;IlW-)_e93CvdO%unz___vtYTbqVK!MhD0pxQ4c! z68om77Q^F|w+!6Z*nNJ=3^Gz~*NLfEXcsaN2KrGHf9*6To!+o?T4X=^89}4e@euIr z0)-|?>PX;)!K~f>YRr`$|Bajm7CTcRiBgFr`WmyWRIBXMl{vVL^b`6nOC9NVfiOB- zMVLXQgYOoZSPpw%zs5=td;YGIg(2C!_xzw6vL~PJfrxMg15s<`1fDBbB2m_z300-r z6CI#j8!x*v8_3AJn&U$qGnNJBZ*EpHwljEF%90vRLDo+D0XuUp_T;1N=tn2R+O!h+ zbY}pteWPgu9OyQo<|6iZhE@6B;u8L6wBE-6Qacm$>K3E0^IySggc6{;lDeL4&Vt)0 z2Q`l_gC$oNoI-5{20h>WQQ&4lt@l0}elCo2PxVuw>;jO^6>L_rQdpj`WSc;#GrL*L zYR1X4$zxch>6s~*E#ITMdk1sf(mr=4>aN*dqQ#V&ZI~NzFq_VZKQgYUMW5}&Yp|r9 z#hU)D9TpEkDkoAic~)n_cA}`h^=C| z3nBxo1k@{So^Eq-wc(Lw0Ssh+{x&dJ>*(-JI47?B@HGl_7G%1%hLtE=VSLgu zdJ34B%(n{GJ6}Twi?g)jBPFdIrpEH}ik_dvs?L_=73pNE`a*}y;0hBOb#3E&;{7Hv zn#IUJx_eKJAKX8=r0c8D#S-z`|NeD#a{ywdi_}5AopKV4(EmcIsr03MzC%8fZx{7b zr(lOO^x>x`6s)b?!{0$l3fzUVY`vpfrM)sM?DO}(k&kr$E7wjafvKhi+0?f#b4H8j1F3cbi-0Nv3S*~I${9w$=t00C%>$^MsnP!#z$)aFn3YYts_e%~& zi=PM2xQK}fas3PG(x&@cgG3v3@i@J1)11ueAG5d#17B!X7g=lIXgNDL4d2dv};5mZdf61J1PEFaEcUG*daIsWXz`f z*suO_s{QzJ+F@AzA-!4YIqzPb&(2M>)UF;u$Jmh z`#ZJ1Q!k6}zPK%!Mm8=-2JU+N!CDEM5izaA9%UxMNedZb)$9PA?{x3G^rZ>ObEc#KtTY*v0aSpu>Gq=pyijx%pk43}7!2YmU*ZG>nOD`4a?OMw6Q*Uv%kp z%Q_^2etdl1`4#|!&7=#-Fb@Vp?6Fg0A31;lL+r|uC6()U|F|Vh$ZLI@rf=9k5=fcL zd3NeD$xmXK_opSl>~^BX^xE!uxd|MF-2w9Lm!&@eo&Zdu4dAJVqL6Iec>z|$cEHX- zOk9wt^t=u*zS~Syc(=nTq+$uv6-$J(*kM9Gl^;9vZ5hxfV|v5_CMf7vVM1NDzsCYQ zGLg1nI|QoM(0>b@Ukh;Ex26r++^+V79M-qP(`HP~mj$Nucgy_|)-vTKP)_fkULLtrbbrN(hdE@O5|}rEL52`ZJ)2BoF0y zSocrm_esYJ0aR3DeV`}X4A?abugsZ@4$tC9mEf~KCnEFxIQe8E zr@n_t!GkSn5+GmTQjW;Kw@Hcb=$jb*-R`VX2Vn1&X4J)vpgL82nDqDbB~K9fu$(@W z%ksSTkn3^E+GE;sTwcM{{_AU-LznNdyJGfhA4!#VCt=-@lHemH_|F7{*y%`9u$E1p zL09=b2ZFtq6Jgla|0!Yr|I#ceI6rOxy;-nisf+l*HCBb8=Urpbht$s7nGc?@v`3wyqbKQAZaJ2Jz!|rm5EXuwurg2$1jtH2n zf51Dtmn&PHuDj}n*cw$qfNJRy4iweK9$8&1RYbdiD*OUVdHMrzg$*cLr^`vn zA~WuIX-g)**EL;!aD@vthZ-{V?br9*G^pybGCEvD6k>sfva*k$47Yq`@dbeLkXj{T z0!SI@{O-hq2dG#~+Bu&4Eh~WC7#OJdBP0Qu2_h$#^g?9-EEXd7VB(SQL+3J*YPB?Y5n^t@i>8}{++Jo z^V9&5ch0hm*g@bLbg3W!zJjj*qr{1AqZ;du6>9K%8#@5`iX#FWS#z zvZt%7LwkY}zVAAt?quU^~1m-*|g{$O}{Q_VQBxLWLXc<|cY- zV`IZ0hi0+}CCuUEDJb_q%hbO{3IdxL%{HSMb76Lcq9H91!^V-=d! zx8A;K1{=GIm{748l4ylGSrhu?Ehc@?2wo5|wjB7$lt(~fKwvDyoh2xPlN-QB0CpSS zs{tGoB0kKDowLuC9~@K}l52yn%m$g#yEgNmNpQJ5V2>B%KB}*vBAPsOK#@$=UxMc( zbowzmK84`WIir!Q$TAf_KiM&H&%fRqZ3x`v1$no%a#!m5ZeMK>b9*5cCh;veqya+_ ziwP4%h-C8c&NWQ@4Zz<8P;e9`tNeaB}?b0iw;X*Ck6(u5g!GUu&qN%1FJK?0Ba_F#MRSEW zLkQ@w;x08SjQ5|Oyvw8Sk4g|6UeBs5b6yP^D=T#yt)~l=v>`Gc$CuP+wm$2D0@v(A zuq}RM%tkB%3(QSe+qn$&4(fs*KsbmP;rVRSc3jS&#IqGFj?4>GXu|G1s7SG(6FJ3_ zj(V3M8cgjruQ^cHD;5XcuV`Y=!=BqgMLiy$w~mKZw*#VZGHI^aIjxu5p63g3lS}PC zvr!xrz9-N}l)oZdsA8JZeD69jDs6=S<3)4#Bq6+M8f z;KsK$B}W>#NccI}=KukUan!o$s+~7+Zl!IU%l1K&K7!mFJ#|fU8g0rwy9yOZA{WM;9hgFsDOxQ zi%gtcP%wXgKT$~t@4c-yVanz+!v@48JXXa&L>Aotp&UzR85FQ z)+2ov@qT>`1PiXl$k%7G7|Q~#2p&s4)(kVGJak!(Q}d}i9D=b#mA;2y&)TY68Uo^_ z8+_u?_B^}~<$1kqL-2XLew7JF_ZF2}<9Y4KHz-eC*aDSBV~tp?UGyO;pXHCLP(HRN zwnJ8FLO8VjLDat>;cI`_c~>Hb(BHDQCIINnc0!kOQ2z51F!IX>$o{};Q!6ZmaZ5Yo zur-shC|+1Xt7rHhO~tci&P)p&uCyGO0@{Ujr2Cop1Sw29S)gw)NFgi3-#{3Na(=#QnZEm8|J99=<}bOh>Nx2`gMB5UP^Mv2;f79+%=hzI`lN9=muI$vPTt4NBtAc{~xd(YUE@Je;j-0ev zkaW!H;bH#|*Y^~mca=JOcJ|u_)iUl)b4T)2=*l+xplpPP**wu8Y)0+NiAtTtFl4X| z885agfgdh?jut%-Wy4!eav}P~T`;$|LTzG9i_5_w{bsBsefWt3){>wSHKv-T{-d{_ zG*}Y8=?P%$w{#FuSou)M#%9x2eVd};;iJ^sgdChvRJkLIr?Ef^m)Mr60=1Q8x;%U@ z_5T(vmqn1nJ-0>Lhkx%(08vJ_;(>9XB$O(0J3R|)4;#cM=yj@a9$2j2!Y>M=*4`Zqcw z?>e@io)X2;Y$dYc>mWv*Q1N7zYbbV4SW)Xou;Y@z)SekL@uXp@*aDBO992|7Qwo9| zmvZ%A7C??HwcxslP-55{>|VYzBUpVQBB7a-gj~;Q5X53Q3ue2RoX$Y8hgVqkKnBQS zDi($$adu0@EU(j1Bl%l_ab5CQdEXx9&DXv&8R;hjvNiQ;96DJm8bSdJM4nA-juSFS z!R3_`q>xQ5d=4LvZ#Ai9)}^Qwfy{J^tNMC{Xb=0dMf^rF<_^t^yT zarm|%GVpiN`?Y~)Y{&KdX8$?kuO0ws&RVcPU#X1T6Z44VD~>4Qb(2ity!}LXL<;rY z!mQbPxok7&i_}zQ%hWlVJvL3)GDB2lw@~By@fm>Lt$h%7kXF|8PiGCasPE?Wx{x^x zHsxzaEC@agT`2B7{ogP@Vnj)a%$qcAbz8A%+ghwvV@lM|?SF174=qnqC^2(#9l@H> zxpjjRh$GK~7u}^#WvxGb@VqJ+U`+ZDnKmLazr~Aj-DP6ci)qh&-M*D>dWOuuOCTc| zsoEdoaZGc@<=R1&2$v1+5bQ(@ znisNozgVi27j?4MgD!D?<;erq@URIZKL8ktR4=o7^g(t2E1}&N?FJX(d{alJrhC`W zgQWon1wrq@rqM4E02eCWcuBiOiV^48PQ2O6`C?!YP+Y9-pmee+s+d~eWCqcPfAv0( zxDuR9b1Sr#@q-3tyrM~gs!np0ByKU`XYcYRoKU*B6(mi+u9|O2X&F&)!FMF*)Y2Hx zE5r^M8Jo4KMn!=&)uZZp)}lFyRCrM#?I8(2EGzx@YEahxs~&xb{vb{GN@vpk%8{Lh z;QXoG<}8PIrbFDseq@Pdk(Y%RQ!P=}_M%ZKxsKI@FIuunZ;>M#m)dLV4usZgS+ zshrT-95LcnxPu!HCtxyfR1WUBhHB@Knmvt2GYP?wtj8TNzlC!B+Tdt;|pk-ys^U(4@LvF%80w^%rRM2wOXW1lq z#?}qgbx6^&ado_Qu?Devl1e31>7U;t$qW=}G{NSWljZb?kV`*67k^bOCN(fok<>wk z5sz2-o}Q(ta))#wuU}dRr{IsBsWEOL-h--{b$k*g4=oC_VCK(PY8G$Cp>g;4d$A#w z6E^`D>IN$|06c}9aC2#SHPGO!7A#jkK0XN(nH!S|@12Q|SV?BFrhuW*+aQz1oW2!{ zJS9FsrgJ*uVSqolT-HEf$-+TcC$t$RdFidX!>QCkttB)$tdy7_>a?ptIrZD0PTDI{ z$@IUSwBSFTG`KDf`hT9N&MU~zOaXAFkn$pjHe;0Nx{Z;wl(ce(Y3xVfp}Y|Sq$Lnc z#8tYomNWF|ZEN;c^r*IoS?6kBw^-=qc+A*tBGhM&UDb7hmh3^1o~a9J)gl5tMN>YA zSiek+E6ec7S}?In8K_EDG#Kd~10kzmBT+IAlbqph_3fJ=k46tBqeA2bOx(Q->UiP| zW*Xp!5O7CL8%HFe03FVoK^H2N;-{wO7_0@G9^;WYxED4&;0OH5*uxhbOcyjAZfS`A zo{QkVJr+3#(H?*e;GY{a6QU>{J2MArk& zQ(}e)GBOhgLPQ-E$Wp}*d4^nAo|;#QC7dk3^CrbyC8CDP6OK!ljvhHbJlywqnl~=F zm4$z7kZVyM=H}w2R&zS^%IYrEP^l^~h8X1{!|K#UGui(7vH!iOrZ|`?7sH2;$nqB7 z&H{Z`o@`x>xYz}i6MT%#JPuFXnxasQn~2Dniy zkkNtf98{Lff8ZlBe)bWUOGwy3l1ijxkgDiV<;-)DuU*d5Nn_Nsdao=5|8-o1wM<>X z0)dhe=QlqzMc)+`go2#;io#`EQzdQEOl}7@>RP*jBVt zCFYI&D~wyXI=o#n65rLAh)`Sz}^>8!^O)AxUPYFWFj?Rl)^?=d z!h+M3u?Y+h<*rRG`)%BNBMDQESrQ>{Qe7 z%6FTgPd~?E4!bG~FdILzrHRQEv4Hc43)0D|vL%bDe`d-C?Ym^@z;XqfB?K6&QQQ{V z(({`Cs%5Btq60jGaM3DL`L$bF-!Pq^M-ig6SQv^5RqB*sIXSvdgEYE9KkH3`YU!I00fxInlac$KfvmK2hPYGIPLB^lgU0&%|N2f@g64fFQP8<{Z zy;z7VK#bSDIn}P6gMpxm2Tcp9vSj;R)!F++Fx$X7gw7_u+?8JylmO|3lI_@dzB5u9@;6mCANPy?{J zDrjZmqe9+F<~oUM2PBJACQbacU|Gc)gvbX#^=Pu~`3%gQb=-Z{d(IQldydAdf$T$) zYAY-$c>vr8Vy*EiEJLPO9$M~orFlBYzUJ%Hr?r~$Z({{wDuwC)V;e=&nG=_Hr3D+AU3LPX;keItJ5M%jO{nqsq=WbbU9jC zp_>Z-DNh~NRAw71s%NBg$HgIb-=n7`UZ_tmjUkJ9;pm<9`i{`B`P~F##at|)t3HJ8 zG5CV29D_AfY63tXow?e8Ale7TBDVImh!(aZ#uQ_8#vN``cHvO${@+u;o%ZF3p#8Hh-+yP34br^lAZH!uggIjF&s6-bqe)v7 z(to&)j#$Z@NG8iYWa9TgKSYKRKW=EtqMQZM&@O*7T!n%FA0G%BX4kegwh<#pjX0QD zlHU+{`&p(&#B=~?XA3n(x$@LbHDf!m1E%(IZEj6hi}c8LMi(77iXmkw*DWJz#Xhf( z2S6tE7esc4bpaWb#qU*RjhuwYK-;CE0${zhU6cc9%wq8+d!o zz(Bb+xM|0O@TE+25c2m>6LB&lBA2NoD-OQ5J`@Ee8mS3$c5nH#shOGYhlwFpv>$AS z15`6Et?ca`9CCSGirH~j0eUW~LolCKm2QX#?azo!3hf5eNXlRBL1Bo11!(R0=}G9< ziBZ77sk2EFUnM3Evr#9Zj{2qQQdaDj$v6+{xS^oN&2=A+(aUy>ww*HibyK@b4TTI2 zI0|OHABZCg1EzEP-yKfR1s`TefZw(4S#9y!Ozgq^K#kh`tt6dYMDc$ca=-N;X?LGD zdZoEB!2YjK_bV5*^p4|53S;*`MET3=yE$>Xl=vmYkNtTL53j;yCu1}rG|;R+7uDWl zDYf&R)F_r9=%pCk?~E-}>TPt?jUAx;zKUZTNH-rSM)X<3Td?mCW$B{{GV50Nc@y1%#{EIqJHvK=<8P& zbu&GJV0!aU09)0M#P>HVCha#0FuF+uEhFDMU_AYKq3WUGhv8)q0vchb&uie;&fweY zj}^14{sbHhy{qq(O<|(N6AT1Klf5+!Uj`84oeL$+qf-s zy4k%5&S~A`V++GwWq3Wr{g&e(12cOT6IQn;P$p$Ycu-QizCCo>{%oD@HQM^ znd{3nb8ydp(F{bYFA1-#&75vlxm;;e@_CHJ`FQ|Fu=S_56;EEGOuV|GA0<_8PqzRF z>*^7(D}i&Xk6YyW+%Iixc%#Tl5wvl=omca3=(IS`GqwE&xM{zEOk)cau4?^8iz{Cg z%ES?GoDF{<$NlYgW^8|A1qp3XXrpD_?|YaB{4S)zdDiIq26Y^Ff*ed=48-)}Zp)33 z-^>U=5Z;~izc*TVxqbq$)->}gO^(y#pUH(Uq7HO@JH=6icAo)X*SwBp5j$_wko%GU z?=(V&ufefA&Rgl0IW3=d23v0S6Nbzic^CcplPCD}iDznrd`B%6dG|1X6tup!8vQ=J z9$P|;NS!nAT8fk1?8UCbbv*yQrX9Ypwleg!HH!H$fAYYrlpmC}Hn%o*asc0w1tR%k zqULfEvLP-YI6F1vcwAPJ^X%X0vjggST;`tay4?Pd((#1v?Y-eMTZ41jI9ZHz@-*Lh z0t@|0o*d1*mZDzVWx&UM)tQ))@c(Y&pZj=-9&e0qkP}G_PEPv&u39W`Vxup15$%S| z1O&cYV@7M1Z!g0&I;{Y!4F;6acmr6AG{b2=bdHWnAPQ zik3=;EnIeIbPOeNkh$x^W3}4r_*J<;e6<26?TfqNC$X^;(!@G5i;Jr(D`WI+$qZVP zIf5lkO&ce7^;l>&J@bruvDr?ZzJe#*wV$R(X#A~#;pUrzNfb!2YLW0w35y{rwK6fz zALv7BSFFh+&fK{mYE`_Fl{U|Ta_JO-*Si{j-`8`56h@su3ifCGd*Mq`Aw#D%<`L4& z{?WFw*|ELjlg-8BQtus$*-OWh20ZRn$5{ZZ_XI>)*EXC1`~NtVYISI+D%YpG=Yxq2 z-=*uOR;?=5$Cu8*eSiLo6Y3`ZOlIZRA^=9g*0zRodN=Ok->{qwsycJ1(mrx*7Ljf! zjOBy(svPdq-dbj{SzGIH=;{FMx=6N*kmB>BAHP4Q*Y9U}Eg$8+-U_4n-A}Z+epWPi z+EvS+7)e*hXDohTOn9z9(ewODj}C^0`hQnI6-K=AG9ONI7N9JUN&$%@CL}?slpxuUgVbq!a ziw9)HTkc?R^?kDMEsc-D z&9Z50O-^@IpZjtO`0Z+1_VELE>cdmc&Wov^Qx6hHOuGEn+db!;idAz$p=-a*eo{sa zW0zIZ=j7t*1Qa^NOucXR10E5EJhy#O%&eL8J+ZMK0#f*Mv_c<4{=FNemf)lQ)dwXt zEm8lhhhY+V$Nc)$6C;Tu@4Y;(w$%k#E$3}rm8dce>aFi&N!}p02e%x3w)-VT{l$ya zV{h+u6!u^4g}Xo1Uo;K{LbaWp*Dk}izcHphw61bh*v&p6fwrXDK=wTbSFsJMOD7H< zTP+84t1dk`$R#!l*$0-GZyc@xLb32zF7CQJj?NB z#I5G5C|EhdAKhc0*5<*L8%$8BG^wxAdp-w3Px}`wCeqn~!2+1TGWDm`qmm0D|5v>+ zKtL9LHCy0y0rz_Yawr5G77So064u>L7Kx5(#f6U>_^+1;iHgu=tKU=KG%^3`I(2CP zY@{{6MwEcfG=%25RWN<{-3nKg$4S-vzJ+PV^-%-Q~HT>%Ie!UTXOKfWA&EndbZ+SaFNHz4L zgX8z>dONOoV<09-6SP!RbKxOALUM1Ylf5_Q*mhmyayak2DmLo)HG`1J?XVAJm+xqt z=RKQcV8P85;ZC2I$+q-=#kBT z7vA9a%LP)U5qGUq-%Au=34A(6=fzyq>iqpNu>~FWX^CKD@AD-+cjeTe!pVh!5BHKh zCDWQ?-`%ct(G!ebG4R9l&rlE>ott{L%LcFeUEkwoKBgK#H}KncodNWGp)0eY#GWKs zstw)O&@$DDVzp$`{I9aeoE{8)RNL(Q*zjNb2IhB(fq6p`M$DT9!-pFP6oE@%{eel~ z-h3NPorD@=ujzp9cs@7yegAzy{;{&ArAQgq>*9&0lUk34r%g-s3s&Uo9|I>)2)+8SH}<7S0oLgkxJ>7AvQHO+G%%W z2-b?{qh=bSSk~w47u!h?Unz4-@Y`f2C4HJWbi zt(R+^sR+30kYlSWY|gb&apKxu{7l8-=*;rv?NKhO#J4oQe`*_wildmOOO!>aKT0YS z!^Y){J}vJeT&${-cjnfz-9N@jzGWc9nv{Rg&uk?)WoVzHN&n{P;xl|yWQ&4)bdw3>Q!Rygt4N6*^^=vxzO!tW1A?DH7mdEfP5<% zu5lbq)0iwXEiSvBd1h#09j!IR&!53BpAf?q>D$%IY70M`(8fJF^PO%DI@W0$>rrQ} zg~o~seyqo{O&vI*MMP2&Uf)>Ya~s{<2f+%~OlB$;>1JexG4y@ToKsnIZc9?6daoHZ zc7zCtPrv6Jt6u2rnO8!yIzMWB%B9!PwlYYp+Lvt15`TC!lDe2MI$HSNg1r-3m|ukT zEnlLYLm*m#VvAM?H9vXiY~mxk_^C+F*i6F8UICnVL7_a`^v@YKeLB7ObAFK48;-%6U>wI@16p(+^%(gcd?fs?>DudNnh&tg~iXn$`v_h!^rS-|^90$pDxyKoP8&M+%Cyl8S|M9T3 znNbPLGv8j=V*7}_ySdttYF!dBVWl-mSiL@F5+^Uqm+xelXouDLWT*s$yMY~vy)g>A z*B<)k+UDIPh@8;k=IB(atjcm68$yoZ1nG4V(4s+#=Yu~foX9}PIZXvK*Kq5V@O z+mS2l1&)}O=SdHri0aj%(+jcSnzv}tCdp9K9bKGE(xCUGqVf$R6z322DjF#~iPS&y4T=Ndk4js3+&WYF3_x9<`_sOxLueBaG zX$s5CsQn=I6_dDJpYec=cW`!oIr1i-%x$ztl?Q;fiGRyl*|SFs>mX*Xf)|H6#X9=1 z3!iY?zb|c$bp_6Ioi_a5bX;LeY8_j884XV28>dq$QPQZbzdAiXQRT0(o1sx23^-*W z|Ddh_jTqMei9#Ff*%DTY zrw|$j#g;!yR_r}DOna_;Outycy^o1&$k)lNcjYkWoBG3N+jJZmu9#K4qWfw%4+|jMg@^>1RlCWy1TnuMCp+3?(XhJI;D{k zrMpYIy9K1ncXJMS&U^0ve!U;=7!JpFKkoYFT64`cmmutHC7`845&I@?RtToPD065N zPY%;4HJ%zeQ@5W}nK_9dC!M!s#@^j_dm{p8YTc<|C|U8XCI;NKD5==B#GC#+EMdA4 zxRr$;v_3Plq4pH9;Tm#VvHA(xcFHs;^)sciv_rgrrjU*|Y_UwLW}Fnd7^?@D)6R1mxyGf~hCcCiMrp-Tg*rKyO$V#H#sL|~5p z799;ol>wOT}xOAB6 z-ZkyRx=#7ZDX{ahRYy^>pWxe6?#Um#iocx0liLL>w#+u|Sv>@Pp7;CxyUy!3H*MS5 zC023Z^A?NmQ{^ryaX2;_{Iqfdz>I%uW!K&bJ4f#*7L_v&c!~@iuI1XG`thj=h6VZ} zz865#HZGw4&tb?dNDJQ=nUQ z)zFfMtm&X?ED-BdXjaSeS1-a%%?fI8Y|yMSXkWh^qv}#!Fdxp4L#O_T^LhKAPD7V+ zNPQLXgv(3W)QQqI!5ubFT;UFS4C?LRQa)DCRyX3Bjg>T2&6hS1T?FOOpP{*iF@oBI zYyIO@es0;F4KM_cux*)l#rGL{c=D%6sa0kR`P3i8?1{SjPKY+t0&yQqW~}&>E>kOL zmaU-v4#Qi8=J2HxH=I)Pw0ym)w8MmP>sEq6qhvFmh0hvH?-yU-*}d~X3QrkY7q5${ z<~Ve`@X>4VdidPzW;w8&>~8d0){-yVqckE#fY`LeQ3M^`!QB>DzIu%iG^$D{Pw+Y5PJJu^m^RDCU#nyevAaG zXpgBb97TEsK6*@6bhDP@Ed0LX~pJrh+mO zrkfSVAebjOmZ)%W6E9OJl@d;moTXhjohFsX*B)@*uoP%}BXJOV7^!+yXL&`_00APSwt?!S5ieY7&}mmCzzBk^w&Ubf=1F>S z{oc5t#A8kS$MpC6Ybz11w=GHY&Xj(Dm{W1;^Q?Id22+sgs)1elbRrzNh_=oDoN$^l zu5UP*bKeJ03%GoMvc?M$iS4o}Rnz~WK5MXLY0J?6c_JF)p`gH6yM_OC-RuGXRU1-G zfZn@zPCycdLWn`ClJYI7>ikDY$i+6~GNrL(c4UJ>60IJL2>2}Cnf5nHQp?s8722NX zta~*-6%sF-8hM3<1JL9(>}R9sM>cGKmTLlKb@~ydN)znjrZ;~5Drrz*dfU5oAcxtf z>Tcj15g{954;h9TMWKOTF|oVmYnynAV$nZo(x~ytY2@Ss*XJR8Xm+JuS}ksIa`e>h zO!B<3t3TIl8CC`Hx}rchSJbQ5lDIjyab zPh<1w>Q6XnqvX^m1sQBOGp2ut!TiCgS$24M@8`#ZnAVXJ`c^wANCS?z5t5h&{fwQZ zvhat3O%WB_Z})I_sNe3IL#8*<*KBu95EEqb=j1yRQ0*O=g6i;ALa9W&dV5569u2Zi`|Lw5DpRpa|1vX zs3tYRM#Zo(y~?vpiAI5c2DGK8KW$k4{zIKki-%S7W<9{R>o<`QH&1*`018Hg&tt|S zA0M53ZD~b;6wH`g(5oXf8*kfa&$4%Z&6B4(YvF$i@ls@f0?p^5?m>dz>~gKjxF|p6 zL?JsRKL4oP*a}IN@{`0d?AN*r$MMwO96eh6V)drOmft^*8vO8cH8t}{G@bj%GP%!H zTuD84i?=_$ue*yd&db~=V1YxVCevbgR}?Q~ zW?MTi-(PSao=r5lxe`}_v|eUb?=Lhq9X8+Yv6?AJ95Ba^76Zz) zmd!-`3 zfV_vz~Yrx~vCm#I#y^vz0*Lfgkw0v_eT60~bA5~jZ zJ-naMI5Ft7q{e5obO!*V-^pKgy@PMaX-bbe9Q!QM`p(sxv~{pnzK za+WioS2uO&Z);ynj<)_FGU1a-uF)iT8N|k^!D!?b#O=)u$5-kDg&k^3YDQ@w@uI zRW(3og}?D^JTisZ>igkE@kD{_(E-oFF$p5yU2uNIH`g6;q_nCDxe9tWzxpmfDU!1Z zAdD9W;Xe$EVpgcPWLNN?R-wrmNY2eUO*>qkIaX->`mAo5FC=X?@1=3E-gQZH6a`X;hdIeY0$o0%hgZgXc?+#|CAsWpZ_?|=+7 zGG!Asc%#C@jO`I^=QBoVyu=WoP??{3(b@N(+Fu41TuMJS?W{>3X}T}#u!#Le;(h46 zLBP3OGFn#V`!)M^f9kT-WqXZ4o6oC$k-Jca_n{#Q&tbvF?WmTWYOg2&4r_cuI#Eq+ zX6vN$=Jq{<^U`#4AfEoww%5Kd@cu6(_~NcdK9?I7+28mPJnTf#-xVvAp~zdl#`uk>0Qa(^l@u!%dTf6Cb>z0-s|QJ;G%a}%0|(>AFp*`Ue~>GT5@>Dsvn}+_nRiAVM#C1 z8GR9|PcK$y+$-iev?oHUlNKB|p8n)x0Y)Y!*wXoyiRHQL_kks6=9#(S{UzJ$_5}VW z)5SEa;aRf^KsxYR9n<|bKS`St(0TLn_+^!We+fX$wt!6GhkKvMNV(5HK8;`kX@%;T zG&B2iW?pS|3q~22=WT5>^^ag>&#y^kd9O>iW&Z8K8bJK;>h0$#4cFa|2gl!kKimS9 zP&R?|D;zBI3`W1V4hKc{_vh^C*6le1EnZa>wFw^8Wo$iOn&->=(-lFJE!tQ2U16BZ zF8gKX#1DQ1cPnArp1VM7kc>Xh&1Ls9|JO4nDj-K|(W*jCG~?ppV&1apaBkldD5#gP zsI_ENZ=;Nf)(FVu!K^tpT`vCgo|44VzMY#U-k+k?yx9Xvgr4TpdlakDcwG-@Yc|?` zWW;E?OEq&`0!VIJANH&p@ZAnVf(Sg5ExIz@Zc;|P9)4MUJ8m}`=DSVSk8Hds`Mt&J zl48EJcapD^;&HL~Tb%!9>~lr)&5Cf;U?%_a;V%N)ULqjPdT?R{sEW~rsr|6+#kW1b z{BQvzC%Vt*ifedY%Ll;)bl24lr8Onds$a~Tm7mBR=`a@n+AZ!(nkC7QIaSj1w>CuB zO>1*z{MYh@3s?JTksPM0)tgqDXZK+)nTsYy?w)(`Zy0zkX7n>%wo=qBn^+Shj;aUf zY5;B3UlG%lS}RObohF$9C+C@r5n@jUupy%zT-DPp$;^v-B zOAW_wS#B{NBmM?26ogr>t3G3543xR&<>d_#@m-PUKY>uCd=0CZZFKlSoS`>|F;7xk zn}8RU3K!8S0{6#d?JIETcx50URJfekXYk4E;TnkB7A8w!IrLwDV@6&%cl<&Nk5Hu+ zehRj9fC2lK3S9^#CFvU@=w>KS>9G*;UgkG8(K)~W)RZ5Pw8prlMq{WqZBu`HKCMk}Q1dXQ zDwG|emhS|l;l&O4`>p@*1S(N&NQTMb_8UHylLJEA7eLT!w*Ud>XTy4AeIo5INF1pE zJv?`HR?(*w8=yw{dqYg{3&~pinZJCq!`K%Evl!Y;f~^4?oekx zm#Tk4FZF36&6IbGsW56)1NZCU)rdgZh-~X0Q7VBpjN$&?v*vYWWf8nxjU#ny_CXW7 zJ3FnX{uC$WUtcS|8J&y|7GcolAatBA051dZz$P{eJtqr9Tx8;K-T2u-Zd|Q;OHJa8 z_TICL3%$-@9V!fYpsFe$LwyDm!|hD+*ydi=Z3e1}+4q++@*VU=WUkGzuzY#-3M@jd z*bxXtRyn+$+aF{ex$70@J6-Js4}h7yfXn2ZSZ6;^>|H3$A}eVRf%vy)Snk8JU= zakxKMbPriublXd!rR9<2?V+@K&qc0IwEjLzBXPT*CBXW-9?rZqIO}qG-Lz;sK>H}d zr|pGwVB8)J1WXtE=g(jV7j^5CU^kJ@JQP0u>5UL^f!QInpDCtdPbdn*5i~s6I)bNF zF~{pL#4j?R3>cBN?Xa4L^lDON8)c;JNXSB(KP73o>t=qx)HthJ2iPxK%6tRTt6W@j zw>Pe~SQ_$C%Rlk@3xu;B>6ZxUr@6Oc7s8^6BFc5E70U3MevUuaai zDVy2ZGMu&(WODC#-%nR~u_s8)98Sf_KlpR^PAxqE)ilq$wlhD%u3AU`YW-RpMbN~Hp$9C>JKBmFLebaBTmWt-n(~AYj(%GFf?I>+1h8ZXxNIFLpWpxo zu~t3oqjbKAb2BxVrXR84J-shed5U>wm$%`9c-sWzrwucEEbN-t%C+ks7Hciy=4y8H zTHXv&)gbIo&6RCUO&O}X@%}!T7~%a*+@giy32CsvuPqzHJ>RKIv?{#Ve`)iiP zt9i<>bXIPnusio9j%t}s(aFwAYv4H)y~fOI#75qug%yFy@4T4ZD%^VToay-wpYc}{Nu640fl z`K#K?pO)N*c9{dDYd}ye031OkG@ZPn=MV@~r?8~H!HfES*0W7_vlIZx3m<<^{*?V6=4c;LdnEZr8{R|^sb**=Y`ek(;(|-V?DeQj+f6*rH~CZU)V9TG|UI>Puh}DZ>bUY zn3fkrA||?KvoT1T=XdI7!w5lH#7mQN7s%>2s`6kXyz&JRs1Aw}qZ}zk)H=irr5*AS zNPi6<3QH5e~yY8kCRUIRJE2_+}P7u^Crj?05CY|t!<$at%b!s4qGxW{@j}Pu^A1LLGgz- z8HjbYj2@2mOq;gl*2%Cm5hP6L#P|0cG+b+`T&0`8f;xV@wc50mmWK1|*_Bx`p}6qx zuU=QGYWU^*>T@D}{a2Uu^q?;!pZGAxe84Umn8w2djO=DC=Dm+>lfHW_| z%!(sNHbR65*Sf7MU#pkiAbMKkjcBd9MZzj-iUQB&PAj2&g1G(~p53=mxuEBXEKml- z5=v?*3QnFGHYeB>si-nPAc%3y8v277^v$x}*}|Fyy>xnd<<^RAlbE?#SijV&vd&7x zIm;vALv{J>NOrEvC7DrXSY|T4%81ve&#S9zmQC}Gkj-~@Shx0FS4f?o-_^^icLGqq zU=TcX)0;6ARXCT8)9X@4?)lf~r`M)hXx!>i=HOjyGdt+m#I8ky;$I1~Lzv0B+?yNA z5N^>d&BHIwvc7{*b47!*f}%BmtbyoMHvMkEv}bIZvF*>4AuZ|HKXR-&RAJe(FrCZk zPNsjLPLPpH;Y!f_+Blrg9KIYZCO|&GDgZ*#&!X=RacGZWBr(t6#w|b$_+jyF(ULqE zUWl_12ivwPxxOk!5`}?_!?=GNhnpJ;3ew!M35ZsbD$`)*!GuP7pP@%}kvlkQ*)7&; ze&@|lE?_ue9^uF}vSVk7exE&sA+*}%e3;p@uV;hHA_f^JYHQ2T8z?{-hl73n#exE+ zz+ye`!_&^Fpbk;0RZ&qVRyy|F1JL2U`Flw2HM2_ivrZ|?*TUDnvq}lCHy9D3KI0!&-U=NkwO$>Zu~6(}*H zdStDAzc|V0y`V#KQ*%>eWh=I0O4v0%-H2TJWZ7Q577=M%8eXFk2iPgt{qh-%-~FSh z;H&r>y7cJcr4j2GnGn!Iui_b+u6hq3p`e1NK#1F?ltj$DsLUYv_?ZttaKRafmaf_; zQzQ+|;3Z?*u|=FzUETAEwWpAW(=JV`v&BTJzy}Gx)J&7^5>l47i~V5{Ky(!A`IZLV zCvNAkienZp-JVREE$%3JToTQTGMVH?!2f$eAob?On+!m;KtGTEhB_(KT-vfWIZRkM zkBLz-Db3=OjMKt(Q~i<#AR-l|>2%0?e~(fpMB!JgK*wBbz_a?xHAbo{n`az0P1V8r z8dnO=D9EjP_Q1UI0SwVL9zIP>51Z&pq=-xL+Wi9*74beI373N`de6`T zZCfYZBv97L+yn^2kxW4G3Qozmi9=cXe$?A^Z_fXYC$H?`_kq~Qj@YMn(;hL2`Wpg5 ztya6jfTd2-&tO8JC0VjeoM>^^=Ij>sOzlii_y4VyxXaT)@8Yi1{%{!yEan>7PboC?*Zx;A$D z9WK=|3XUpWtSGCBY^4+_GWCYmNg~%B;S^*I8azaa*(EB>*KQ)!nxg)<4qerJQ_JWR zDY`Xe-d?bfP;6EgS9gSZ30M^|(&yrH+~nI~WRCA)gM_~CPjfF?UTiyZrVCW6m;OK) z*`-U`KC}vP_Qg*T3K#XIo4!e)wm87BtWr$zXqD|CmqBhc^iox%qiFJ;XyN6t; zLX$FcoZe!=Pnukpkzo2Xmjn)xf#og3C_m=uM632AxTE(DFAsq$WHCUE5;H<%h7p>& zGnUZ+fx1^MP;3fy{>5y%+<^=$eb_=`ODW>I@5_1L6J1SEBJ8d(E>&)xH7_MnYc%r4 zSwml5He;h)pGvGd9ar^S>07~d>n~sG%$F*`MnsnoRRf3+5~M_QU_kmhJkTWk(u1~~ z8mPQVaxkpl5fH*dQKzaI4XHo_ojdWc=`cbp`V0$9w*6`Y-M)NxV>({^`!7A(2z4VI ztW%s9)-RcD3*IA$?jcIb_`DcP#!w0|CWTeZJiu>P3W^%cD93s!eU~~nH^Uw}NG7!+ zAS@=WzJ}KL)h&K`L?NWLpO4sjrL3gJJF2gD)3i@$IJ<-y)pW8nDGwT6bg5k$9^K|&%#uYiT&+4jfP=_KNpu$qq zrB|#%C0A$%J|wT4QVDUDbLy}uJ@77xojxm`YCzseynUy-?MtweuVi9u+`DTnKf`IJS1Z?Qj^B^8njqF(+2QVR4Q@A-l(PRJGrgPs z*^ZYM@GZ6^_)h*eJ&gec8q_i#k-16g7at_fqWqTi4VD2h&oT^GPz?P}@4I+mdNrp5=?77JKY!_>wamP?$Xnh+I}eff zcS#v?7yZolMPv5#2VS2()vg;i%BPC0zu|U;3WzUI%HR21hSdMQl@Kz668}|Mq>i4@ z6}c)7)_Wo}JzSv0v(lyyZ7&F!9Jr)WgzD)%R>HSk9B{x98 zCv`^I(e*y8P;m8B_G!E^qfZODoiy_>+zK7KGs^8unz~wIqF5i$*pHjIda{1AECDLO zkHg&Q&Ziy&2dn0X&DYCTu#UhrG<(a8J|;akWJt`j1hfuHo61n59S7d<-fzrJI2pFM z#d6tZiXF7+FMs#{o*&Rg9y~@vPi?e@!tk#CWJ^{fK?x8(zHCV6tC-NYjc)5y-zZhR z?M$mOWKo1x(Z<=f=PKJOK_dGdNpIZP~W;Pc2D&wp3d=9%r*0L3kk9(@>okYK@kf+@RokPt=X%FpMfQFC_2a_!?}T zy4{5C=vfP(zutZT^ZffXx2ekByXS>K#|@Z#7D&9cPw(f0C!;g)(@vNctB){z_dc=L zqPWGcr|f9o?}he2nOr&oga~7+8MPN2GI~Z*oc2T5)#yOG44c0Jz%Pq_71JpDagy}nqMmgovE#1g$944UbJ zYYXhpkZkGamBnX{5O@~Oh-B{bg=MI%WG85g982oD14}YLX*WJ#$#}gKA#j2QOFOK1 zpr_Q9USCQ3CfuDX_tbv^5OtZY+V>T=7eUGpe{PHv=Y8S_=n^*5-tR%()hQ@tRFPM{ z1uLQ7R?~my{`B8K%HTinFdQCpVE*TyR(=^Em6JfMF!0al`Om@QT6ly1zpn<=l`sKx z^{~CL`#)Fx`6qBO;{PWXGw{J|{JK^uHegGmP2BrE^^PjRwsFwKJyJFI4-X#Leu^Bx z>po<9|5-JOkNIJ)-1+QLp|tu&Wzsn*uQLQsJ0VpmPM|oME5Bg@mC{HasYA1A6G9YkDsqFgRpH}km!z`1xjds-awYYeQjCx zwf|{CJ<%GQ9tSoiAM0~4`9!kSux2&(L5x4hibX=Ek|~iHUF7)K>sWSd2f33zomH|5DG6LqB)GQ zY2eF^=DwH|;X)vsLf81^uDV)VAD1^BecehmLuQ#N>h2a>KqH1p^V4bnX)WlHIEDgo zD=dJGdPuE(-46)-k(F~+%Bhf$vzo}!&k*bZDKT;}68@?nIw@oUN+M!wi(sNPJIq7T zkCv->(5w!7D!6ETVWu2!j>K{mk5CEXSb?Hk-IA0BXy?0#O(OK+DeA1p5-cMB^mmao zc)w1`Ku@{k-@PXD8ax>@JtjM@_XyBWd}rBwZAeGiHn3uLEQP~vDYBF0tibP_b8+FA zj_3(>D_r~V;MuW=44dY+uu8Hk!9kLW*54bN!&d@rzK|bHLvi;yW_u*F1<)F_=aO}5 z`q|7a$YwFe^07a@(Qdn3vnkQo!oot0^afz+6JEB%co>MXva$k6G}nL>mw^t5 zY^_4QJ~Ezi%}ysPcyMfCLFOKaR4|r=obIq7{qrHo3cx@opIo0bc-nmh6Fw;L9u)dT`>KqO~c%k6M$D6NVUsQ%6vahG zA|FU<%J}F3u~Ce&3h0D&lG$7WQP3r?-}jPw#GL!Pg_+Gad!LKt8>`;BeCO30`m@Ce zK$!Y^|JeVb1<87n$0G667BjG5H1M2K8&Ip-(M)DjrESzz)#0d#H8HjR)or`cVP{wC zQE_zky+lFBC=>QT?(~Xf(XWi3?%TjOyhMd+=~&c{LRuHpm*(BV-QIq>XtF_ z{^gIFe7ka`PV<0%q>YR`@A+hD%sLQ!5hi-fkBZlJT6BOgFa?5DEY;y!;pa}EFJ%e} ze=_+VSBOaREcpK`G-nP4MuXHS8caF}cg@!Wge8HJAH#YqeATz(3GcKd=TI&Jzkhwd z-fhZ}H&Xp^!1MgGcA0hy{y@hmi*493N2x{!*!_IVJq|w9fUvCZN{MQPmMj`ZMNFi@ z!L0x1HM-9m02PwF4tn;iPu?@uqv?J7V;R3#i%$RE93hp%Z@-2QCj8^{qTylU90q9X z8RVoSyonokHt1qf*^R%+82orkq3+=1~TJvgD`k*92VRf={L0b4DtqtPK zI1|?i2vJpiL_LCG7G>Rw%lcxyDij+y!@YaGok{U7xUMu?cn2ASv2%9D2S->0AFWJ6 zq+J*#MTXMxU@JUF2uCpzofrXjQ-t$nD?-&pM3v2u!fYChl_*OIRxUBLVW<{}r~!Id zinl~Db9$yq^7>%5CIVkkda_N%16t)rz`qQIRw%%5I};2kZhkr$P69qdK;jX?NTjD) zs%8U}Wy`gq6xp;EJ5m@{mF*T=p>gaOv(^bK3Y5M%iSimWtp(T|ag*6V05vL)D6!B6 z6Dfc~ZxfH3QE_M|pv$Qq-#Vh(JEeX&CuyTbc_RL0X}crbrAAylI9 zaDQ@xA-bjm+i^(>JU|Ti;CM((VkW)TY#O^!uJ_FmKA^*A9^}s~M5-HK$-~k^Hin`} zm>tcROd2S+fj>-B9cNf)bx>O!y5IYA@6b9_71!!%o0kXAPZ-=G(=e zkCF6)r6JP>;A#kJf2(%b)j~B!admspNO-NomrWhgg?2HUq(}y-BD(TNrNl>xJqJyrFB0$F2u)9&?_O)+{u`zVG@{Yss4mA2x8rp zoE)h@0Z@`~XF(jqU?G;cm|0*hx1z=Li=epzTZP*6=&P>AqC`y4Ck6&Ms|iJt*ssT|g*EI}TsbB} zJBd4y7ZnLE{n6X)6yt;f1*gSu_v{E0)Oy5`bLqRiKjer9`U??zg#55(VHA^d_s0wk zjryh-nX0L&32zwR`zHL~ONudSrR(<}UadvK&kdc00Ru8MBwZ}@jcjWNkp+&n;p!UI z;Y#3cAoll-%J*W2qml=2v9O@VM=Lm!sg}pY8%&D4B=Ue3RYGMA5wj+J%@_8=WpYC& z1&5$m02UQCq<@CY*RCLX)4`bBS0_wus0Uiobqym$il#Of#sWs#X@lKj*OxHdrzt)< zr|SS(DQrYS{~sXA$Mw-;Ly~`Q_qWHETnV8ddW9E{fZ2_uEDS@C!fYBSw1Gk(2`$8s zZL1&=Or(OiNr_(_${;8vhlU!&1x@a|;>RJu%JdB#y;7o=+)_6HI_w28;(4){`%Sx6 z3{e;2vdW6Fi3!nUcLYwKZ4UP&ss+{y;t++iFsmHeX$V~g6~jv#KGcY)Gr$5I!|c#` z5#E_R?eCHP2Ri~RYyh1Gn8F_R*bPtZn=I1}l$N7vA0`AWguol3*&em*z_zw?P%Lzd zelV$0Gy{i5%?^Ahi#QSrm~Q86gXlQ&O-Pgo;cLMnVchCa1!)Z(-Dy6!RPiHGGn-xA~1{*tTtle@Cm!fDZ47ldA6K^Z7+&) zd!DWR$Dzr92xE*?U~{Da=t2IC{LQF>b6~!PueA{aVXe_Lulnn6FR%y>JkPT-bugKVf2?uE9M^%-OSYN~ z2BfWs8<#Dh(RA`@ULC&!Y|jVSAigIDR=@&&@HW87H~v;jH4dte6pX zp3orb_3(<%jAjOC+J>r5Q%Dd)&-8l$!htD(xe#E*`+GIKA#O#o)ptQ@8R6`OXrr^K zoOkl@88tTxeR11V6CLMgMGuA9U0+{7p56Axo4P{>?$Oeojr8MNnc;zLRKWVj2iq~Kr|`&H&ksMQj=^2hVpChX>=%A)u)+8jtbl-(si1F=Z=M~p%D3Q&Wr-yU&#TEZMEtk60Q_AS1FVZ|zDlf5L+}ZL{)0IGKZOdO zdiyU|9ym590Y2FRBH#4itL?v!)gN=_9O$V2G}3<_F(dZqctcD5`14NxIjEBp@HN@j zP@#JMsAtCV(^D!VfN@xd5?}I%@<^yamlW2O)3FMW%m7-E1|Vk#8x$1u>-yV}As|+? zCPReIgC7O>$sfuVs1l?m<;mm3Uhydn3$o6kW-Apdql)>CBSp4VB_A)a#*bEQ%cGO?S-kTif2Es2s+hbvM*jv22U zw2Gz40J&$K1eeNeSYFJ=gig)a$hK!)Kr~d01eJ=GX~&gOAa03e$G*nQ^EDTIbCQCF zOoYJ{5fKraS1ARx(xB%3%XmgPDPtk)Zb`LuRYGJPt=D`1w4YBZ;p0GIWmG`;8*K%; zcf|`;Ky8JIfe-}Sp>6F$cAd!@fm#ZM4-w-7#rQ3JRXNyuj=>#jD*$#Ozf%Ff6F#T| z5Nj|sH5GCvMTPk=2ImmmElT-X4-{yTO(GT`+?Ixnj3r{12SWz%{vZl{mw!g}taq}f zB2$3iQDJ>VJzyggQG!;|P1+?Ix>rCb!7`jNFl*ej%7g} zXJTfiuk{+-AFjU*ni29_K4J!H1VH(E0gX^>CGsiSXdT7cu$c$5*X~m^^N+lzp_3)| zXe{>mO|d)~i{xM(3Fb{Y?P)K-hjfEcMUYJWP(9~!6%{D1NDxU11DOk6?(1^k9VsD@ zc*Ct=OvhV0>FB(G^P|>fnA{%J9G=0zR`R(?e zQ)Ifwe#0R}Bv5=QvYtiG);TJmLQYfd9jJz0WGnb89nLn$bOsG}cXE_5e$=0f1G_}G zDOfmLmw+<6dJLncL;M34FRBlhh<-GDNRv!x8-d$Fl{0~_y_3^OGNYfu{!|(Bhp|^L zb2-0*7*WI)a)^w|LpZ%2?f@Mb>~#ugUL}cF5L^X z?G1u?N?6(1?Fv#?8{u6K4-W~2{IS-v(PQ4ri)Y7-h_ZG|3jCAgPR#HCL^&dU8|9%q zhfIr51aen2;9GtH0tc68XVFo90)p;m=jYjMTwGinB;wJmS17{Ol`kSx2sZ{!HwPr4 zb2#N7^nKUbzUjCq5EN>715X=E@!Gs0NT*4g>| z`Eyucc(f`|)}WIJe8{A|x>|x%$O%H%-@a3@r88UmMu-&93Jc+IT5_S-%mW8-0_9SX zb>CE(%KG{w4tsICHD*X*&unMt0ai_~4HAOv2?QKA2FMSJLHX`kS&fQ%gGr6T-g3p} zXO=T$kixh;D6%g^@m?>@)tZ?|w=4*0D#pS2`X%`hp`=$1#=o&+kPQ@3^Fv3s|7RU< zy`snhAk^PjCjL)L>Txp!GW!)K5A-af1T3GePq9l2WZ-Vr+#|IaAjUpoR3cd`g3$ssC=8F!bqYfiYaxSq(7j!J0zk7#jJ5L#@MWu!o9MG0hp?5COkx;8o)ZT z%##lQQ~^aRotv9G=6wVH0zzbX-R;ko-++JTo1cS?rzkPqxvd9qk|Ew#>VWkLst|(v z6i%kO%iJ=)LGHB;N~-C?f2@O`$=DZpTg81tx>Y&+trL{70KEK?19Q?Q#L6W~x|)#8F;Pf67sQ|6T7k;Vz`8?Sm+E2I9jE7maF#qfHAdp`sVb|8&==F+^}e;@m*) zL_3EGL)W7N!A`=1jp@!XrXym`P>#nW(m z`ffM$%7Umvd;>esc)dFU!U#V68c4CzSrgFFQ4vPUUbU77ur%DQ9e_mGH5(Hqf*Dw^ zH`tkHHVNWX4JS~5v=G+O9R=#>ETaX{fgo!PB^FAWeio7cgBYhhBK5x-P5xMwNGPou zIx_Gc%J+AA}aN%pC?U{pAc?AhY@Y`JNfFs!QKl3x z&~Q7foxz}!<2_eVat5Jr@{ykN+#b|i;c38_0{GF_9H8vMk1t;Vs50cPD$2wBN6A}b z8Y*|7JivDVYP4bRjR)M+OaNoS)N&>|hY{7*7{xQ~c2>k256x!!1PsFPrtr`+>TYie z;`5NEgX8%MTd%02^*T8SfQ%>jaY6xl5w&Uqn9Xnm`5sr^cr=^l z^D?W|u^kj?(YxilE%q$G4geOFE1yue`rgH0+(kQ^@3q5S{>()NoZP_H+k(-b$*V>0 z{;c;^q?MxTSq#>=GiaT#S}+xYzNk)$%va<`#Fmtu9xEaYG`)xmuyRm#1`=2z=#Zgz zcweA5X&%}kh{P+I^O$Tf(K@j*MAWG&`4C5Z)rpFsxcx2#M=OUTX+w*It>47ZG>&MUSct21HVBGbDD6rT3?gu|sILc=9Uizj)20 z@@*WTs1M4`Z@jIFk7`twC=3rJKkL#d>N|@_rh(6khE*vL@3p>MbXq3Ugae;puN+Kc zkJ@F1hpf}>yEd$zyI0o|9AVW?CA}dY96ZF(@m~2J~fyH4467+0EIZ)|8M?bGAu-`MN5t28x-`lnFujuECy+f)Lr5 z3Vw;dR%$_9JLnhMGxIl@-FM1Lh%EOh#eXLmW!BjV|$YWwO zWD?0IH^2`?iLxnI=QxaejZb7J!PbE|ixkWcc!#b9lis`{{Xvg(uUr%EB^A&~PzrV9$GoP$*`L5s z49GPmkSC*vz;r`+2)hgKkEX-*j;Aw=1y%-=P6bqLqv(avkQ$&V!DP1Ua4^3}wG))7 z{=5^s7`)`Xj<7ZgVbp1ig*0X?>MqvBTGr6Y#Ykm39@-4A?bnvwtmwO@`yyUB(-xze z@l|43RpWRM|AUPTnxv!w5~HkdT&}-)jyMMhb`V67fvp;hA2O)$J_ZWnLWP(!?2aTh zHr$8t)lCj~P#f|=ID`m%0@q|=&&Pqpn4QiuMtl1qY%XX9(t;5mab|PEiOIit1%I`} zdb)RFZ^SRCywztH!WgwRict1g#2M`jA&7M(07pdt;E6Qsx!4vl^en646S_Oj7#)>aacU1@;u2Vchy=xcCPED~ zAHD>0x$~_0qg@%7dUrJHG|wc2ye9e(6YPJIjW`_OgMusDApxS;f{qdvccvReiP=p; z-4czXMGRAqp9qC>-RQCw!%x)ph4!^2eg#8A%Gd^_+S#M7=;Ph0_ zxH0#O&_8${K$dun?&?dvt9k|;(otI1bp)cxDUo7B)X2ME>5I7Nyk)>Mm?ab$#f8Q^ zfWe3I>O5t{Xs{XDT2KSky)@dk+JJ_d*Np4$+?oYM?U+mhSuLS4tNnMN1v^r%y2TIR z`}m;n?2$c#5E>bo?4_oGC?}HEF3$25s{r`{N73*LPw46w{t$Cq)@5mVx+6 zPVUB*odX1WU^pfgxA`tPw+6%!Q+4h^vDsgDf_MLwP3vNu$ z(ov$!gUQI@Z)lBGNU00=uQg9+R2CtWgu)bl1u0CqxBlMw9X1x%fs21F5kK^NySIrP zbqUsbA*PtRw`NarS^m6#G`ciC^hBeDgB2o3y2jk7Qx@sAn0IL$adY--B+D?I(CW|uY zrLf*qc-*QD!z&>hIB$`4>>-hUkVQy{6*1zbj1MX#3|BAmZ!ed4Ru4EP9%FK2o&s97 z$^we~wx{J-fENuwIh$QSwQN1(VXU$*IDMf6EsllpGISeI&pK~qL$%K*w}W^cL&%L^ zA3z33poK%~1;CL)qbu;Mv}$#q z3LJ#}jRVf`M56S#S9pdZczeunxlBXF;V?PTO<&qiM_DHXnHljcZH7R>GW3(WckrK} zae;+da|3!-J>EMQ4A~!REFfF1$O>6sOeG7Gc)iy+Opf?t2s!i{NJ&<)_TI)U?juxx z6uI7)G=q`M%3A;%y&Z;ASyt)^qS*ryzLYa~jAj4U7}C*06%al~^O%1XN2JPtXN;n1 zt803UogPEt?~8)DgOuwZ#etX(-S1~6d_(xYNI!mbs#OZFe~dAn5H6hj3J-RSGX0G~ zbP>c{><-Gbuv+3eBjE9-l<&qclZ8hJX~3@=ZLsLR2=r>3CNvJV zn!)l5)TIVNcC5=#qn26=z=tTrgA7B-v$9~&6@=oM188kSE8$;CsT6P<3VQTrr2{Z2 z-*_|}fE^bNYWi9iXty&_i4CwKC*!>ge4)tzzonP-s5Hf<{?F)>vv$A&)PMsY zJaGWh>Zy7+^6#7ebId>l?9sBpJw=lLIBEaa`$dxI%-d*wHhSLGeGbVOcz9%jg|c;O zX1{4!HEIC7ka*DMJdWm}sE#?b8V$zhAA7cq97$q`H>=WSY-v3O!lHi=Igs6B1U6C) z@9A-^fY5(EXWRAr;Zjd93PNf1$99CN@Yi3P z+kuwn662+M7WD{jRYM!)U&1wO4}GR@>Px)H1(=P9;ixHuXDqRUD!dW86|l76^WrZU zO>zBKn@rUdJ1MJ8Zoxm-J_l4<5dj|UmyXfT&qUT4PjL0b-tSJbxcrNK$g>YCp^t39 z7X&x#h?B>8UdkL|->Lug4nQYj!DKa{;DT^6^Pgv8(WCC1!(}h4?(-o0*N?!k2WbMo=ME^3 z{b%YuBXmcP2;CydwEv$s{lEUBj{PHALnkl#zn|~(^^qd&1mLj$M5g}ze4nvLQsq}3 zblLxneD47hSb34xR`&mmBAo^T#xO1BWFb%Hh&l~Me_S7_AV3ZS z7;<25SZ;DKOJ>d4j>Z$=z`tsgt5AtzPOj2z3lK&RY=%dD$tTy(#U}e5_5C(-s;m@= z2fyJ=I%pu@2{|v965};Q7j_e>S!|yglc9{BjA4qwN7d?rk$%$Lf|U3AYn`ZOuN8W* zaZHh^p^3j=0(t^`$1-x|)+?XjFo$nH{aW-oI6pAr|3B=#Wn5HW^gb$xNFxnH!_eKO zfOMBgNH>V2D2RlB62cHemvkvgDjiCvFd!u%DIn4E^>^>@^M7^kyZcU=nRE8p zXYaMwde*a^wRqrjU>g3*@95Sq@_|DRy<*-tRnISuYlt;xiXnR}_P%U&MKp#R0T^<0 zs05+S)`SM-9xLs!6o%UJ$UjJ?M>0%exSIT&s#~{lat(-LNQ{f`jaGB1|fax zN9AlVC7v=6tLx1Z2AqPo!o>;RLL$>%IlO|Bz~%KEv$#HQ{OHSFBd7QAb$E;t!O@8J zT2BA*a+`duk+FM%228P#-$Mo?1f#!sj7F#yCAni8@#p`=e5g=7H(8b~p%Sk)4r5c_ zy)Qb?75ZZYnq&_O>9Y#6xvD00xCv8R_BwST4!5~sbQwxL0LV*wll{mmu~gddcyQ zH)Y)*X^^hgv8dNz#qq}VB5O5j_4{glBjk1HFaHbtWr)V2n7eUz6@(rhNL5;l2N=I|+q}}DD z{m!fHIG~dKwLCeATBpl?qFP>{MsRIF4LV#Xn9plXBUabLN-@I~QXcI={5eu$oi+TW zDxw!7n0ip9LPrc<%Gqslr$RNGa3|Cvoa+O*gL;(s67$7G{vPYKfX0uNSkhCGf5xxI zkAevCp>Vi82aO=7xZj@KHx5)L?9(gy#d{W!5w{FrhE19@3=2rqmI~t#7gPYj3HRtOshiIBX{9MHgcxDen=ECG_gEw&FxN(X(QYM3F_Hg zTTzNY*qiG(`GLY%gK*w!u9uU9Z_4GgR zgfz`lZ)C>eoUMB{{er7<6Qs5`?HM(_KbU?>DVIGI<$p5o+3>?elG?`g>48G+NStKR zr;FDBUJZ7tw9x|T{*Ke5KU=jKGs4B?p>jm8z^g5sd*7eV{Cf4AQL?j}jh^-k@=2?K z9LVW-#_71(=@&C)V)2D&S?=m@s1l>01%UClG*5eem~pm}fh@ZA0fafEK8Zg-1aE=P z*{m##=ld?_AIhCP>*Ir-!%sb02^2fU_&*_K8vr%tNbK_yykwww?yJ&22=qK2Z!hzl zZ}Xg=ZJfV)E)KddHv)#?36P+U547fH9Jq_@o6zgN5TjUo^7;Ho%5f8XWP(KkV^Rz= z5kJ{%q2LH2?|un2I{kKCs#H4Bl6jc)kjUxD@8?9CD|2zoxQ~MApWhZT{;^&Mgx|ca zDZ6wbbzQkU%BYqWNpv%a52vvAF)?yOo%rRd>$0@R0~se>8DkOo;gP+Ofr1RQw#TP4 z=e{#8a1Ia9zMRq;D%~Xj7q@25Ay6KU7i4HTYmqt$!eQ()1*2UDo!tQvU5D;}7~(Jb z_&a8>pZIEFPAL1YZ4Z|EYAB?K4hc-|9fPR4SEo#$|MjcgTowwIn;@Q#7DkJGBDEb8 zwd~i7a^2OALGb-r!PZd$jo&}UH`R`u;eY#P3mUA%5?KJBpWw6lYbWFUren*Qt2igTiQ~YrY2PvG>Eyf* z<4TyO&l~_kPe7xCQ?UOsjGEsQLctSJe4o)RIu( z$Eh!f(lg`qb;lI5`M>r9qnH8}Ghe8sO#9+M+>w!fIdI5H;3(h&LoI3L=Hu^wx?WST zL3^b8LEVWnDvD28lp$)2ib7LxsH*AI>FJ#0ep%G%FaPrm8?hp)c~@!V(WW{dQC~ox z>cs#Lm4};J`w+hLLNtFXPdEb$|3ua7814b3y*C&9Acrw3*y>Y9u;kAt3|;EO@Xi;j zYrLUbeXN6F<`CZZA}_k>2R)L>I@KW;1`b$ba5!V;-(`4x`_D2m&&C{6S2!*eLxlU_ zbtN|v@Yy`<7yEwL8Nr4YK+t*v+N zXXr#r)Xq z)^CKvvI9gpGt~IWjO0iL748R{AY$5eO6DbIj`~OImxMk{DI0IweLI`?KZ)}03Ydcn zK6JfHm@cSiHHvJ|z@ovX!C+Th&5JT{7ipIPyW1WJJT3AV1@b5uhL^}Ci+rr4GyO{X z-vNXQEgb>eW3q4M_jE2^klPLO`<-N6x7`CDPr50-RvJgJK}Wi{^psR$T*!+O1LbhG z6jR3gZ8s>DTPF%f5nLfK7w$!@B9Us--WU%So1jkl3ENbs_6~RwV}`|C>Lf!{rDsuJ zlKf5Gm~wSjbc^y(sEHaqU?4+y6>GysFl@DzrDC+$f3C&W<$fH{NsHTx?-)%X?Wk~y z|7%NAFgQUMlzE4jRR$NAZnxddGl!xpnq7T|Evs83ZBF4BLYTxX4k2Dr{QpNndz^%aPN?A=>^ zm*=MT6RzRjdCe*rW zuCr4HSh>^e^{+mmJ(63cV2BkTV~y3(g@ksP2zM`uZg+B-w8~PlEra4$MwxykAfa*B zzS)VQBgvVpWFJ1ss9wEwF=0`3D3YDaaeYZGi=r6m zimoHUk*YP~yFXk>-41wwTkOaanP&_(qtm1u~~H z-`brsYDKd(y;r0rKwe)duqO~ta_QYCkpf%c@I2*A&XZxzeH{%rMLjc3~BU+mX}w~b74;(dqKd|$(wPY4W-xb|*) zcN8W!g{FvI9w{NJfK2X+IU@Dv54=%m);_(}P}-Ay*{Scb=^+nNI+|0&QSoKW>$4?S zd`rkq&kLxAf^PiKyBOm(Ca45Cry#NT^Z#!sn5C+I3BPLkxyQMy&KjEaQq zF08#7G_|pBa=Uy0?x(ZRrdOe z{|6C}OZ7X2Qc8^~qTi7kro}NlV1CwoL(OzFORk|=7Oo*}|Kg8h1;RpEC}78jep|g* zg-K|r_3)Wh0jben9}b{5<1CXHQA&P95K?b!Qb~+(Ud|LI_*F`Ch>p_5N~$5GOHvrp z-lV6L8Xi)6*rs3igy9>>UUuXL_tbalR|TOd!k0h&e=dL%t%F}aX{VM|pIJW&Mn&Zm zeOEHa$lt2%uWx4mKGtA+xmwle0?vZB&?MtzF1-Ec$#D=Kc;E=(&qZ(ZPmM~X>WYu2wTwo zE4;!KgeR#pN99T$qthV?M{G?yzNFtlxsO?yWgBQKU%0)L>M|m2XasFKouJBlMu0V)tybQHK_D1rD?29FFm2mT^ z-|18Qxt5J1dP7})e$_R!JqLTkO2K6E?mS|gllx2RJ&R?Qc!eQEZc33pZ~1P0e1pDtu_xSa}{3O5Zj z1TyIW{pZX{rR7w=^x;@$6GW{G#ADSay<@ywHyCO>!0Leh=nPSRtWL2o%6hN`g`Q1C zpD;iXcE>%2)E)5H7hb=4$G?1o8*+zN4o5W`kz7Y{uOpd=;;fA>GE2!TmSCuq1rm|K z4zn#BR67loYe=ZYR!$~ykKpdlr>s#`gT?G2Y*DrQPywWL-)T?L&u$pIZ-T za!$!hAO0~_z$hf$2Zp-H&*vKL#p!ShwU^+#`;fW_ZVP`g?tz6S)zHE^Xft)IxG!4h zq~Bgj4U2<6t}mKL{I&kP`xsGBM9dUnaz$Ovc&TpAJTH%21eDE_I>#@RJP}ge#sGVQ zYdJ>$Y*J|3^{y4Su;Ow2mdLm@k*7mBLBZ`{a!5>}$I3vsF(^^|p{8O|IZ6{ zBHP>D$Ui=bSh^WFEPZoc3oNjRBF^AJ`7ydTqrRLf0-ha7va2-`cvm0)MP?jWO08EA zumWG)w#%ofL<>Hn3m${fI}%q%T}S<7Z!gWlW%MCpDf*W{WhbDYfi|pnTB0`Tf+39L zFS}<|1QQko-Cs<)7tOnJ$i5Gi3%JVv0-ZbE4uTV#9dPVc{R0uf4HL!_l5LGZW2(YY z8Yr}Ot)ys(BZ(;b_RjQ^UIiez2qEJ*qAws?1KlD1U?j$`!TlnD{n>lS#K`+n%){%{%@ zNdPJ3#S1b=S`Ksk%EgNZr-`^SRJPSlxl#M$T?r^XE5gmUvc@!VYa;`k~E@7p)jYxyca1z>G*)FJa$qiCmaIfQdH$| zPioVUzEj79KEL+itFxsj03{Ui&{G_wYRT`5GSgE$9RBgfy%#%>`=EFyV=v7%D8f7Z z(ejUX6%g%}H|96&Y+&&cmp(*j^DZGcM>Z-rwl9pveWD~jI&VK97rJB!<#wY2XW=4f z6j+4#-VhS5-G$Y4FI%5A$?3y568~seEAL??c{&hmkYzyDeX$m*Iywn1qaC$=Yj5k=fJex1YC}HI$X*tj;{t04c=NNT+oT3FpuI{M8 zV3JIpY}$N61>XzkgQq`?BX4=zQW+6w^JjXK%gaA4vs!28$P_uoYB>!4i9|vF9EC%` zFA;3`{I2uDguYRKHb|%Ck>jv{LZG&W4s!>cTi;BAmT{*PLhqilf<1N&ss}#>gN%1C z%5|#E(+&Tbsacx{>0@5D`d$E;zJ_;{DffxyMi?qU2)bmtfjlxfvM}{QIJ(bE)hba= zq4jl8@Vm+eJyJX&xLXNyn;}myKo3EHyAp~s3spaPAvcmVQEyAd(3OOE5&WZ!swHI? zP_cw#vKC|w6NAz$OF#+YO<5!&r zc{2~Nm+xUbL>ojR0^)aqIL#AlGXU%xj39I6OVTHEiZHK@{PW(yl<IdR)-L7*LhkGAJf-aYq8NU0oHkD$V%~!+7 z5g`l|!AkAX3AvbyzXLQT`a4L>#Er=={ZWFE=9BI_cv2^>Itt!+)UrRkWQL<+GJZ{d zPnp6{U@kBP;&wpDe1^%mcN+lRsTqXrF^V+GGn+!e-Yt~Vu3b!tpw3rpg8U~C6#WGv z!%TpV?DMa8(R;x=DNvX@W!5~1URzw-L?kwPMBUoYEKK68(bvmDiLo{f!v1P3j_jA_ zJ2_8NU@p&o#AwoLj%IPyu@+P*-dWfk8hW)asG{L%*YgCRiJUSq&um@ck>9{a z?=Nb{Mo2EK2@cYl;AqweU3iFRQYh;|Ib{+NXJeou*a>%@|D<1kXZoOV^hpTy>j-z~wH4+c4g5j|#S?zSPBe8q9u zNUXDa4SqY6^5GB5u^>G0Wuuv6(pV(&SfP{_$gjQwitJO^`Rn4cK_T-*L8ek+jCq>{ z=I306oIFs7u2?|}_#=-qFkd}IH0NT);fg@ZLOpYm$ptM#cb{~rnp&Eml1420uc(?K z4&&}I9+Jukk85LuH#hFTFS)hxyx~x8mcdS zKsob!7|^8rMP(AZi+BL{My3Nf>3WMH0+-rK%f=FPqO(*G#vP}(2@_-R5T*NBL;0bt zBiJKLeqnozxLvwT%-fA2tp{|>e87rpPv+Ok#s5NwkHB%BXkslw1!lwLyOTQO(bL%< z0jGeqP}StRV?gY*i}^nN&nP z8|#Mx&>RwpWE7^k_{>v2BX7-(EV~s=ED-u4Y1GiONcm<86{{Pfu}R&YZL%C;p|ecBA~lRZ z+A^g@Gr>!@uEWBK(~w;T34Vbx_(~+E7<%2#=VxgCptM<-%rTswyN|y$x`Kde^`~Qn zYQNlem=Y~>+ce)^yI|vr0v2JaRZF^3H92~R32m2foz)nFuNX6RUv{GGbach|6ocp5 z646nJ)QLsEBdC;rFRi+ibVz+w#ms$lUpl30S|p)QF-A@)c1tCAv1Xlm@0C<^2UW5z zFWyjcBLBlq*zrJ!r1j@aO+Dq#?AYMk3eyJkTQPOSsYUX6oL3yHq^a42IT-qT*AqI^ zLtb6g8aLEs{^zuc)y#}y(>N1g|8#LyLl~P4OIdI*C@sH|seX#zhf0ZJW4x|Nqz1*o@@dg|%qFl7s6VsZ-0wr@D$D4>92 zMt56?!SGrze!a4!%owiwi#cg@S^w`=~ z45sb5mcr?f3TWWna4IAJbVOqqJZc&LZ$D-!a`p6ZJdLxOVm&X*!3*bWC`ssLDTeRUIvj~t8+98%A`Su$Io7!-$M54H@Fr*Y5P2T$@b zzJOk-I%Iq3rU^MY$~do4n%@6Vl{`qd;df9MMjk@`YV$eE$sEra14^sf5solQZcZ-C zJo#e}vQnjXQZYvIG%InCY(bmf7;u>9nD8JGV8W?U9Y$2V+T(OqD*5cY{;)V2BC(Au zq{&u9#B$qGRliWgf z=bOIY+Sn2~^xM<-zdJ|X@}fZ4c6i!7&W>A=JpD2b4sBi_GH@I(UyP--A@~oZ;6RZK zEX@=iqgvr@v5)4g5Y{U~$@%E9DIrK?Xyr07p!U48ltt%3M3)=+q!znFj9XD&WIvuz z9E?xs;fh+0sOnf&HgwOG!vmf&>)Y#Cs9!%vVq3+hPR5{c(-@vLNiAemkm^l2>9l^B%SeH#m@`=kcy5TO9&g+#%5^CDAFKegJz~O7yR|0Q*Kn13q-Vb<4#izl897 zF_kp7>{d<)M3gxQeH2FEaSShh3I7!TFNl>h#{$>L-^?O$zc{}pFgnf4qt<}zH`Uk1 zV13Kh5kK-!2KteRPaCB--n@>X;OKoBJO?4IaXl z5gc$Zl|RW(a0PnT4rC&Y-r8zCr--U z%25;`-&h4ro#lkU_u5Ub@|23Ss{Z)tE7)TH?(&&O=m3@gc|NXnVJ{R@n-6bG7Q<+N z;?>9i(eH*&In*jU^5*?$_1P2rS*E&%4@}Jy$MeK?-dy00=7dNba;u2=&c!rGqMh(p z4L-2G3X8H+y&5wzOlzOY*qqXbjgrd5R9>cvNUSdtnenNBGUpfNet7bP>N445z=-4v zpjic7M3_q<&t?|TV&5u6Ab@C%7_;3Xf6~(TCxSmFF``xGzr?#%fx-a0M5~R3N%X?I zf9_6)N~CE17I45djkT^3U#`YUj6*{WmOd!oQJCaDg|@~+%1k|ir0b~sNxiEL@lGt* zpB>j4sCjm7$_>R`I#|V+s5ZQBF<-(iQXNSwsD2s0#Q94E_q&hUBtLXIRSqib5Ni)9 zn14ZY1{f&4ta4tNc0cyb#=A|c{qHX$Dsu+p{A`zj7G%HIh`BQwIa3jBqw#!*_)u=9 zZQ6HX>9-63EmvYHd841G51GNH4E9)<>64Ydgw8{oTbKbjIfg-b38(s_`FMoggBYs% zrA4XE7N#|20$}+KoizIsd1WRuvursV0kQf9S>dp49DvGoGRWAf3c2>^Ph6)2rN z+DP<8O8}<@fWAKfj`Jg78>Ax}_a_{Gyg?i#JjL&egXQF`BRNa^?LZDlLa1>iKh;Bb z~jkQ^Rr@$U-lyw2|azBP_R$027P*aXjEGY+x1A^8X zsEfJNkA>r{Zz#8NZ>fW{)7#&?vzzd`g>Uc$@tWaRY2^xuDcY*+pgd$7$YZS_RC)3; z1cf=dY#3h|7j)%n>;ChicjSTXF zsO~sDE;&C2H5)Z8coJ#}L*sTCr^s|U`_Rs-gZf+hWAmS|c~0h~&a9qj3U}WEtR|nq za8o&YidC?-e}P^j?!d_*GH5#KX*yW|WGGx0uNHvCHQ!RUymvKAMDFs40TvIHp>!Yh zo~mjXSP=j>;}l0Njuxs=8iwP$yMWaIw9Xnf_d?Y%mhI_v_J(xLn0oj{i-zWN09`du z#Ur_r8%xf_-+3mQv1r=pBNo1pGsPdTa2BH}#3y%uH#@eI?dzfso=@G+4?AsnEH`H{ zqn`9yVX$4pg>oFpKAND+u+rHiaaD1|<<83b=VT$h0(=)#uTZW(TalCp#X<2PpW|&0 z8lRkGGnA)GB#E5iI=328ACw@BWxK|CMjv70gKGSe=wmL*K_#dq|(b25GsH$I0Ce}gB7Mke-M`B@og={s`b~c zm19fgCB~RQRCxSNoK-f7|9IcOkHF>}IWVoP0nRBG)5>^TniO$(4FA~09Hs=s_~It> z0DZF+8F|kFC8!oYhox&4)58ua zBz{f{s-NTtx$=_N)8f+Igh!N<;P`t=#d&;}-U2w^^O63ucZNyK#q;|`UXl6O5%0y2 zI)x{B`0W&=F@V`jeAvpQt%}t8d*6EN$hiuQ$thPTMoxEljLP%p`%xVw$uf`Qh7MrW zya6pN4au+t#Z{Eko9KIM@ZvnjcQx&th;+eqO0|Z+%r`pJ1Nz^*l8rlGY%bT@kjyjH z?59c@U~^3!#Ir)Vm6LTDl8P-8QhI(tNv4+AdU*v)U?IK=NSsJob=Fy>~aqj(LLqZ&!Zq!H7nPALxA^W4>K>dn8D zgS{U$ycvcE-P`7W8bT(Lq<4|AOR@wr%I4?w;v%@)?C}AoC4Eww3$RKteS+mOfkXje zOw!ONCJhZ(z`b}yb_i(3tY<>yu^KpQn#>)DxqpW zmqhx17)H<}J+3vCb*ygwZ0he3e0!)W+|hLpvhh~5BlTt(3Yo#>0(k>o>^-LJOUgK4 z&Cq_Ap^_Rrbw0@P%?(lWo@}c!z&yMXTjfz-R+QIvOj^1A+qM|kBi)IgVITgGG7WUm zPE@LXbS(4)L_SwC^or&f@4ztijgJ(4<72YjtggwwPJw)lQ>>{_*LmQ7u%0h8qEK- zLoK5}Br;AKWUytkezhwC9ys$I;IY4QHaVt9>$l1ZW-D^FV8-l;4@O0GBd^bnRR3C* zA`>(p2b3{z-eAQ&pWXQm{HpZr99lJj{u`Swy{&vXdxy4Oc~MPMxD5A72n?0WTu#~$ zl6=Lb`C9bC?h(oTcIMERDf5+YTMxrqXiFdFIeOsngp*b3}yJt+iVUyDDwU<4sIwE24wJnB&zQePf>po@0#Y*eu7oHLbh zW)anO6~@#sKZLDmrm`V~vLuvO8<&i^?=>|4pRgJ@D;UJOf#DQXp|S;UF=rjXrfQ!@vx%tYIo zL*3N!c=|@??WZp+exc<0zn@R*p$|`eF*63mi>xgJ|Zr|Ln=MafusK_lH zx8nQ8fy#wv_r^iH+D32yshSb$1UlRutG(F9;MCGX^2Pthlw}AqIOcM40X7qGhJ8RD zaGuj}W%W&oBwCxUy8^TRps`&cRq zN2{bLz{&|CC6AX2ZZ5J-_94Z0{WgPWZyqhgA}w8F{Q?s*9Eq%qWMic<@enMpiCEFO zlK=2Y3$Dn6vNlpi@*Nckgfx$hiSW2}?4Z6EUR5RWTXfwRf_6#KE!K$5U5JAUx;~q` zo$}pPWI%^3@%TtDd^JTr%}Ou`qn*x{iij+2X#O6`}(LmYe2j_B`k$n#|6PLyGmQV09V~@n0v{A z0a;O%0_i)mt(W>IfTw<{>3@9ZCsLpo^c+reN6Kn80!p>J0&y=81$lbBQjod}yqp3D z!G}-!ghn}mJnu%sfpAIG!ykShuaRkKgC=36S|RL&!JEZ0s@!YT=k=UrOP>DqtiMwlxiyOrpzWsy&a+Bl{^{HBJ?do~9Sk!;*z%0SE+R+cuX|TM5b2rJzLz;9t;=#6ryA z*r&nbe(p?BU@q<9E4Cw=f-7EerW z>;ZAc-dKm^W01S%i;QEyTLy{mU;_wscNh_~$V%FPipmBRL0StMgn1@QPkKk0wS zsh$>`{hYh$#GP3c=Q%_fYHp|j;@5B(osw;LDI=HagOPq~!;+6HTvneNL79aSHiYJR zH?o$ZKP{c_#cR_C`RM*i@2-1`hrgcg;XTQR@@&w^;criKv-5LZ(BydYkZuy*I#Mz48cn+UdIy(k!)Y?C| zatz~}$Pg+MGuUVV3Vlj&IUfe&PW|^=n&tQvva|yRAQ0dVke~AbSzP#r>_;jf>YxU5 z(*Sa)qYdkibe=D8D2E{w5}m51^rmCijR_TZR`r{_)m~%I>}KKnDy1R=OL-pe!4ANT z2iIlI#=p9(%wiAYmw2q?C8jKc(7U`YZGi$7nIs8gK$=okR1+R`l zNs%0-N$$CpRA8);wPz}-BID0aW&y0QuR2yQT|nKBPJk}5g2}p$ob~CP6312k+Yu3k zXl!g@3xpNAjD4-2Noxcn1IZ)UXkagHub0XM5=T+&2_@QQ^EVQ+T>Conoa>jeA!WDP zm_mLUaq&G<5@HNde^95R8(n0L2?*?-rG&xp6K4vr0J=L}{3h&4N++M54Gym+^Hss!v>RxQ(4^%mZ2jLemu>(t;cPpF6I z(vE9J;0Wa)lG)*Ntv#KD*$FtRL-x{J~g?SE*2}-?NRe9#(4Z2tc z&l&h6@bhjJ$!R2Ozl+_+i|-}gvSziq&z7Z3sVbc4#re@*&&l;COdxQWfSVwa2|En? zAx?U3coFWWCt=VI!Cy}eoi2~vAx+@DoPI7Px$yM$fK=t}K&7fI`A%wv6`+7#1q(+j z8&uQ5Uuk(;N^cysA?UzE5wc9LS84HLES6$xfl}7&Y&)@t;@8?FVyaP$RKYvu=cC{` zGkfMRDdEH-matw1(-uO($5mpf{fB5Ui;TTuyRMu3C8p(mKHgz{gJWD--CmZFtSpTR z4JcEPE=6u?{(?z{nqqA{;-;K3blcoy&hbd$@b}gk{_qlOZLD_f{AhFfcv0r5y9hPZ zA${oc#P+}{L6lfGZrspdiaj0k;@!n2IaJ6GC|fU=sWL39wN(7`>kofT>wSz$dFT5y zcLOe#HH{>SUk9Bo9wv~`X^a`Qk|Hbn^a;tT0O2^xobkz`==AfuvvNudX3TSXX>8Cv zNHlZfVjpm^%+MoL1`LvH`AwBw7?ka-Gq;_udXhspgeALu zfi+q6!?038IU4q0(o@1xqloy$!*5kTy$~^+9c10(d_e-njZ!unuPlo4KyOYMpkqON zH@|)mb%WgA9#tAeMCcP%bhepW&xL6a!kRbDWsv;Z)D8342R=apyFM&+ZK#- z?X<6R5=0?Rhs>)gsxy#-#Q9f8y>|R^Kwgk}FGt5*Cr{2&r@cFu6MlQx;KhD_yIYTp zJ$1&F8JDJ0`;ro$MUL$Fkyctmx+sNYF%N=XUak3@DR|AYu!CM;OghwSc3eL6Z6({o zEk(j=wZ2<5dC4vUCSJ2Ie5k&S*!=WP?i8Ic3N5cQaI=_CAg5U#Qp=z4Xly)!5 z6$mu#D4gXxt!4B~(R}npD|Ls4=z= zZ|v=oxdH3d5I*YT$o1oL0rUc|O8SDsYai^_o(bSx%Srs$mI*@;=Mex;|>xT3YuoC-_6lZ$3?*JK}cMI+z>{KHCgIwJ}^RtN)yXxi2nE7`F2Nfa9=_<3obK4A#ckm>UK{umvL_QzU52x{2*>dZh3c6QKzXNr17L}3wA1e=b^D_B} zm5*E)mG$uZjawFrn`2x_KkS_p}yT5nc>EK|B%%5qy)wBr3vI$Yh9Pvd;| zR+UC@>~hPtzEGMaCl2cKq_kr!-A!;3v?Q<5wZ1`+AFd+1!tkIQfXcyN`O}U11Np*h zCUpd! zt~atJU%+Tq(3KU#fGnoMN>&C_$L!}Gzi5o0%w0VmTU5v1MFdyueFwp7o?pndKAOaH zx6Kz9n^M|T#gS00uS%j5e8w=vvFb<*tDLPN4z^7(d{5x9_BW>q>ITlc4ZS{Bl3a$& z(a+Sa1+jj7tS@DG-Z@sFaFc-bCWeDdVqvaQgiMSpg9qHS+j)icG~^aI6BGNP;gQ|m z4od@`5t%tH#KGJ0}OySE;S#LS?~W&`9p#R-|Lm&YJbnps!oeN&qqP0~FQsfL3iR#X)jYM&yk#Zi%%_LNh{K0{ zH#J~8;BZ+|a^L##wgz)z%r!=IJI}Yy#hv!cVo@j(e_{4(3Yk$pJf%Qz?{_R}?nXez zUrnB(5Kuxdhk>UOx?x~UckibzLFRO*6b~1ECyloQsCv_~cs-(BdU-CMUh3-NlNYmYG|<)WM1`M>`0jFlI^a9c3g5On7dddNSj%h9~$&~C*gx`qz($>r{3`1#b!=o)z;|>U$s(0 ztUrjM4D-74J~HqUoBL)4h@xjNz?x0tO)UcCeO8a}_9R~Vl)-qL7 z-SNi)kHg&WL3>=1FQ)ODGEmg{>qxiBnEQB!-P@K;ci1kdKC$;a+-o$rQQ6N1yZw0qwK2{&J2eKnd0 z7l%caBwgIGg3!L?SFyqCe{lW3`XA-g?T95M|YSiV3 zkmKw{e)=x=KG~n_Mv^qDY67PrNZpJAAykR=YAcW`XgXR--2iD@px*i&fPs3A0PRAS zvRQBxzaP2B9#7(tEM^>uzTQ6xiNEhb)sO~T)3NduO4=iDsm)84wXkKFx+VtBv|6S! zkf3$~7+XD1@DKs~gGIi>h6ag$o>>hy;88v^0`feRF;?I6Wmc3^D0c9zb${pk)H{`26Csel6jez} zsw|+OTsg;*M9$YY*|3;k)L9-M^7fyc4(}{jHAH96uz3C~s!emW1#SY7O1$`%9{cVy zQ6#w{9?8zw;Z+yNv0rf&6C6-$k5++EV>!d>1mbgSjdf&}(kWaQKdH?muivM#GT`c1 z6s5FlW%G0Z9i50jm{;nG#f^6}>i-qoAV@-PJ7qud%Ys~kYXKljXS;&PYdPB#BJ^L* zEz^(mVsH+Yg44>4N1S91dZoc6u`2snqi!bDgY^^+nvEXm>1Hnc&jcqIWR`QQNnppnS+ukN7)P_ zbAM4u;mIPO9lGwyfLrg6;+yR6EgB=F(0wL`Ugi6M)fCluK-+YYY5l#ix{%1HwPa%U zC4c}*#pMPzg?=x>nPWXR)kKrW6owChIF70JO5V(2yNKWMPzGF!KgfWEraM!Xa!Vp- z4RQOp*O)z`ZvH{#psh_&rMuy;WZA^QF~htbH=NHl34_|Vuf|Ux!Ch|UJdpT0L6&hW zi8LJ{Wm5#M>ixT8u)s?x@>08YSsa`GECL!BR!r}_KmBwAB(ImZdw9OXY0eKjzS@5P zH8g%eI~*x@KtYwQXq6!Xu~mF}EqYKQEntayPptXGq%{1_;rh=Uvm3a%j^I0CIm@Xs1@fwT-jxK{^L)s z=~c)_7S(|K5O>W2oJjNL)=2FK$B0c#xU-ltGSeBRLZIGz2=b zjdzhU5L|o4A+hkG*@P_eLV}&n z1|6P{-Q%T_E!N=~Ju#DlKZyUeC7?id{QQwI-%l_b*UGS%#uo&WG}_!fw5EvzkuR2}-_^8Sn7@Mlbx zw9UaNzqsYBY*T>Ot}5$YEz?YFyM_Y>d zXMl7vk7UOHd6p2lZ6lz6a|}0Jl>*F*Lct>5vrYW-&EY$mOm6|{`5ChA!+)E_e;c_H zka8hFUja?wWpcksP~u0Ab2}6DOZ?}5*{UdFx67kNsVQU`e<=FhxGg{xtNzgcSCxMc z)s`rzJLTGtg*JeA^E7?tC><%%qZ{c$b8Cq&>PYi^MQTg31IWX2m~LVDCq!C_FJ}fJ z@oAviGbK5@3;JB`z2Nc5Qe(ZYiR`PF=DR;WO!x%)Rp|f#)OJ7?eD~TGQc~#VAX1&; zXmtpPPH4RT0hG7buuV4ukwh;ILUtFXdL-PbHiDF!2u8xVgvDnq{)hO1g9y60++@A7 zHRr#cAch(_IIy4JivD!Lv?<0zc^sw>Nl^~Sj45AeQD{6m4R1OOH#dzqfdKW9R5Pe&rJBKa-eL=egpV^n^z50_w&c?6==3p zI}F4mkGeEw7W$@>HhxEB!dk3IU-|fObsr4Borypb&*=;h&QerCjgy}sBJ@Fe zP6o0Ft?M~_*<-tlrH-j#8RT=KuQ2&`&Ag4JVRGuuGipp<2vUh4%(J0Q`$xuv4ZHLGQ--@na(bu6fF@d{%E`E1r zfTYaKBThJWt1a)}kto`sE(CQ2QOu)+6rfn=l^)N9#So?oH1?ym*x0FY*B>x&hkYe%PHZ|^48oT@G}-VT1>`|LH+KW}sdB4z?e?+D zrGcD!{McrAo~npnlI-xUOE39|p?N|SgRX9NjKOFji7rw>BYGG*qTCJV%mrDvc0AE( zMJrm}@{j7>giM0&p&q*hbKo42@|Wr_Xnea^T+u%P$Rl5V`4C>W7NNovI^(A%@))xS zag!Y;q_rNUx%8uflVrt`f{{$^pgym~>|u+W>IL_Q zwsaa^{mDjSXQ(=#qtw4JNbb6H^T)px02rJys7^6P&+!|WejB;wPb|$$Z|jwu?U%Da z4-{@9ot%$q9Z#Q5-n!G|KA$Yip|AIkXJ*g~{K4>}m8$*UE7_nFP4BUE3zfD!hyJwP zBBPr!*l`maqEXxekq-Y?ZPy)-Rrt3{j}XeLcqp0K*(8y@6Ed?m$)4Fn#AEN7y_1!Z znZ2_kkyWy1Ht&7%X#C#a=kxyc`uOAV9Os<-oO^x8bzNUu2m3&lRW|YHzlKmi6c|E% zeET3e09^nx_8gkN6Z^+N40*!H5~g?&dG}zRsvW`_(YY@49Si3F)pEcQ+5PzBl&}FH zkMvThs@M6$BFCqrmCTt*20A!8@<^8Xmf!#7d#S8}CSxAsmY<@6RnlR`uZLI#^2~V0 z2k7D%<;NNCoec$a26JHJG#?#4aXt0-9-*+siZ!Ak5Egmri-EjwruLnf7H&2Z8F(P4 zd4YB58Po;p6Mz^2=WgadF*!Y#38fKMP;Hnre;i5A^(e`NIW0u#omASobo{~A@?Soh zb{}w%7{|mWKm1ybwaIWQ0Ip!9a_6<@AyozBY`DlsdKFxj^_~hSCMVb=Zn?Zz(bK!no$9ID9TS0;Aa~c^Ih1m+B>toSz!grvwq0GcD~BU zk*em9o0NYf(m!hmp4Ecu$c}*(%C7lk{mn$4#7OMu*q-sl0rlUf;6hvk+(2xkI!_EK z3WVSWzmjZwuC=y5`hnKQ0}{7#i{f_bP5ehCqS}{Xl=Kqn{cBNA%vFnYeWXUE^TVT^ zF4sIg6Zom8#G~;x!Aq88LApBM9(rhOJV{iurkMnAakq>~_s-AbX!!=PnluDpOyLyVjYY459>jsZqWf@V#4` z*AxIRLlK5f4Y%c{E_`!+7iet4?`-B)+5bknU_jpHM>GW$zXHJnB!llx|$1b{rBOjN|zdWW~1Dh&YkPs`do> zu0apgC=ORiqp?e;DBVDH^I#LM%GL8cU`bT6H4E76e~zf>S`K@1wQ>d7bJP&_(K=Zs z6!~(LBs|2cZa?`k;X*~KZN2Ny#L{H3t@yJ*0QN1wLwPWOXR~E&FsG%H_qb}@MP#V@ ztmk-1P>vQir#|LcoZ4*KNf>Rg^%56=5!uH7GPP4ly5Lsci5>gBEshW6igZ2wLMBkQ zp?CGaQggh&D5f@W!C~RynYo8AqrL1O^yVu$PN@lD+_`g&~PBY##LqYBH#y_ zR)e}NB-U+YD-)ppFo412y$4hCF2MDcj~Ws?L?XUMwe&hqYy#LJBL#750i*!(7Qn^7 zdR!RiI0kH~(uRw~kh}rF3MFtZ7S`2Xy!d!8rj_^GSMMtm`r$G#!_;_-TGgHZ+hu)m z*+FO@OLa!xGflO!yTgFe7e2dW&PX*Tb4xrNaEdiKM^aT$2a6TX@86&vT4>0ouM0Z8h zAc4_-U-*0A+uqHD$OBFIvejGtQhPaoSkgfdFi*?WkwG1EZBEak?^mPWhJX=$ZDZis zv(rDnV&uQkMtLzGT44*cIjx1P{mgXqeDn!xW~uI^*qLeG1MF1eOfbB9ze1i$8ChU9 z`LY3FJMWnL$q_>6V*6blP~?1@=jSIq4WMj~AtD)(o&}jehl3Cn2gsbP=Z7;zL~YSr zYPSW2#CHIABKL3{*suWmk^_qZe4L}=PNo5mUbVF=Z&0aYo}(eH3e3QIwazF;Eo~{& zyU6$5EaY(^OkS8!f;$YSY4wx(&s$!i&i*3u2*30TS9mRfSTN-J*sVXEM1PD+|4SWE z5Wk`6_|peNt@tNcp-g&f8B)<>It~Cjl?*`s9mbu4y02$>5-mSqz=M|`eqi?LBum+) zE%|J}^RB#R8Iq(-za4$C6w#9v2Y`+OBDnI0(ZMV~${cS_Ppjwb3=_|}ex<50)FitT zxMOM6Z#;x-De40-GS1!9l{#G$nx(+Vz;`VHp@=bWYFhvCP!{2bd!e%P*(`AvYtw`; z?j6YcVa>9PlC*N|6^Fs7uK5dShXywK z&@MKcCb}Q*rKS>dv)T|P+p1Jwv9rqOZ7AhVvO!`=MTCKBXu!Nq`6(Ms#Sw-Mk272$ z-5c(I8Inyx?YB?nVY{P{OgSB&O*+>f{l61EOZT7lH1!GV3INJnlozC1 zroCKtuzCQypC(wzf4(sLx?OdRrpBE$N9Wba;@@GyOrLt50X$=?vpv?*9R<)a8!hbW z`d2GvF8ctypbc0QYusl{$(hMu)p@`;90^q3|6cCMXtDZ2ScUNH<>WJgAjbk;-l^Ry z4AB3U5^kW*k`v&~T^bD24}n6|Ua1Tb=!XoR046_4EOpz-L`=u8FKpEWq0C;Fh4;*P-csi{-DrGyFD1dQXovo?7jzzzKdNcV7 zv58DLzESAeC}MU1{a9I&i_D2=PowX;u1K{*%;C0KZ*lbvZ0fp+&*WTI!MPqv88qq; zkC8|{Ye(hC(To(a1MVXjU;df6URUU^MLAh-AnvcEz*#eY`G?1_;aYb^jGlOce{*iMqNGL4=<4X(JW<%WS5X}=5 z@61xVl=uFe(6h?9D2OMY0u~K#5Q^9Vv%*UAi`6yL17NCN!8U_TG$4=JbxtdL@h|@B zSKzWjdbDU&%E(y{sOt|swo*cN)4*(B2GVzP-YS%FLWpATgO47^mTw>CEye`qL^s9& zDCr!bLS z`Q#{VDc=Bt)AcOv+wV58i(5-95exrp)nUCA+*(wX{q)ankQoQ|s?7cMt1hSBygf?b z#V-_E{*S{X+4ve~r7v2Mu3TQ6mBz~%v@Q$z!JY1}DZ0i&J+Ma?-tCab(IfaQu(9n% z$%8{GKQij?^#7~55e-`p^mCCLi1hjMAN)pMvn3eN^+`vdq0K6cQ&jpMTRMo-lz&Pc zQj64)A$w|)O+Nz|x@LBaL#Ex_8VHuGaP9KwHDJftNHFVgXynl#cXnw1S_AOYe?3qf zQGzf4neZN%i63W;gz|R9-g0nOV@IkLZ)UXu6}zm5KY`&6c?T{Z_kk+aJ{U_zVxLSA zC69FH=JfeAq?VPdq=3BMIOVK764Gk&@iLJGi~1+Os%Wu5$%C z-FHENv39rl&RCTlz}Bl<0@i|-3rjL9$ty{>7xYCLQn5s2sx(PbcWNREM?sFnU>b)7 zgrR13RkjO{7YdU=CITqDpJXFa-qh}!hC>dc+Wy!Mkx_!q&Nw7$_X;HJh#S7*4DbLy z_3obF95<=$M}gqVJ+0nP1C$+!GnrZ0t-`mO>A#fLq0` z@C;P@zzQX-pnM~CCrIM{kb#B=j%Hx+&v&~Rgj5((#<5Bvjg*%&F!T5VJx6{-aw6F6Xb0-WVw7k-)~bL1jV z9e*ykF@B9g8qIlQ2xQ5inhuh2LTcT>^K~lr4ziNS0j@EweQzI+xtz#jg8S(sKvur@ zvkC~d*Q^3~{0b;hyITMt=zubhl4iTsfOaxgcm$W(9CQwbk{^eCfXT4}DDB~(`a`Qz z?DguesG0XbVT0WWKslNGZalI5#v##h+8?Sf3~B>Y=7K^ONrJ_BL3HHivk#7%wGL68 zIaq^B!GEp?IT&vODE#c+NnwpMW{t}}F?Z6Y4&loN3Vah$b`ip?abtFtj=C1^SzEGc zNeQNiFd_9@88mU~;-PwQlQ4BlPXcSgMpRa!)`&*#Kr0tb_q@6-UbbEzxAnuHZti#9 zt!hq|@hz52IBRve9prBxjMQ;c2Z>S_R&QL2Op96oL}GqWO!6}CT0PLQ3#P0BwQLJU zP-KDI%GSf}owu>)vODz>vNb{U!h9ESgY~X%e($|L4kvZn)k!g1XZtZ1t<{6C+5wiB za_hMcqh!}8Iv!AmS{4Mqfe$U&5TRB8wk)%#UfQe#-93=$j*aNy-IOk6Txg>Rj+h1G zs(o%7c0 zg(28c4O%@LWK9IF6_2i8}TbNud72gY9sk=!g?b>FQpP zE_MYIh8M&)b|&3V0IQBHsK;E>WnIL+s=PSGQ!yWl|*TmyKdT;L7f`2nqtadOfI0s+?;@hq3e%B(qa zUO-ZYAH=_dcX$l=ronvZhuz{Sk~eiy(IZ& z)s#dt#B$?(x*Jq&*`~2BMve5HdTN*1L>|JzBA`Y0HNGQNrX5Gba{v#WEK5gB7n zfESSyoN)D*T|no1(`gnrCBOOdJ?F)x51T(h6m6iPYM0i8O&AtiOl{NNVco{jEIj>I z&oqr8b}BW2;o~*_QMyO+33CPZ;|23Tc#yIaog(dBQ!C0DU?z3{rm!4L&~gn4yvw=qx2?TC&wh5T2`h6I5gm$j{P9;eD- z8HLVY4Q_+iO1NOe_)Sk)5t1e%_6$k^f*nSmYLiT>c5F}1673hK)f7vZDdk!^Ho zRL|pWwyUHz0jD;u1l4P9zv8J!gS#OVt)%X}i#kQ@XsrT-cM*OtolsxORL5}zGM>Cz zGT&^PAm7JuBR3l%62u0Tpcqb;)xA6ZCgkb9;jF?Gu4xw}nn)^XuJ$>SCho!eO2xYv zSoPydZG`f}^?B!>4(`S%zn@BvVXq*LRN4wZdRiUqLxWibLU6QXw6xbZ#4r02FAkhY)Yk{6hI<0v&RrrGFm37+jC^2(kJ(cl*MkYCt;*Aa-E3RqGH=6C1uA)X64;rQK zGJeL8=}}{~2M&3$JfVRphnQjY^1IIo=M1!n57@@B9Yk*fID}CURvlnAlDyGJv;?8j zMOr7|FX~Gr;wR<{>b^S|eej*}c1C>aZ4v1nNAgZeev()b@Z?|9!`dEZg_YnjyC=z- zuV}pO98Ha(He+b0nZDvpTYICoCmnB`hX<6FfG zcXiwE?|rx*%8;jAsKjLp&`JCF7CprBMClCkAk!%y(-;`S$By&z!+C6$bRPe3I7&0( zi_uv3haOA$Z+4BHA4NMN{H{j#%(N`}wZn)$NXzl+-Am$oSJx*mFONNIw@XVU-fwR> z08~-5BfdOS$9pQY&mHVfOdiH?Nki==!aAnB?o!s5hN*WwIpJY-lJxHI-dw6)(jt+| z9M{E?0vS*($kL>LXSPDlo8xZbA089*I^mECNLW`F6vtycuc`SI_rC7ItxqAG<|NVq}MQ@#dFe^{?UfR zXqO|zOzmr7N27yezCX!J=xZGROc;nDzz^kA=1iz60DFQ^H*N`JN#{9}f9*>3 z9&h0q6kU>cLZYNan}Ifxmfh=7qkn5B44+cif^bo+@lf-EdXDs8dXr3Mdb) zAvT0O{JD>!j;Sc3{}V%t>pZVMJ{l#8#Wb$#%mnt08BLNfADccjP7Cbywmzx+S5ngQ zhFg^CG{m9Ux50WJYbi9bK$}leiaA$cyze|TIru6J9Uk8!%q)5DeiOOG#FG^1Sv<)H zY?7Mr2{n!s3hug05f`229Y6H61j<6T)tOLOm-$s&yz}}5Y*go$&DB{OB(u9W`aFm5 zwAgI~$h`7t=qwDphE&ps6&iKP{>WKrKZn$Ozn7S2{MzdPCtjRL?S-F?sHof(e%(ZU zv;hG>8WecHFe#6(&U!?d+z%ly8Ub=KU;1^wART#OOOwZeX=^Ws`yO`Su={@GntsGN z*8DgqJ*$%&wzL@6V4zuHjWM~0y#bAe+S zMXlzuC)?4VB4^tv>tj?t(d11D>|QQZ`$QDx zBfs9^utw2^UMTF2b6?FiO=R47WF9wC=;}K4#MnX0J3s0m`)y%!c59kf9^&K*gde`m zXL@JB5S|2)a$NzUiLL5t+_!Q}2dDd!k92ZMuW%S@?)Y}9y7%!$#><6m8LjF+y(;QN z71))L#eJ>9?dZAlj_*p}=v-ghWMFE$!iM=L7Sr|+yTxHuFH_nZ3@~iOlWDz(1S$%z z4Vv$Y7~IyI;%QVd=J`_3<*!!%vCpKhfTeiLWw_w9J{Wb#xn99(<{QH;Pb*59Ibb-S z1#7Wo6+j|LMj_5rTtk}BIU!%V1W&{5G1s&W0NQ@c_=!!8N{w zz7S!an8&_v+g_v=XUU>G^e%oD@&Yf!q#y0l-Xd*K4t7M~NcTo)pnVR{Qwex6hpE2q zwLL6?)E`1W&yyP}|4#76+ytPM_@v&d$X^~=Jq%BqJv&M#fDoUrxdu*J3p9UaUt(D| zKF1Oh>i9@*Isa|Q_fFc@h^J19kPY6WpMgl_3;Sn`fK)^SCscF7!*78 znJN@zVCNO3n3Qb&`uJ;G@QW(WDr?2agRn5|(bp&Tzj{gf;`!Edh{*+5xGv7UZ;#pb zD$NJR^%>qON>iRkv)F6NRf9)cEbab$elK+KnH8#RbwCyaePUaZK9nn8UMV|iyLS#(z)}9Jr)x&TetxvtI zs(YSk4^cqA+DmJj_3RMo%0 zf3(xURQcGQ=AzZJu~G@0Hc(8HXNum!sU)`E7n^wZ(c}|*e=}=6j@l}?(PnW65-0|Q z`0=C_j@HX}f5o?AmSGi?Qfnm%Ch}#o1zJm&ggDK|xj#oM9pg)D1EKpzMp>rq(Ze}9 zjr4mzsbi>jg)B}R#Jmo!gMV@cM*=7=N&dq}$@d)IOfI-3(_dT<_(CLo!-%atd4Eak zb9Tw5-FC*d#$1dG!Zp^_&gf)X?W_!)B!+FwbcEa5U-+dzkyrFC0;UTIT4W|}7x&v^t=`sQ1MMgVaQ zZ}{F7tM@t!XgVc|hSn_H+8~7VT;5v|t-!f>0=Y}h8PNWf0yy-q=d}5@w2@W*blv@rO9RXss2)*bXO>umfK#&0r2ymQ#R7X>K-5Nh4p3ENuaWHRy%Fh?k~{m zmCaLAxmMp0>_G6fm~)=MZ~J=4I=Ku-q~7tk(|EF8lNh5|9&*|t^)UQT_Ff~y{b#+l zOA2ebBm=N&y6vY@XmrVy*T z+8Vn$k`~K>zbpFS&qj>nsVV?6BlLWkVMw#F3+udOTQgA+FJ+J^g_Rw5EbG!n~m zKWsqgh+a(B2ZR+lYEAYw=5i4&EAKd-4(DstCB>?h8sm1CT22W@%E>01*fBAKKmCLES-w)|z>kE;L*X}q+Ft(y56LTu literal 0 HcmV?d00001 From 44edf320a004d1027630e3bbae21c298430725af Mon Sep 17 00:00:00 2001 From: Alex Holliday Date: Tue, 15 Sep 2026 10:13:07 -0700 Subject: [PATCH 35/35] reset job state on success --- client/src/Pages/Logs/index.tsx | 2 +- .../src/domain/jobs/job.repository.mongo.ts | 4 ++ .../test/integration/jobsRepository.test.ts | 53 ++++++++++++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/client/src/Pages/Logs/index.tsx b/client/src/Pages/Logs/index.tsx index 9d71e3253..b21e331c5 100644 --- a/client/src/Pages/Logs/index.tsx +++ b/client/src/Pages/Logs/index.tsx @@ -8,7 +8,7 @@ import { useTranslation } from "react-i18next"; const LogsPage = () => { const { t } = useTranslation(); - const [activeTab, setActiveTab] = useState(1); + const [activeTab, setActiveTab] = useState(0); return ( { it("does not release a lock owned by another worker (fenced no-op)", async () => { // A worker whose lease expired and was reclaimed must not clobber the new owner's lock. - await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7 }); + await seedJob({ lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, runCount: 7, failCount: 2, lastFailReason: "timeout" }); const ok = await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000); @@ -189,6 +189,55 @@ describe("MongoJobsRepository", () => { expect(row?.lockedBy).toBe("other-worker"); // lock untouched expect(row?.lockedUntil).toBe(NOW + LOCK_MS); expect(row?.runCount).toBe(7); // not double-counted + expect(row?.failCount).toBe(2); // failure state untouched + expect(row?.lastFailReason).toBe("timeout"); + }); + + it("clears the failure state so failCount counts consecutive failures", async () => { + await seedJob({ lockedBy: ownedBy(repo), lockedUntil: NOW + LOCK_MS, failCount: 3, lastFailReason: "connection refused" }); + + await repo.recordSuccess("check:mon-1", NOW, 60_000, NOW + 1_000); + + const row = await readRow("check:mon-1"); + expect(row?.failCount).toBe(0); + expect(row?.lastFailReason).toBeNull(); + }); + }); + + // ── recordOneShot (park after a single run) ───────────────────────────────── + describe("recordOneShot", () => { + it("parks the job, releases the lease, counts the run and clears the failure state", async () => { + await seedJob({ + intervalMs: null, + lockedBy: ownedBy(repo), + lockedUntil: NOW + LOCK_MS, + runCount: 1, + failCount: 2, + lastFailReason: "connection refused", + }); + + const ok = await repo.recordOneShot("check:mon-1", NOW + 1_000); + + expect(ok).toBe(true); + const row = await readRow("check:mon-1"); + expect(row?.nextScheduledAt).toBe(PARKED); + expect(row?.lockedBy).toBeNull(); + expect(row?.lockedUntil).toBeNull(); + expect(row?.lastFinishedAt).toBe(NOW + 1_000); + expect(row?.runCount).toBe(2); + expect(row?.failCount).toBe(0); + expect(row?.lastFailReason).toBeNull(); + }); + + it("does not touch a lock owned by another worker (fenced no-op)", async () => { + await seedJob({ intervalMs: null, lockedBy: "other-worker", lockedUntil: NOW + LOCK_MS, failCount: 2, lastFailReason: "timeout" }); + + expect(await repo.recordOneShot("check:mon-1", NOW + 1_000)).toBe(false); + const row = await readRow("check:mon-1"); + expect(row?.nextScheduledAt).toBe(NOW); // not parked + expect(row?.lockedBy).toBe("other-worker"); + expect(row?.failCount).toBe(2); + expect(row?.lastFailReason).toBe("timeout"); }); });