diff --git a/server/openapi.json b/server/openapi.json index 0a57093ea..e95617426 100644 --- a/server/openapi.json +++ b/server/openapi.json @@ -7191,10 +7191,12 @@ "type": "object", "properties": { "domain": { - "type": "string" + "type": "string", + "nullable": true }, "expiryDate": { - "type": "string" + "type": "string", + "nullable": true } }, "required": ["domain", "expiryDate"] diff --git a/server/openapi/routes/monitor.ts b/server/openapi/routes/monitor.ts index 02626d8ab..ddb207eb8 100644 --- a/server/openapi/routes/monitor.ts +++ b/server/openapi/routes/monitor.ts @@ -152,7 +152,10 @@ registry.registerPath({ summary: "Get domain registration expiry for a monitor", security: bearer, request: { params: getDomainParamValidation }, - responses: { "200": okJson(z.object({ domain: z.string(), expiryDate: z.string() })), ...standardErrors }, + responses: { + "200": okJson(z.object({ domain: z.string().nullable(), expiryDate: z.string().nullable() })), + ...standardErrors, + }, }); registry.registerPath({ diff --git a/server/package-lock.json b/server/package-lock.json index 548f99575..a581bd18b 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -40,6 +40,7 @@ "sharp": "0.33.5", "ssl-checker": "2.0.10", "swagger-ui-express": "5.0.1", + "tldts": "^7.4.10", "whoiser": "^1.18.0", "winston": "^3.13.0", "ws": "^8.19.0", @@ -9670,24 +9671,6 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, - "node_modules/isomorphic-dompurify/node_modules/tldts": { - "version": "7.0.17", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.17.tgz", - "integrity": "sha512-Y1KQBgDd/NUc+LfOtKS6mNsC9CCaH+m2P1RoIZy7RAPo3C3/t8X45+zgut31cRZtZ3xKPjfn3TkGTrctC2TQIQ==", - "license": "MIT", - "dependencies": { - "tldts-core": "^7.0.17" - }, - "bin": { - "tldts": "bin/cli.js" - } - }, - "node_modules/isomorphic-dompurify/node_modules/tldts-core": { - "version": "7.0.17", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.17.tgz", - "integrity": "sha512-DieYoGrP78PWKsrXr8MZwtQ7GLCUeLxihtjC1jZsW1DnvSMdKPitJSe8OSYDM2u5H6g3kWJZpePqkp43TfLh0g==", - "license": "MIT" - }, "node_modules/isomorphic-dompurify/node_modules/tough-cookie": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz", @@ -15258,21 +15241,21 @@ } }, "node_modules/tldts": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", - "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "version": "7.4.10", + "resolved": "https://registry.npmmirror.com/tldts/-/tldts-7.4.10.tgz", + "integrity": "sha512-GgouD1B+sWwvkaEq8vXC15DjQitxbvs12oIXELpconwm+Tg3zfcEv4jgzq3vtKverDXsg3VI8aRgNL2Nra0Iog==", "license": "MIT", "dependencies": { - "tldts-core": "^6.1.86" + "tldts-core": "^7.4.10" }, "bin": { "tldts": "bin/cli.js" } }, "node_modules/tldts-core": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", - "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "version": "7.4.10", + "resolved": "https://registry.npmmirror.com/tldts-core/-/tldts-core-7.4.10.tgz", + "integrity": "sha512-KnQjp53ZekKgm/r3l+u8kJGGzYgrWdP8+Mql7a4vijh2WE0IrZWspQj/TpTxDho/YxO+AnOZnIjQcCD+q6iJsw==", "license": "MIT" }, "node_modules/tmpl": { @@ -15326,6 +15309,24 @@ "node": ">=16" } }, + "node_modules/tough-cookie/node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmmirror.com/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tough-cookie/node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmmirror.com/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "license": "MIT" + }, "node_modules/tr46": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", diff --git a/server/package.json b/server/package.json index f30495c5c..a43871d78 100755 --- a/server/package.json +++ b/server/package.json @@ -63,6 +63,7 @@ "sharp": "0.33.5", "ssl-checker": "2.0.10", "swagger-ui-express": "5.0.1", + "tldts": "^7.4.10", "whoiser": "^1.18.0", "winston": "^3.13.0", "ws": "^8.19.0", diff --git a/server/src/api/controllers/controllerUtils.ts b/server/src/api/controllers/controllerUtils.ts index 7b014e656..608c39d57 100755 --- a/server/src/api/controllers/controllerUtils.ts +++ b/server/src/api/controllers/controllerUtils.ts @@ -3,6 +3,7 @@ import { Monitor, MonitorTypes, type MonitorType } from "@/domain/monitors/monit import { UserRole } from "@/domain/users/user.type.js"; import sslChecker, { SSLDetails } from "ssl-checker"; import * as whoiser from "whoiser"; +import { parse as parseDomain } from "tldts"; type SSLCheckerType = typeof sslChecker; type WhoisModule = typeof whoiser; @@ -17,8 +18,50 @@ export const fetchMonitorCertificate = async (checker: SSLCheckerType, monitor: }; const WHOIS_TIMEOUT_MS = 10_000; +const DOMAIN_EXPIRY_CACHE_POSITIVE_TTL_MS = 60 * 60 * 1000; // 1 hour +const DOMAIN_EXPIRY_CACHE_NEGATIVE_TTL_MS = 10 * 60 * 1000; // 10 minutes const EXPIRY_DATE_KEY_PATTERN = /(expiry date|expiration|paid-?till|registration expiration)/i; +export interface DomainExpiryResult { + domain: string | null; + expiryDate: string | null; +} + +interface DomainExpiryCacheEntry extends DomainExpiryResult { + expiresAt: number; +} + +interface DomainExpiryCache { + get(domain: string): DomainExpiryResult | undefined; + set(domain: string, result: DomainExpiryResult): void; +} + +export const createDomainExpiryCache = (positiveTtlMs: number, negativeTtlMs: number): DomainExpiryCache => { + const entries = new Map(); + + return { + get(domain: string): DomainExpiryResult | undefined { + const entry = entries.get(domain); + if (entry === undefined) { + return undefined; + } + if (entry.expiresAt <= Date.now()) { + entries.delete(domain); + return undefined; + } + return { domain: entry.domain, expiryDate: entry.expiryDate }; + }, + set(domain: string, result: DomainExpiryResult): void { + entries.set(domain, { + ...result, + expiresAt: Date.now() + (result.expiryDate === null ? negativeTtlMs : positiveTtlMs), + }); + }, + }; +}; + +const domainExpiryCache = createDomainExpiryCache(DOMAIN_EXPIRY_CACHE_POSITIVE_TTL_MS, DOMAIN_EXPIRY_CACHE_NEGATIVE_TTL_MS); + export const extractDomainExpiryDate = (whoisData: object): string | null => { const matchingKey = Object.keys(whoisData).find((key) => EXPIRY_DATE_KEY_PATTERN.test(key)); if (!matchingKey) { @@ -54,24 +97,31 @@ const parseDomainExpiryValue = (raw: string): Date | null => { return null; }; -export const fetchMonitorDomain = async (client: WhoisModule, monitor: Monitor): Promise<{ domain: string; expiryDate: string }> => { +export const fetchMonitorDomain = async ( + client: WhoisModule, + monitor: Monitor, + cache: DomainExpiryCache = domainExpiryCache +): Promise => { const monitorUrl = new URL(monitor.url); - const labels = monitorUrl.hostname.split("."); - - // WHOIS registries only answer for the registrable domain, so try the full - // hostname first and walk up to the parent domain (www.example.com -> example.com). - for (let index = 0; index < labels.length - 1; index++) { - const candidate = labels.slice(index).join("."); - const result = await client.domain(candidate, { timeout: WHOIS_TIMEOUT_MS }); - const firstServerData = Object.values(result)[0]; - const expiryDate = extractDomainExpiryDate( - typeof firstServerData === "object" && firstServerData !== null && !Array.isArray(firstServerData) ? firstServerData : {} - ); - if (expiryDate) { - return { domain: candidate, expiryDate }; - } + const registrableDomain = parseDomain(monitorUrl.hostname).domain; + if (registrableDomain === null) { + // Bare IP addresses, localhost, and other non-domain hosts have no WHOIS expiry. + return { domain: null, expiryDate: null }; } - throw new Error("Domain expiry not found"); + + const cached = cache.get(registrableDomain); + if (cached !== undefined) { + return cached; + } + + const result = await client.domain(registrableDomain, { timeout: WHOIS_TIMEOUT_MS }); + const firstServerData = Object.values(result)[0]; + const expiryDate = extractDomainExpiryDate( + typeof firstServerData === "object" && firstServerData !== null && !Array.isArray(firstServerData) ? firstServerData : {} + ); + const domainExpiry: DomainExpiryResult = { domain: registrableDomain, expiryDate }; + cache.set(registrableDomain, domainExpiry); + return domainExpiry; }; export const requireString = (value: unknown, fieldName: string): string => { diff --git a/server/src/api/controllers/monitorController.ts b/server/src/api/controllers/monitorController.ts index 4bd1a4fcb..f05e0df5c 100644 --- a/server/src/api/controllers/monitorController.ts +++ b/server/src/api/controllers/monitorController.ts @@ -82,7 +82,7 @@ class MonitorController implements IMonitorController { msg: "Domain expiry retrieved successfully", data: { domain: domainExpiry.domain, - expiryDate: new Date(domainExpiry.expiryDate), + expiryDate: domainExpiry.expiryDate === null ? null : new Date(domainExpiry.expiryDate), }, }); }); diff --git a/server/test/unit/utils/domainExpiry.test.ts b/server/test/unit/utils/domainExpiry.test.ts index 7d6e02e49..185a759ee 100644 --- a/server/test/unit/utils/domainExpiry.test.ts +++ b/server/test/unit/utils/domainExpiry.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "@jest/globals"; -import { extractDomainExpiryDate, fetchMonitorDomain } from "../../../src/api/controllers/controllerUtils.ts"; +import { createDomainExpiryCache, extractDomainExpiryDate, fetchMonitorDomain } from "../../../src/api/controllers/controllerUtils.ts"; import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts"; const makeMonitor = (url: string): Monitor => @@ -27,26 +27,76 @@ describe("extractDomainExpiryDate", () => { }); describe("fetchMonitorDomain", () => { - it("walks up from the hostname to the registrable domain", async () => { + it("queries the registrable domain for subdomains", async () => { const queried: string[] = []; const client = { domain: async (domain: string) => { queried.push(domain); - return domain === "example.com" ? { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } } : {}; + return { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } }; + }, + }; + const cache = createDomainExpiryCache(60_000, 10_000); + + const result = await fetchMonitorDomain(client as never, makeMonitor("https://www.example.co.uk"), cache); + + expect(queried).toEqual(["example.co.uk"]); + expect(result).toEqual({ domain: "example.co.uk", expiryDate: "2027-08-13T04:00:00.000Z" }); + }); + + it("returns null expiry for bare IP addresses without querying whois", async () => { + const client = { + domain: async () => { + throw new Error("should not be called"); }, }; - const result = await fetchMonitorDomain(client as never, makeMonitor("https://www.example.com")); + const result = await fetchMonitorDomain(client as never, makeMonitor("https://1.2.3.4")); - expect(queried).toEqual(["www.example.com", "example.com"]); - expect(result).toEqual({ domain: "example.com", expiryDate: "2027-08-13T04:00:00.000Z" }); + expect(result).toEqual({ domain: null, expiryDate: null }); }); - it("throws when no candidate has an expiry date", async () => { + it("returns null expiry when the registry has no expiry record", async () => { const client = { domain: async () => ({ "whois.verisign-grs.com": { "Domain Name": "example.com" } }), }; - await expect(fetchMonitorDomain(client as never, makeMonitor("https://example.com"))).rejects.toThrow("Domain expiry not found"); + const result = await fetchMonitorDomain(client as never, makeMonitor("https://example.com")); + + expect(result).toEqual({ domain: "example.com", expiryDate: null }); + }); + + it("serves repeat lookups from cache", async () => { + let calls = 0; + const client = { + domain: async () => { + calls++; + return { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } }; + }, + }; + const cache = createDomainExpiryCache(60_000, 10_000); + const monitor = makeMonitor("https://example.com"); + + await fetchMonitorDomain(client as never, monitor, cache); + const second = await fetchMonitorDomain(client as never, monitor, cache); + + expect(calls).toBe(1); + expect(second).toEqual({ domain: "example.com", expiryDate: "2027-08-13T04:00:00.000Z" }); + }); + + it("re-queries after the cache expires", async () => { + let calls = 0; + const client = { + domain: async () => { + calls++; + return { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } }; + }, + }; + const cache = createDomainExpiryCache(-1, -1); + const monitor = makeMonitor("https://example.com"); + + await fetchMonitorDomain(client as never, monitor, cache); + await fetchMonitorDomain(client as never, monitor, cache); + + expect(calls).toBe(2); }); });