fix: handle IPs, avoid wrong parent-domain lookups, cache whois results
- Bare IP addresses and non-domain hosts (localhost, IPv6) now return a null expiry instead of throwing, so they no longer spam error logs. - Extract the registrable domain with tldts instead of walking up hostname labels, so a subdomain can never accidentally return its parent domain's expiry date. - Cache WHOIS lookups per registrable domain (1h positive, 10min negative) so visiting details pages does not re-query the registry every time. Addresses review feedback on #3834.
This commit is contained in:
1 parent
e5959ca91d
commit
75516a220d
7 files changed
+160
-53
No files matched your search
+4
-2
@@ -7191,10 +7191,12 @@
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"domain": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"expiryDate": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
}
|
||||
},
|
||||
"required": ["domain", "expiryDate"]
|
||||
|
||||
@@ -152,7 +152,10 @@ registry.registerPath({
|
||||
summary: "Get domain registration expiry for a monitor",
|
||||
security: bearer,
|
||||
request: { params: getDomainParamValidation },
|
||||
responses: { "200": okJson(z.object({ domain: z.string(), expiryDate: z.string() })), ...standardErrors },
|
||||
responses: {
|
||||
"200": okJson(z.object({ domain: z.string().nullable(), expiryDate: z.string().nullable() })),
|
||||
...standardErrors,
|
||||
},
|
||||
});
|
||||
|
||||
registry.registerPath({
|
||||
|
||||
Generated
+26
-25
@@ -40,6 +40,7 @@
|
||||
"sharp": "0.33.5",
|
||||
"ssl-checker": "2.0.10",
|
||||
"swagger-ui-express": "5.0.1",
|
||||
"tldts": "^7.4.10",
|
||||
"whoiser": "^1.18.0",
|
||||
"winston": "^3.13.0",
|
||||
"ws": "^8.19.0",
|
||||
@@ -9670,24 +9671,6 @@
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/isomorphic-dompurify/node_modules/tldts": {
|
||||
"version": "7.0.17",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.17.tgz",
|
||||
"integrity": "sha512-Y1KQBgDd/NUc+LfOtKS6mNsC9CCaH+m2P1RoIZy7RAPo3C3/t8X45+zgut31cRZtZ3xKPjfn3TkGTrctC2TQIQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tldts-core": "^7.0.17"
|
||||
},
|
||||
"bin": {
|
||||
"tldts": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/isomorphic-dompurify/node_modules/tldts-core": {
|
||||
"version": "7.0.17",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.17.tgz",
|
||||
"integrity": "sha512-DieYoGrP78PWKsrXr8MZwtQ7GLCUeLxihtjC1jZsW1DnvSMdKPitJSe8OSYDM2u5H6g3kWJZpePqkp43TfLh0g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/isomorphic-dompurify/node_modules/tough-cookie": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz",
|
||||
@@ -15258,21 +15241,21 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tldts": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz",
|
||||
"integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==",
|
||||
"version": "7.4.10",
|
||||
"resolved": "https://registry.npmmirror.com/tldts/-/tldts-7.4.10.tgz",
|
||||
"integrity": "sha512-GgouD1B+sWwvkaEq8vXC15DjQitxbvs12oIXELpconwm+Tg3zfcEv4jgzq3vtKverDXsg3VI8aRgNL2Nra0Iog==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tldts-core": "^6.1.86"
|
||||
"tldts-core": "^7.4.10"
|
||||
},
|
||||
"bin": {
|
||||
"tldts": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tldts-core": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz",
|
||||
"integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==",
|
||||
"version": "7.4.10",
|
||||
"resolved": "https://registry.npmmirror.com/tldts-core/-/tldts-core-7.4.10.tgz",
|
||||
"integrity": "sha512-KnQjp53ZekKgm/r3l+u8kJGGzYgrWdP8+Mql7a4vijh2WE0IrZWspQj/TpTxDho/YxO+AnOZnIjQcCD+q6iJsw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tmpl": {
|
||||
@@ -15326,6 +15309,24 @@
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/tough-cookie/node_modules/tldts": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmmirror.com/tldts/-/tldts-6.1.86.tgz",
|
||||
"integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tldts-core": "^6.1.86"
|
||||
},
|
||||
"bin": {
|
||||
"tldts": "bin/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/tough-cookie/node_modules/tldts-core": {
|
||||
"version": "6.1.86",
|
||||
"resolved": "https://registry.npmmirror.com/tldts-core/-/tldts-core-6.1.86.tgz",
|
||||
"integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz",
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
"sharp": "0.33.5",
|
||||
"ssl-checker": "2.0.10",
|
||||
"swagger-ui-express": "5.0.1",
|
||||
"tldts": "^7.4.10",
|
||||
"whoiser": "^1.18.0",
|
||||
"winston": "^3.13.0",
|
||||
"ws": "^8.19.0",
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Monitor, MonitorTypes, type MonitorType } from "@/domain/monitors/monit
|
||||
import { UserRole } from "@/domain/users/user.type.js";
|
||||
import sslChecker, { SSLDetails } from "ssl-checker";
|
||||
import * as whoiser from "whoiser";
|
||||
import { parse as parseDomain } from "tldts";
|
||||
type SSLCheckerType = typeof sslChecker;
|
||||
type WhoisModule = typeof whoiser;
|
||||
|
||||
@@ -17,8 +18,50 @@ export const fetchMonitorCertificate = async (checker: SSLCheckerType, monitor:
|
||||
};
|
||||
|
||||
const WHOIS_TIMEOUT_MS = 10_000;
|
||||
const DOMAIN_EXPIRY_CACHE_POSITIVE_TTL_MS = 60 * 60 * 1000; // 1 hour
|
||||
const DOMAIN_EXPIRY_CACHE_NEGATIVE_TTL_MS = 10 * 60 * 1000; // 10 minutes
|
||||
const EXPIRY_DATE_KEY_PATTERN = /(expiry date|expiration|paid-?till|registration expiration)/i;
|
||||
|
||||
export interface DomainExpiryResult {
|
||||
domain: string | null;
|
||||
expiryDate: string | null;
|
||||
}
|
||||
|
||||
interface DomainExpiryCacheEntry extends DomainExpiryResult {
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
interface DomainExpiryCache {
|
||||
get(domain: string): DomainExpiryResult | undefined;
|
||||
set(domain: string, result: DomainExpiryResult): void;
|
||||
}
|
||||
|
||||
export const createDomainExpiryCache = (positiveTtlMs: number, negativeTtlMs: number): DomainExpiryCache => {
|
||||
const entries = new Map<string, DomainExpiryCacheEntry>();
|
||||
|
||||
return {
|
||||
get(domain: string): DomainExpiryResult | undefined {
|
||||
const entry = entries.get(domain);
|
||||
if (entry === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (entry.expiresAt <= Date.now()) {
|
||||
entries.delete(domain);
|
||||
return undefined;
|
||||
}
|
||||
return { domain: entry.domain, expiryDate: entry.expiryDate };
|
||||
},
|
||||
set(domain: string, result: DomainExpiryResult): void {
|
||||
entries.set(domain, {
|
||||
...result,
|
||||
expiresAt: Date.now() + (result.expiryDate === null ? negativeTtlMs : positiveTtlMs),
|
||||
});
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const domainExpiryCache = createDomainExpiryCache(DOMAIN_EXPIRY_CACHE_POSITIVE_TTL_MS, DOMAIN_EXPIRY_CACHE_NEGATIVE_TTL_MS);
|
||||
|
||||
export const extractDomainExpiryDate = (whoisData: object): string | null => {
|
||||
const matchingKey = Object.keys(whoisData).find((key) => EXPIRY_DATE_KEY_PATTERN.test(key));
|
||||
if (!matchingKey) {
|
||||
@@ -54,24 +97,31 @@ const parseDomainExpiryValue = (raw: string): Date | null => {
|
||||
return null;
|
||||
};
|
||||
|
||||
export const fetchMonitorDomain = async (client: WhoisModule, monitor: Monitor): Promise<{ domain: string; expiryDate: string }> => {
|
||||
export const fetchMonitorDomain = async (
|
||||
client: WhoisModule,
|
||||
monitor: Monitor,
|
||||
cache: DomainExpiryCache = domainExpiryCache
|
||||
): Promise<DomainExpiryResult> => {
|
||||
const monitorUrl = new URL(monitor.url);
|
||||
const labels = monitorUrl.hostname.split(".");
|
||||
|
||||
// WHOIS registries only answer for the registrable domain, so try the full
|
||||
// hostname first and walk up to the parent domain (www.example.com -> example.com).
|
||||
for (let index = 0; index < labels.length - 1; index++) {
|
||||
const candidate = labels.slice(index).join(".");
|
||||
const result = await client.domain(candidate, { timeout: WHOIS_TIMEOUT_MS });
|
||||
const firstServerData = Object.values(result)[0];
|
||||
const expiryDate = extractDomainExpiryDate(
|
||||
typeof firstServerData === "object" && firstServerData !== null && !Array.isArray(firstServerData) ? firstServerData : {}
|
||||
);
|
||||
if (expiryDate) {
|
||||
return { domain: candidate, expiryDate };
|
||||
}
|
||||
const registrableDomain = parseDomain(monitorUrl.hostname).domain;
|
||||
if (registrableDomain === null) {
|
||||
// Bare IP addresses, localhost, and other non-domain hosts have no WHOIS expiry.
|
||||
return { domain: null, expiryDate: null };
|
||||
}
|
||||
throw new Error("Domain expiry not found");
|
||||
|
||||
const cached = cache.get(registrableDomain);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const result = await client.domain(registrableDomain, { timeout: WHOIS_TIMEOUT_MS });
|
||||
const firstServerData = Object.values(result)[0];
|
||||
const expiryDate = extractDomainExpiryDate(
|
||||
typeof firstServerData === "object" && firstServerData !== null && !Array.isArray(firstServerData) ? firstServerData : {}
|
||||
);
|
||||
const domainExpiry: DomainExpiryResult = { domain: registrableDomain, expiryDate };
|
||||
cache.set(registrableDomain, domainExpiry);
|
||||
return domainExpiry;
|
||||
};
|
||||
|
||||
export const requireString = (value: unknown, fieldName: string): string => {
|
||||
|
||||
@@ -82,7 +82,7 @@ class MonitorController implements IMonitorController {
|
||||
msg: "Domain expiry retrieved successfully",
|
||||
data: {
|
||||
domain: domainExpiry.domain,
|
||||
expiryDate: new Date(domainExpiry.expiryDate),
|
||||
expiryDate: domainExpiry.expiryDate === null ? null : new Date(domainExpiry.expiryDate),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "@jest/globals";
|
||||
import { extractDomainExpiryDate, fetchMonitorDomain } from "../../../src/api/controllers/controllerUtils.ts";
|
||||
import { createDomainExpiryCache, extractDomainExpiryDate, fetchMonitorDomain } from "../../../src/api/controllers/controllerUtils.ts";
|
||||
import type { Monitor } from "../../../src/domain/monitors/monitor.type.ts";
|
||||
|
||||
const makeMonitor = (url: string): Monitor =>
|
||||
@@ -27,26 +27,76 @@ describe("extractDomainExpiryDate", () => {
|
||||
});
|
||||
|
||||
describe("fetchMonitorDomain", () => {
|
||||
it("walks up from the hostname to the registrable domain", async () => {
|
||||
it("queries the registrable domain for subdomains", async () => {
|
||||
const queried: string[] = [];
|
||||
const client = {
|
||||
domain: async (domain: string) => {
|
||||
queried.push(domain);
|
||||
return domain === "example.com" ? { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } } : {};
|
||||
return { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } };
|
||||
},
|
||||
};
|
||||
const cache = createDomainExpiryCache(60_000, 10_000);
|
||||
|
||||
const result = await fetchMonitorDomain(client as never, makeMonitor("https://www.example.co.uk"), cache);
|
||||
|
||||
expect(queried).toEqual(["example.co.uk"]);
|
||||
expect(result).toEqual({ domain: "example.co.uk", expiryDate: "2027-08-13T04:00:00.000Z" });
|
||||
});
|
||||
|
||||
it("returns null expiry for bare IP addresses without querying whois", async () => {
|
||||
const client = {
|
||||
domain: async () => {
|
||||
throw new Error("should not be called");
|
||||
},
|
||||
};
|
||||
|
||||
const result = await fetchMonitorDomain(client as never, makeMonitor("https://www.example.com"));
|
||||
const result = await fetchMonitorDomain(client as never, makeMonitor("https://1.2.3.4"));
|
||||
|
||||
expect(queried).toEqual(["www.example.com", "example.com"]);
|
||||
expect(result).toEqual({ domain: "example.com", expiryDate: "2027-08-13T04:00:00.000Z" });
|
||||
expect(result).toEqual({ domain: null, expiryDate: null });
|
||||
});
|
||||
|
||||
it("throws when no candidate has an expiry date", async () => {
|
||||
it("returns null expiry when the registry has no expiry record", async () => {
|
||||
const client = {
|
||||
domain: async () => ({ "whois.verisign-grs.com": { "Domain Name": "example.com" } }),
|
||||
};
|
||||
|
||||
await expect(fetchMonitorDomain(client as never, makeMonitor("https://example.com"))).rejects.toThrow("Domain expiry not found");
|
||||
const result = await fetchMonitorDomain(client as never, makeMonitor("https://example.com"));
|
||||
|
||||
expect(result).toEqual({ domain: "example.com", expiryDate: null });
|
||||
});
|
||||
|
||||
it("serves repeat lookups from cache", async () => {
|
||||
let calls = 0;
|
||||
const client = {
|
||||
domain: async () => {
|
||||
calls++;
|
||||
return { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } };
|
||||
},
|
||||
};
|
||||
const cache = createDomainExpiryCache(60_000, 10_000);
|
||||
const monitor = makeMonitor("https://example.com");
|
||||
|
||||
await fetchMonitorDomain(client as never, monitor, cache);
|
||||
const second = await fetchMonitorDomain(client as never, monitor, cache);
|
||||
|
||||
expect(calls).toBe(1);
|
||||
expect(second).toEqual({ domain: "example.com", expiryDate: "2027-08-13T04:00:00.000Z" });
|
||||
});
|
||||
|
||||
it("re-queries after the cache expires", async () => {
|
||||
let calls = 0;
|
||||
const client = {
|
||||
domain: async () => {
|
||||
calls++;
|
||||
return { "whois.verisign-grs.com": { "Expiry Date": "2027-08-13T04:00:00Z" } };
|
||||
},
|
||||
};
|
||||
const cache = createDomainExpiryCache(-1, -1);
|
||||
const monitor = makeMonitor("https://example.com");
|
||||
|
||||
await fetchMonitorDomain(client as never, monitor, cache);
|
||||
await fetchMonitorDomain(client as never, monitor, cache);
|
||||
|
||||
expect(calls).toBe(2);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user