import http from "node:http"; import https from "node:https"; import { analyzeHeaders, type HeaderMap } from "./headers"; import { createDeadline, remainingMs, timeoutError, withDeadline } from "./deadline"; import { InspectionInputError, InspectionPolicyError, InspectionResolutionError, normalizeHost, resolvePublicHostDetailed, validateResolvedAddresses } from "./policy"; import { CERTSCOPE_VERSION } from "../version"; const HEADER_NAMES = [ "strict-transport-security", "content-security-policy", "x-content-type-options", "referrer-policy", "x-frame-options", "server", "x-powered-by", ]; export const HTTP_TIMEOUT_MS = 5_000; export const HTTP_TOTAL_TIMEOUT_MS = 15_000; export const DEFAULT_MAX_REDIRECTS = 3; export const MAX_ADDRESS_ATTEMPTS = 6; export const HTTPS_DOWNGRADE_ERROR = "HTTPS redirect downgrade is not allowed."; export const REDIRECT_LOOP_ERROR = "Redirect loop detected."; export type HttpHop = Record; type Requester = (url: URL, deadline: number) => Promise; export type CandidateRequester = (url: URL, address: string, deadline: number) => Promise; export interface HttpRequestOptions { resolve?: typeof resolvePublicHostDetailed; request?: CandidateRequester; } function safeUrl(value: string | URL) { const text = typeof value === "string" ? value : value.toString(); try { const parsed = new URL(text); parsed.username = ""; parsed.password = ""; return parsed.toString(); } catch { return text.replace(/(\/\/)[^/\s@]+@/g, "$1[redacted]@"); } } function sanitizedHop(hop: HttpHop): HttpHop { return typeof hop.location === "string" ? { ...hop, location: safeUrl(hop.location) } : hop; } export function validateProbeUrl(url: URL) { if (!["http:", "https:"].includes(url.protocol)) throw new InspectionPolicyError("Redirect used an unsupported protocol."); if (url.username || url.password) throw new InspectionPolicyError("Redirect URLs with credentials are not allowed."); if (url.port && url.port !== (url.protocol === "https:" ? "443" : "80")) { throw new InspectionPolicyError("Redirect to a non-default port is not allowed."); } normalizeHost(url.hostname); } function errorCode(error: unknown) { return typeof error === "object" && error !== null && "code" in error ? String((error as { code?: unknown }).code ?? "") : ""; } function transportError(error: unknown) { if (error instanceof InspectionInputError || error instanceof InspectionPolicyError || error instanceof InspectionResolutionError) return error.message; if (errorCode(error) === "ETIMEOUT") return "HTTP request timed out."; const code = errorCode(error); return code ? "HTTP request failed (" + code + ")." : "HTTP request failed."; } function requestCandidate(url: URL, address: string, deadline: number): Promise { const budget = Math.min(HTTP_TIMEOUT_MS, remainingMs(deadline)); if (!budget) return Promise.reject(timeoutError("HTTP request timed out.")); const host = normalizeHost(url.hostname); const client = url.protocol === "https:" ? https : http; return new Promise((resolve, reject) => { let settled = false; // The request constructor can throw before a timer exists; the callback // therefore needs an intentionally late-bound timer reference. // eslint-disable-next-line prefer-const let timer: ReturnType | undefined; const finish = (callback: () => void) => { if (settled) return; settled = true; if (timer) clearTimeout(timer); callback(); }; let request: ReturnType; try { request = client.request({ hostname: address, port: url.protocol === "https:" ? 443 : 80, path: url.pathname + url.search, method: "GET", servername: url.protocol === "https:" ? host : undefined, rejectUnauthorized: url.protocol === "https:" ? true : undefined, headers: { host, "user-agent": "CertScope/" + CERTSCOPE_VERSION }, timeout: budget, }, (response) => { if (settled) { response.destroy(); return; } const headers = Object.fromEntries( HEADER_NAMES.map((name) => [name, response.headers[name] ?? null]), ) as HeaderMap; finish(() => { response.destroy(); resolve({ url: url.toString(), address, status: response.statusCode ?? null, location: response.headers.location ?? null, headers, headerPosture: analyzeHeaders(headers), }); }); }); } catch (error) { finish(() => reject(error)); return; } const abortOnTimeout = () => finish(() => { request.destroy(); reject(timeoutError("HTTP request timed out.")); }); timer = setTimeout(abortOnTimeout, budget); request.once("timeout", abortOnTimeout); request.once("error", (error) => finish(() => reject(error))); request.end(); }); } export async function requestOnce(url: URL, deadline = createDeadline(HTTP_TIMEOUT_MS), options: HttpRequestOptions = {}): Promise { validateProbeUrl(url); const resolution = await (options.resolve ?? resolvePublicHostDetailed)(url.hostname, deadline); const addresses = validateResolvedAddresses(resolution.addresses); const request = options.request ?? requestCandidate; const attempts: Array> = []; for (const address of addresses.slice(0, MAX_ADDRESS_ATTEMPTS)) { if (!remainingMs(deadline)) break; try { const result = await request(url, address, deadline); return { ...result, address: result.address ?? address, addresses, resolution: resolution._meta, attempts }; } catch (error) { attempts.push({ address, error: transportError(error) }); } } const message = remainingMs(deadline) ? "HTTP request failed for all resolved addresses." : "HTTP probe deadline exhausted."; return { url: safeUrl(url), addresses, attempts, error: message, resolution: resolution._meta }; } async function safeRequest(url: URL, requester: Requester, deadline: number, markPolicyBlock = false): Promise { try { validateProbeUrl(url); if (!remainingMs(deadline)) return { url: safeUrl(url), error: "HTTP probe deadline exhausted." }; return await withDeadline(requester(url, deadline), deadline, () => undefined, "HTTP request timed out."); } catch (error) { return { url: safeUrl(url), error: transportError(error), ...(markPolicyBlock && (error instanceof InspectionInputError || error instanceof InspectionPolicyError) ? { code: "REDIRECT_TARGET_BLOCKED" } : {}), }; } } function normalizedUrlKey(url: URL) { const host = normalizeHost(url.hostname); const port = url.port && url.port !== (url.protocol === "https:" ? "443" : "80") ? ":" + url.port : ""; return url.protocol.toLowerCase() + "//" + host + port + (url.pathname || "/") + url.search; } export async function followRedirectChain( start: URL, requester: Requester = requestOnce, maxRedirects = DEFAULT_MAX_REDIRECTS, deadline = createDeadline(HTTP_TOTAL_TIMEOUT_MS), ): Promise { const chain: HttpHop[] = []; const redirectLimit = Math.max(0, Math.floor(maxRedirects)); let current = start; const visited = new Set(); for (let index = 0; index <= redirectLimit; index++) { let key: string; try { key = normalizedUrlKey(current); } catch (error) { chain.push({ url: safeUrl(current), error: error instanceof Error ? error.message : "Invalid redirect target." }); break; } if (visited.has(key)) { chain.push({ url: current.toString(), error: REDIRECT_LOOP_ERROR, code: "REDIRECT_LOOP" }); break; } visited.add(key); const hop = await safeRequest(current, requester, deadline, index > 0); chain.push(sanitizedHop(hop)); if ("error" in hop) break; const status = Number(hop.status ?? 0); const location = typeof hop.location === "string" ? hop.location : null; if (status < 300 || status >= 400 || !location) break; if (index === redirectLimit) { chain.push({ url: current.toString(), error: "Redirect limit reached", code: "REDIRECT_LIMIT" }); break; } try { const next = new URL(location, current); validateProbeUrl(next); if (current.protocol === "https:" && next.protocol === "http:") { chain.push({ url: next.toString(), error: HTTPS_DOWNGRADE_ERROR, code: "HTTPS_DOWNGRADE", from: current.toString() }); break; } if (visited.has(normalizedUrlKey(next))) { chain.push({ url: next.toString(), error: REDIRECT_LOOP_ERROR, code: "REDIRECT_LOOP", from: current.toString() }); break; } current = next; } catch (error) { chain.push({ url: safeUrl(location), error: error instanceof Error ? error.message : "Invalid redirect target.", code: error instanceof InspectionInputError || error instanceof InspectionPolicyError ? "REDIRECT_TARGET_BLOCKED" : "INVALID_REDIRECT", }); break; } } return chain; } function errorEvidence(hop: HttpHop, probe: string) { return "error" in hop ? { probe, message: String(hop.error) } : null; } export async function collectHttp(host: string): Promise> { const normalizedHost = normalizeHost(host); const deadline = createDeadline(HTTP_TOTAL_TIMEOUT_MS); const rawHttpResult = await safeRequest(new URL("http://" + normalizedHost + "/"), requestOnce, deadline); const httpResult = sanitizedHop(rawHttpResult); let httpRedirectsToHttps = false; let httpRedirectTarget: HttpHop | null = null; if (!("error" in rawHttpResult)) { const status = Number(rawHttpResult.status ?? 0); const location = typeof rawHttpResult.location === "string" ? rawHttpResult.location : null; if (status >= 300 && status < 400 && location) { try { const target = new URL(location, new URL("http://" + normalizedHost + "/")); validateProbeUrl(target); const resolution = await resolvePublicHostDetailed(target.hostname, deadline); httpRedirectsToHttps = target.protocol === "https:"; httpRedirectTarget = { url: target.toString(), addresses: resolution.addresses, resolution: resolution._meta, validated: true }; } catch (error) { httpRedirectTarget = { url: safeUrl(location), error: error instanceof Error ? error.message : "Invalid redirect target.", code: error instanceof InspectionInputError || error instanceof InspectionPolicyError ? "REDIRECT_TARGET_BLOCKED" : "INVALID_REDIRECT", }; } } } const httpsChain = await followRedirectChain(new URL("https://" + normalizedHost + "/"), requestOnce, DEFAULT_MAX_REDIRECTS, deadline); const finalHop = httpsChain.at(-1) && !("error" in httpsChain.at(-1)!) ? httpsChain.at(-1)! : null; const downgrade = httpsChain.some((hop) => hop.code === "HTTPS_DOWNGRADE"); const evidenceErrors = [ errorEvidence(httpResult, "HTTP"), ...(httpRedirectTarget && "error" in httpRedirectTarget ? [{ probe: "HTTP redirect", message: String(httpRedirectTarget.error) }] : []), ...httpsChain.map((hop) => errorEvidence(hop, "HTTPS")), ].filter((item): item is { probe: string; message: string } => Boolean(item)); const hasSuccessfulObservation = ![httpResult, ...httpsChain].every((hop) => "error" in hop); const status = evidenceErrors.length ? (hasSuccessfulObservation ? "partial" : "failed") : "ok"; return { http: httpResult, httpRedirectsToHttps, httpRedirectTarget, httpsChain, httpsDowngrade: downgrade, finalUrl: finalHop && typeof finalHop.url === "string" ? finalHop.url : null, _meta: { status, errors: evidenceErrors }, }; }