CertScope-Public/lib/inspect/http.ts

import http from "node:http";
import https from "node:https";
import { analyzeHeaders, type HeaderMap } from "./headers";
import { createDeadline, remainingMs, timeoutError, withDeadline } from "./deadline";
import { InspectionInputError, InspectionPolicyError, InspectionResolutionError, normalizeHost, resolvePublicHostDetailed, validateResolvedAddresses } from "./policy";
import { CERTSCOPE_VERSION } from "../version";

const HEADER_NAMES = [
  "strict-transport-security",
  "content-security-policy",
  "x-content-type-options",
  "referrer-policy",
  "x-frame-options",
  "server",
  "x-powered-by",
];
export const HTTP_TIMEOUT_MS = 5_000;
export const HTTP_TOTAL_TIMEOUT_MS = 15_000;
export const DEFAULT_MAX_REDIRECTS = 3;
export const MAX_ADDRESS_ATTEMPTS = 6;
export const HTTPS_DOWNGRADE_ERROR = "HTTPS redirect downgrade is not allowed.";
export const REDIRECT_LOOP_ERROR = "Redirect loop detected.";

export type HttpHop = Record<string, unknown>;
type Requester = (url: URL, deadline: number) => Promise<HttpHop>;
export type CandidateRequester = (url: URL, address: string, deadline: number) => Promise<HttpHop>;
export interface HttpRequestOptions {
  resolve?: typeof resolvePublicHostDetailed;
  request?: CandidateRequester;
}

function safeUrl(value: string | URL) {
  const text = typeof value === "string" ? value : value.toString();
  try {
    const parsed = new URL(text);
    parsed.username = "";
    parsed.password = "";
    return parsed.toString();
  } catch {
    return text.replace(/(\/\/)[^/\s@]+@/g, "$1[redacted]@");
  }
}

function sanitizedHop(hop: HttpHop): HttpHop {
  return typeof hop.location === "string" ? { ...hop, location: safeUrl(hop.location) } : hop;
}

export function validateProbeUrl(url: URL) {
  if (!["http:", "https:"].includes(url.protocol)) throw new InspectionPolicyError("Redirect used an unsupported protocol.");
  if (url.username || url.password) throw new InspectionPolicyError("Redirect URLs with credentials are not allowed.");
  if (url.port && url.port !== (url.protocol === "https:" ? "443" : "80")) {
    throw new InspectionPolicyError("Redirect to a non-default port is not allowed.");
  }
  normalizeHost(url.hostname);
}

function errorCode(error: unknown) {
  return typeof error === "object" && error !== null && "code" in error
    ? String((error as { code?: unknown }).code ?? "")
    : "";
}

function transportError(error: unknown) {
  if (error instanceof InspectionInputError || error instanceof InspectionPolicyError || error instanceof InspectionResolutionError) return error.message;
  if (errorCode(error) === "ETIMEOUT") return "HTTP request timed out.";
  const code = errorCode(error);
  return code ? "HTTP request failed (" + code + ")." : "HTTP request failed.";
}

function requestCandidate(url: URL, address: string, deadline: number): Promise<HttpHop> {
  const budget = Math.min(HTTP_TIMEOUT_MS, remainingMs(deadline));
  if (!budget) return Promise.reject(timeoutError("HTTP request timed out."));
  const host = normalizeHost(url.hostname);
  const client = url.protocol === "https:" ? https : http;

  return new Promise((resolve, reject) => {
    let settled = false;
    // The request constructor can throw before a timer exists; the callback
    // therefore needs an intentionally late-bound timer reference.
    // eslint-disable-next-line prefer-const
    let timer: ReturnType<typeof setTimeout> | undefined;
    const finish = (callback: () => void) => {
      if (settled) return;
      settled = true;
      if (timer) clearTimeout(timer);
      callback();
    };

    let request: ReturnType<typeof client.request>;
    try {
      request = client.request({
        hostname: address,
        port: url.protocol === "https:" ? 443 : 80,
        path: url.pathname + url.search,
        method: "GET",
        servername: url.protocol === "https:" ? host : undefined,
        rejectUnauthorized: url.protocol === "https:" ? true : undefined,
        headers: { host, "user-agent": "CertScope/" + CERTSCOPE_VERSION },
        timeout: budget,
      }, (response) => {
        if (settled) {
          response.destroy();
          return;
        }
        const headers = Object.fromEntries(
          HEADER_NAMES.map((name) => [name, response.headers[name] ?? null]),
        ) as HeaderMap;
        finish(() => {
          response.destroy();
          resolve({
            url: url.toString(),
            address,
            status: response.statusCode ?? null,
            location: response.headers.location ?? null,
            headers,
            headerPosture: analyzeHeaders(headers),
          });
        });
      });
    } catch (error) {
      finish(() => reject(error));
      return;
    }

    const abortOnTimeout = () => finish(() => {
      request.destroy();
      reject(timeoutError("HTTP request timed out."));
    });
    timer = setTimeout(abortOnTimeout, budget);
    request.once("timeout", abortOnTimeout);
    request.once("error", (error) => finish(() => reject(error)));
    request.end();
  });
}

export async function requestOnce(url: URL, deadline = createDeadline(HTTP_TIMEOUT_MS), options: HttpRequestOptions = {}): Promise<HttpHop> {
  validateProbeUrl(url);
  const resolution = await (options.resolve ?? resolvePublicHostDetailed)(url.hostname, deadline);
  const addresses = validateResolvedAddresses(resolution.addresses);
  const request = options.request ?? requestCandidate;
  const attempts: Array<Record<string, unknown>> = [];
  for (const address of addresses.slice(0, MAX_ADDRESS_ATTEMPTS)) {
    if (!remainingMs(deadline)) break;
    try {
      const result = await request(url, address, deadline);
      return { ...result, address: result.address ?? address, addresses, resolution: resolution._meta, attempts };
    } catch (error) {
      attempts.push({ address, error: transportError(error) });
    }
  }
  const message = remainingMs(deadline) ? "HTTP request failed for all resolved addresses." : "HTTP probe deadline exhausted.";
  return { url: safeUrl(url), addresses, attempts, error: message, resolution: resolution._meta };
}

async function safeRequest(url: URL, requester: Requester, deadline: number, markPolicyBlock = false): Promise<HttpHop> {
  try {
    validateProbeUrl(url);
    if (!remainingMs(deadline)) return { url: safeUrl(url), error: "HTTP probe deadline exhausted." };
    return await withDeadline(requester(url, deadline), deadline, () => undefined, "HTTP request timed out.");
  } catch (error) {
    return {
      url: safeUrl(url),
      error: transportError(error),
      ...(markPolicyBlock && (error instanceof InspectionInputError || error instanceof InspectionPolicyError)
        ? { code: "REDIRECT_TARGET_BLOCKED" }
        : {}),
    };
  }
}

function normalizedUrlKey(url: URL) {
  const host = normalizeHost(url.hostname);
  const port = url.port && url.port !== (url.protocol === "https:" ? "443" : "80") ? ":" + url.port : "";
  return url.protocol.toLowerCase() + "//" + host + port + (url.pathname || "/") + url.search;
}

export async function followRedirectChain(
  start: URL,
  requester: Requester = requestOnce,
  maxRedirects = DEFAULT_MAX_REDIRECTS,
  deadline = createDeadline(HTTP_TOTAL_TIMEOUT_MS),
): Promise<HttpHop[]> {
  const chain: HttpHop[] = [];
  const redirectLimit = Math.max(0, Math.floor(maxRedirects));
  let current = start;
  const visited = new Set<string>();

  for (let index = 0; index <= redirectLimit; index++) {
    let key: string;
    try {
      key = normalizedUrlKey(current);
    } catch (error) {
      chain.push({ url: safeUrl(current), error: error instanceof Error ? error.message : "Invalid redirect target." });
      break;
    }
    if (visited.has(key)) {
      chain.push({ url: current.toString(), error: REDIRECT_LOOP_ERROR, code: "REDIRECT_LOOP" });
      break;
    }
    visited.add(key);

    const hop = await safeRequest(current, requester, deadline, index > 0);
    chain.push(sanitizedHop(hop));
    if ("error" in hop) break;

    const status = Number(hop.status ?? 0);
    const location = typeof hop.location === "string" ? hop.location : null;
    if (status < 300 || status >= 400 || !location) break;
    if (index === redirectLimit) {
      chain.push({ url: current.toString(), error: "Redirect limit reached", code: "REDIRECT_LIMIT" });
      break;
    }

    try {
      const next = new URL(location, current);
      validateProbeUrl(next);
      if (current.protocol === "https:" && next.protocol === "http:") {
        chain.push({ url: next.toString(), error: HTTPS_DOWNGRADE_ERROR, code: "HTTPS_DOWNGRADE", from: current.toString() });
        break;
      }
      if (visited.has(normalizedUrlKey(next))) {
        chain.push({ url: next.toString(), error: REDIRECT_LOOP_ERROR, code: "REDIRECT_LOOP", from: current.toString() });
        break;
      }
      current = next;
    } catch (error) {
      chain.push({
        url: safeUrl(location),
        error: error instanceof Error ? error.message : "Invalid redirect target.",
        code: error instanceof InspectionInputError || error instanceof InspectionPolicyError ? "REDIRECT_TARGET_BLOCKED" : "INVALID_REDIRECT",
      });
      break;
    }
  }

  return chain;
}

function errorEvidence(hop: HttpHop, probe: string) {
  return "error" in hop ? { probe, message: String(hop.error) } : null;
}

export async function collectHttp(host: string): Promise<Record<string, unknown>> {
  const normalizedHost = normalizeHost(host);
  const deadline = createDeadline(HTTP_TOTAL_TIMEOUT_MS);
  const rawHttpResult = await safeRequest(new URL("http://" + normalizedHost + "/"), requestOnce, deadline);
  const httpResult = sanitizedHop(rawHttpResult);
  let httpRedirectsToHttps = false;
  let httpRedirectTarget: HttpHop | null = null;

  if (!("error" in rawHttpResult)) {
    const status = Number(rawHttpResult.status ?? 0);
    const location = typeof rawHttpResult.location === "string" ? rawHttpResult.location : null;
    if (status >= 300 && status < 400 && location) {
      try {
        const target = new URL(location, new URL("http://" + normalizedHost + "/"));
        validateProbeUrl(target);
        const resolution = await resolvePublicHostDetailed(target.hostname, deadline);
        httpRedirectsToHttps = target.protocol === "https:";
        httpRedirectTarget = { url: target.toString(), addresses: resolution.addresses, resolution: resolution._meta, validated: true };
      } catch (error) {
        httpRedirectTarget = {
          url: safeUrl(location),
          error: error instanceof Error ? error.message : "Invalid redirect target.",
          code: error instanceof InspectionInputError || error instanceof InspectionPolicyError ? "REDIRECT_TARGET_BLOCKED" : "INVALID_REDIRECT",
        };
      }
    }
  }

  const httpsChain = await followRedirectChain(new URL("https://" + normalizedHost + "/"), requestOnce, DEFAULT_MAX_REDIRECTS, deadline);
  const finalHop = httpsChain.at(-1) && !("error" in httpsChain.at(-1)!) ? httpsChain.at(-1)! : null;
  const downgrade = httpsChain.some((hop) => hop.code === "HTTPS_DOWNGRADE");
  const evidenceErrors = [
    errorEvidence(httpResult, "HTTP"),
    ...(httpRedirectTarget && "error" in httpRedirectTarget
      ? [{ probe: "HTTP redirect", message: String(httpRedirectTarget.error) }]
      : []),
    ...httpsChain.map((hop) => errorEvidence(hop, "HTTPS")),
  ].filter((item): item is { probe: string; message: string } => Boolean(item));
  const hasSuccessfulObservation = ![httpResult, ...httpsChain].every((hop) => "error" in hop);
  const status = evidenceErrors.length ? (hasSuccessfulObservation ? "partial" : "failed") : "ok";

  return {
    http: httpResult,
    httpRedirectsToHttps,
    httpRedirectTarget,
    httpsChain,
    httpsDowngrade: downgrade,
    finalUrl: finalHop && typeof finalHop.url === "string" ? finalHop.url : null,
    _meta: { status, errors: evidenceErrors },
  };
}