from __future__ import annotations import posixpath from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit TRACKING_PARAMETERS = { "fbclid", "gclid", "mc_cid", "mc_eid", "ref", "referrer", "source", "trk", "trackingid", } TRACKING_PREFIXES = ("utm_", "pk_") def canonicalize_url(url: str) -> str: value = (url or "").strip() if not value: return "" parts = urlsplit(value) scheme = parts.scheme.lower() host = (parts.hostname or "").lower().rstrip(".") if not scheme or not host: return value port = parts.port if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)): netloc = f"{host}:{port}" else: netloc = host path = parts.path or "/" normalized_path = posixpath.normpath(path) if path.endswith("/") and not normalized_path.endswith("/"): normalized_path += "/" if not normalized_path.startswith("/"): normalized_path = "/" + normalized_path query_pairs = [] for key, value in parse_qsl(parts.query, keep_blank_values=True): lower = key.lower() if lower in TRACKING_PARAMETERS or any( lower.startswith(prefix) for prefix in TRACKING_PREFIXES ): continue query_pairs.append((key, value)) query_pairs.sort() return urlunsplit((scheme, netloc, normalized_path, urlencode(query_pairs, doseq=True), "")) def domain_matches(hostname: str, domain: str) -> bool: host = hostname.lower().rstrip(".") target = domain.lower().rstrip(".") return host == target or host.endswith("." + target)