56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
from __future__ import annotations
|
|
|
|
import posixpath
|
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
|
|
TRACKING_PARAMETERS = {
|
|
"fbclid",
|
|
"gclid",
|
|
"mc_cid",
|
|
"mc_eid",
|
|
"ref",
|
|
"referrer",
|
|
"source",
|
|
"trk",
|
|
"trackingid",
|
|
}
|
|
TRACKING_PREFIXES = ("utm_", "pk_")
|
|
|
|
|
|
def canonicalize_url(url: str) -> str:
|
|
value = (url or "").strip()
|
|
if not value:
|
|
return ""
|
|
parts = urlsplit(value)
|
|
scheme = parts.scheme.lower()
|
|
host = (parts.hostname or "").lower().rstrip(".")
|
|
if not scheme or not host:
|
|
return value
|
|
port = parts.port
|
|
if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)):
|
|
netloc = f"{host}:{port}"
|
|
else:
|
|
netloc = host
|
|
path = parts.path or "/"
|
|
normalized_path = posixpath.normpath(path)
|
|
if path.endswith("/") and not normalized_path.endswith("/"):
|
|
normalized_path += "/"
|
|
if not normalized_path.startswith("/"):
|
|
normalized_path = "/" + normalized_path
|
|
query_pairs = []
|
|
for key, value in parse_qsl(parts.query, keep_blank_values=True):
|
|
lower = key.lower()
|
|
if lower in TRACKING_PARAMETERS or any(
|
|
lower.startswith(prefix) for prefix in TRACKING_PREFIXES
|
|
):
|
|
continue
|
|
query_pairs.append((key, value))
|
|
query_pairs.sort()
|
|
return urlunsplit((scheme, netloc, normalized_path, urlencode(query_pairs, doseq=True), ""))
|
|
|
|
|
|
def domain_matches(hostname: str, domain: str) -> bool:
|
|
host = hostname.lower().rstrip(".")
|
|
target = domain.lower().rstrip(".")
|
|
return host == target or host.endswith("." + target)
|