Initial deploy setup

apps/sources/services/canonicalize.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import posixpath
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+TRACKING_PARAMETERS = {
+    "fbclid",
+    "gclid",
+    "mc_cid",
+    "mc_eid",
+    "ref",
+    "referrer",
+    "source",
+    "trk",
+    "trackingid",
+}
+TRACKING_PREFIXES = ("utm_", "pk_")
+
+
+def canonicalize_url(url: str) -> str:
+    value = (url or "").strip()
+    if not value:
+        return ""
+    parts = urlsplit(value)
+    scheme = parts.scheme.lower()
+    host = (parts.hostname or "").lower().rstrip(".")
+    if not scheme or not host:
+        return value
+    port = parts.port
+    if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)):
+        netloc = f"{host}:{port}"
+    else:
+        netloc = host
+    path = parts.path or "/"
+    normalized_path = posixpath.normpath(path)
+    if path.endswith("/") and not normalized_path.endswith("/"):
+        normalized_path += "/"
+    if not normalized_path.startswith("/"):
+        normalized_path = "/" + normalized_path
+    query_pairs = []
+    for key, value in parse_qsl(parts.query, keep_blank_values=True):
+        lower = key.lower()
+        if lower in TRACKING_PARAMETERS or any(
+            lower.startswith(prefix) for prefix in TRACKING_PREFIXES
+        ):
+            continue
+        query_pairs.append((key, value))
+    query_pairs.sort()
+    return urlunsplit((scheme, netloc, normalized_path, urlencode(query_pairs, doseq=True), ""))
+
+
+def domain_matches(hostname: str, domain: str) -> bool:
+    host = hostname.lower().rstrip(".")
+    target = domain.lower().rstrip(".")
+    return host == target or host.endswith("." + target)