Fix release blockers and deployment build

apps/sources/services/robots.py

@@ -10,6 +10,7 @@ from django.conf import settings
 from django.utils import timezone
 
 from apps.sources.models import Source, SourceRobotsCache
+
 from .url_security import UnsafeUrlError, validate_public_url
 
 ALLOW = "allow"
@@ -30,7 +31,11 @@ def _origin_for(url: str) -> str:
     if not host:
         raise ValueError("Host ontbreekt voor robotscontrole.")
     port = parts.port
-    if (parts.scheme == "http" and port == 80) or (parts.scheme == "https" and port == 443) or not port:
+    if (
+        (parts.scheme == "http" and port == 80)
+        or (parts.scheme == "https" and port == 443)
+        or not port
+    ):
         netloc = host
     else:
         netloc = f"{host}:{port}"
@@ -61,10 +66,7 @@ def _rules_from_text(text: str) -> dict[str, dict[str, list[str]]]:
         key_lower = key.lower()
         if key_lower == "user-agent":
             token = value.lower()
-            if token:
-                active_agents = {token}
-            else:
-                active_agents = set()
+            active_agents = {token} if token else set()
             continue
         if key_lower not in {ALLOW, DISALLOW}:
             continue
@@ -85,7 +87,7 @@ def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict
     selected = {ALLOW: [], DISALLOW: []}
 
     for agent, values in rules.items():
-        if agent == "*" or agent and agent in normalized:
+        if agent == "*" or (agent and agent in normalized):
             selected[ALLOW].extend(values[ALLOW])
             selected[DISALLOW].extend(values[DISALLOW])
 
@@ -95,10 +97,14 @@ def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict
 
 
 def _longest_prefix(path: str, rules: Iterable[str]) -> int:
-    return max((len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0)
+    return max(
+        (len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0
+    )
 
 
-def _evaluate_path(path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]) -> RobotsDecision:
+def _evaluate_path(
+    path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]
+) -> RobotsDecision:
     selected = _pick_rules(rules, user_agent=user_agent)
     allow_len = _longest_prefix(path, selected[ALLOW])
     disallow_len = _longest_prefix(path, selected[DISALLOW])
@@ -184,12 +190,7 @@ def _persist_cache(
             },
         )[0]
 
-    if status_code in {404, 410}:
-        rules = {}
-    elif status_code >= 400:
-        rules = {}
-    else:
-        rules = _rules_from_text(content)
+    rules = {} if status_code in {404, 410} or status_code >= 400 else _rules_from_text(content)
 
     return SourceRobotsCache.objects.update_or_create(
         origin=origin,
@@ -257,12 +258,18 @@ def assess_robots(
             return RobotsDecision(False, f"Robotscontrole mislukt: {exc}")
         except httpx.HTTPError as exc:
             if stale is not None:
-                return _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=_build_ruleset(stale))
+                return _evaluate_path(
+                    path,
+                    user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""),
+                    rules=_build_ruleset(stale),
+                )
             return RobotsDecision(True, f"Robotscontrole tijdelijk niet beschikbaar: {exc}")
 
     if cache.error:
         return RobotsDecision(True, cache.error)
 
     rules = _build_ruleset(cache)
-    decision = _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules)
+    decision = _evaluate_path(
+        path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules
+    )
     return decision