feat: release regional radar and mailbox integrations

apps/sources/services/robots.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime, timedelta
-from urllib.parse import urlsplit, urlunsplit
+from urllib.parse import urljoin, urlsplit, urlunsplit
 
 import httpx
 from django.conf import settings
@@ -11,6 +11,7 @@ from django.utils import timezone
 
 from apps.sources.models import Source, SourceRobotsCache
 
+from .tls import trusted_tls_context
 from .url_security import UnsafeUrlError, validate_public_url
 
 ALLOW = "allow"
@@ -123,20 +124,37 @@ def _max_bytes() -> int:
 
 def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
     robots_url = _robots_url(origin)
-    validate_public_url(
-        robots_url,
-        allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
-    )
     own_client = client is None
     http_client = client or httpx.Client(
         timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
         follow_redirects=False,
+        verify=trusted_tls_context(),
     )
     try:
-        response = http_client.get(
-            robots_url,
-            headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
-        )
+        current_url = robots_url
+        for _ in range(getattr(settings, "FETCHER_MAX_REDIRECTS", 5) + 1):
+            validate_public_url(
+                current_url,
+                allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
+            )
+            response = http_client.get(
+                current_url,
+                headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
+            )
+            if response.status_code not in {301, 302, 303, 307, 308}:
+                break
+            location = response.headers.get("location")
+            if not location:
+                raise httpx.HTTPStatusError(
+                    "Robotsredirect zonder Location-header",
+                    request=response.request,
+                    response=response,
+                )
+            current_url = urljoin(current_url, location)
+        else:
+            raise httpx.TooManyRedirects(
+                "Te veel redirects bij robotscontrole", request=response.request
+            )
     finally:
         if own_client:
             http_client.close()