feat: release regional radar and mailbox integrations

tests/security/test_policy.py

@@ -9,10 +9,20 @@ from apps.sources.services.robots import RobotsDecision
 
 
 def test_platform_domains_are_denied_including_subdomains():
-    assert is_denied_domain("www.linkedin.com")
-    assert is_denied_domain("be.indeed.com")
+    for domain in (
+        "www.linkedin.com",
+        "be.indeed.com",
+        "www.ictjob.be",
+        "www.jobat.be",
+        "www.stepstone.be",
+        "www.careerjet.be",
+        "jobs.randstad.be",
+        "www.roberthalf.com",
+    ):
+        assert is_denied_domain(domain)
     assert not is_denied_domain("jobs.example.org")
     assert not assess_url("https://www.linkedin.com/jobs/view/123").allowed
+    assert not assess_url("https://www.ictjob.be/nl/it-vacatures-zoeken/example").allowed
 
 
 @pytest.mark.security

tests/security/test_robots.py

@@ -92,3 +92,61 @@ def test_robots_cache_refreshes_after_ttl(db, settings, monkeypatch):
     assert calls["count"] == 2
     assert SourceRobotsCache.objects.filter(origin__contains="jobs.example.org").exists()
     client.close()
+
+
+@pytest.mark.security
+def test_robots_follows_and_revalidates_redirects(db, monkeypatch):
+    validated_urls = []
+
+    def validate(url, **kwargs):
+        validated_urls.append(url)
+        if "127.0.0.1" in url:
+            from apps.sources.services.url_security import UnsafeUrlError
+
+            raise UnsafeUrlError("Privéadres")
+        return ValidatedUrl(
+            url=url,
+            hostname="jobs.example.org",
+            port=443,
+            addresses=("93.184.216.34",),
+        )
+
+    monkeypatch.setattr("apps.sources.services.robots.validate_public_url", validate)
+    source = Source.objects.create(
+        name="Redirect jobs",
+        source_type=Source.Type.ATS,
+        base_url="https://ats.example.org/jobs",
+        domain="ats.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+
+    def safe_handler(request: httpx.Request) -> httpx.Response:
+        if request.url.host == "ats.example.org":
+            return httpx.Response(
+                302,
+                headers={"location": "https://jobs.example.org/robots.txt"},
+                request=request,
+            )
+        return httpx.Response(200, text="User-agent: *\nAllow: /", request=request)
+
+    client = httpx.Client(transport=httpx.MockTransport(safe_handler))
+    assert assess_robots(source.base_url, source=source, client=client).allowed is True
+    client.close()
+    assert validated_urls == [
+        "https://ats.example.org/robots.txt",
+        "https://jobs.example.org/robots.txt",
+    ]
+
+    SourceRobotsCache.objects.all().delete()
+
+    def unsafe_handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            302, headers={"location": "http://127.0.0.1/robots.txt"}, request=request
+        )
+
+    client = httpx.Client(transport=httpx.MockTransport(unsafe_handler))
+    blocked = assess_robots(source.base_url, source=source, client=client)
+    client.close()
+    assert blocked.allowed is False
+    assert "Privéadres" in blocked.reason