Initial deploy setup

tests/security/test_robots.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import httpx
+import pytest
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceRobotsCache
+from apps.sources.services.robots import assess_robots
+
+
+@pytest.mark.security
+def test_robots_uses_user_agent_matching_and_allows_default(db):
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+    calls = {"count": 0}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls["count"] += 1
+        assert request.url.path == "/robots.txt"
+        body = (
+            "User-agent: *\n"
+            "Disallow: /admin\n"
+            "User-agent: vacatureradar/0.1 (+local-personal-use)\n"
+            "Allow: /admin\n"
+        )
+        return httpx.Response(200, text=body)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    decision = assess_robots(
+        source.base_url + "admin",
+        source=source,
+        user_agent="VacatureRadar/0.1 (+local-personal-use)",
+        client=client,
+        now=timezone.now(),
+    )
+    client.close()
+    assert decision.allowed
+    assert calls["count"] == 1
+
+
+@pytest.mark.security
+def test_robots_cache_refreshes_after_ttl(db, settings):
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+    settings.ROBOTS_CACHE_TTL_SECONDS = 0
+
+    calls = {"count": 0}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls["count"] += 1
+        return httpx.Response(200, text="User-agent: *\nAllow: /")
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    assess_robots(source.base_url, source=source, user_agent="test-agent", client=client, now=timezone.now())
+    assert calls["count"] == 1
+    assess_robots(
+        source.base_url,
+        source=source,
+        user_agent="test-agent",
+        client=client,
+        now=timezone.now(),
+    )
+    assert calls["count"] == 2
+    assert SourceRobotsCache.objects.filter(origin__contains="jobs.example.org").exists()
+    client.close()