Initial deploy setup

2026-07-21 14:00:00 +02:00
commit b8091e59bd
285 changed files with 27854 additions and 0 deletions
@@ -0,0 +1,148 @@
+import httpx
+import pytest
+from django.test import override_settings
+
+from apps.sources.services.fetcher import (
+    ContentRejectedError,
+    FetchError,
+    FetchTimeoutError,
+    RateLimitedError,
+    PolicyBlockedError,
+    fetch_url,
+)
+from apps.sources.services.url_security import ValidatedUrl
+
+PUBLIC_DNS = [(None, None, None, None, ("93.184.216.34", 443))]
+
+
+def _patch_dns(monkeypatch):
+    monkeypatch.setattr(
+        "apps.sources.services.url_security.socket.getaddrinfo",
+        lambda *args, **kwargs: PUBLIC_DNS,
+    )
+    # validate_public_url captured the default resolver at definition time, so patch call site too.
+    original = __import__(
+        "apps.sources.services.url_security", fromlist=["validate_public_url"]
+    ).validate_public_url
+
+    def validate(url, **kwargs):
+        return original(url, resolver=lambda *a, **k: PUBLIC_DNS, **kwargs)
+
+    monkeypatch.setattr("apps.sources.services.fetcher.validate_public_url", validate)
+
+
+def test_fetcher_success_redirect_304_and_hash(monkeypatch):
+    _patch_dns(monkeypatch)
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/start":
+            return httpx.Response(302, headers={"location": "/job"})
+        return httpx.Response(
+            200,
+            headers={"content-type": "text/html; charset=utf-8"},
+            content=b"Vacature",
+        )
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    document = fetch_url("https://example.org/start", client=client)
+    assert document.final_url == "https://example.org/job"
+    assert document.text == "Vacature"
+    assert len(document.sha256) == 64
+    client.close()
+
+    client = httpx.Client(
+        transport=httpx.MockTransport(lambda request: httpx.Response(304, headers={"etag": "x"}))
+    )
+    unchanged = fetch_url("https://example.org/job", client=client)
+    assert unchanged.status_code == 304
+    assert unchanged.content == b""
+    client.close()
+
+
+def test_fetcher_respects_rate_limit_and_retry_after(monkeypatch):
+    _patch_dns(monkeypatch)
+    client = httpx.Client(
+        transport=httpx.MockTransport(
+            lambda request: httpx.Response(
+                429, headers={"retry-after": "45"}, content=b"too many requests"
+            )
+        )
+    )
+    with pytest.raises(RateLimitedError) as exc:
+        fetch_url("https://example.org/too-fast", client=client)
+    assert exc.value.retry_after_seconds == 45
+    client.close()
+
+
+def test_fetcher_rejects_dns_rebinding(monkeypatch):
+    _patch_dns(monkeypatch)
+
+    def validate(url: str, **kwargs):
+        if "/final" in url:
+            return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=("198.51.100.12",))
+        return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=("93.184.216.34",))
+
+    monkeypatch.setattr("apps.sources.services.fetcher.validate_public_url", validate)
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/start":
+            return httpx.Response(302, headers={"location": "/final"})
+        return httpx.Response(
+            200,
+            headers={"content-type": "text/html; charset=utf-8"},
+            content=b"Vacature",
+        )
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    with pytest.raises(FetchError, match="DNS-rebindcontrole"):
+        fetch_url("https://example.org/start", client=client)
+    client.close()
+
+
+def test_fetcher_rejects_policy_content_size_and_http(monkeypatch):
+    _patch_dns(monkeypatch)
+    with pytest.raises(PolicyBlockedError):
+        fetch_url("https://linkedin.com/jobs", client=httpx.Client())
+
+    client = httpx.Client(
+        transport=httpx.MockTransport(
+            lambda request: httpx.Response(200, headers={"content-type": "image/png"}, content=b"x")
+        )
+    )
+    with pytest.raises(ContentRejectedError):
+        fetch_url("https://example.org/image", client=client)
+    client.close()
+
+    with override_settings(FETCHER_MAX_BYTES=2):
+        client = httpx.Client(
+            transport=httpx.MockTransport(
+                lambda request: httpx.Response(
+                    200, headers={"content-type": "text/plain"}, content=b"long"
+                )
+            )
+        )
+        with pytest.raises(ContentRejectedError):
+            fetch_url("https://example.org/large", client=client)
+        client.close()
+
+    client = httpx.Client(
+        transport=httpx.MockTransport(lambda request: httpx.Response(500, content=b"error"))
+    )
+    with pytest.raises(FetchError, match="HTTP 500"):
+        fetch_url("https://example.org/error", client=client)
+    client.close()
+
+    client = httpx.Client(
+        transport=httpx.MockTransport(lambda request: (_ for _ in ()).throw(httpx.TimeoutException("timeout")))
+    )
+    with pytest.raises(FetchTimeoutError):
+        fetch_url("https://example.org/slow", client=client)
+    client.close()
+
+
+def test_fetcher_rejects_redirect_without_location(monkeypatch):
+    _patch_dns(monkeypatch)
+    client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(302)))
+    with pytest.raises(FetchError, match="Location"):
+        fetch_url("https://example.org/start", client=client)
+    client.close()