import httpx import pytest from django.test import override_settings from apps.sources.services.fetcher import ( ContentRejectedError, FetchError, FetchTimeoutError, RateLimitedError, PolicyBlockedError, fetch_url, ) from apps.sources.services.url_security import ValidatedUrl PUBLIC_DNS = [(None, None, None, None, ("93.184.216.34", 443))] def _patch_dns(monkeypatch): monkeypatch.setattr( "apps.sources.services.url_security.socket.getaddrinfo", lambda *args, **kwargs: PUBLIC_DNS, ) # validate_public_url captured the default resolver at definition time, so patch call site too. original = __import__( "apps.sources.services.url_security", fromlist=["validate_public_url"] ).validate_public_url def validate(url, **kwargs): return original(url, resolver=lambda *a, **k: PUBLIC_DNS, **kwargs) monkeypatch.setattr("apps.sources.services.fetcher.validate_public_url", validate) def test_fetcher_success_redirect_304_and_hash(monkeypatch): _patch_dns(monkeypatch) def handler(request: httpx.Request) -> httpx.Response: if request.url.path == "/start": return httpx.Response(302, headers={"location": "/job"}) return httpx.Response( 200, headers={"content-type": "text/html; charset=utf-8"}, content=b"Vacature", ) client = httpx.Client(transport=httpx.MockTransport(handler)) document = fetch_url("https://example.org/start", client=client) assert document.final_url == "https://example.org/job" assert document.text == "Vacature" assert len(document.sha256) == 64 client.close() client = httpx.Client( transport=httpx.MockTransport(lambda request: httpx.Response(304, headers={"etag": "x"})) ) unchanged = fetch_url("https://example.org/job", client=client) assert unchanged.status_code == 304 assert unchanged.content == b"" client.close() def test_fetcher_respects_rate_limit_and_retry_after(monkeypatch): _patch_dns(monkeypatch) client = httpx.Client( transport=httpx.MockTransport( lambda request: httpx.Response( 429, headers={"retry-after": "45"}, content=b"too many requests" ) ) ) with pytest.raises(RateLimitedError) as exc: fetch_url("https://example.org/too-fast", client=client) assert exc.value.retry_after_seconds == 45 client.close() def test_fetcher_rejects_dns_rebinding(monkeypatch): _patch_dns(monkeypatch) def validate(url: str, **kwargs): if "/final" in url: return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=("198.51.100.12",)) return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=("93.184.216.34",)) monkeypatch.setattr("apps.sources.services.fetcher.validate_public_url", validate) def handler(request: httpx.Request) -> httpx.Response: if request.url.path == "/start": return httpx.Response(302, headers={"location": "/final"}) return httpx.Response( 200, headers={"content-type": "text/html; charset=utf-8"}, content=b"Vacature", ) client = httpx.Client(transport=httpx.MockTransport(handler)) with pytest.raises(FetchError, match="DNS-rebindcontrole"): fetch_url("https://example.org/start", client=client) client.close() def test_fetcher_rejects_policy_content_size_and_http(monkeypatch): _patch_dns(monkeypatch) with pytest.raises(PolicyBlockedError): fetch_url("https://linkedin.com/jobs", client=httpx.Client()) client = httpx.Client( transport=httpx.MockTransport( lambda request: httpx.Response(200, headers={"content-type": "image/png"}, content=b"x") ) ) with pytest.raises(ContentRejectedError): fetch_url("https://example.org/image", client=client) client.close() with override_settings(FETCHER_MAX_BYTES=2): client = httpx.Client( transport=httpx.MockTransport( lambda request: httpx.Response( 200, headers={"content-type": "text/plain"}, content=b"long" ) ) ) with pytest.raises(ContentRejectedError): fetch_url("https://example.org/large", client=client) client.close() client = httpx.Client( transport=httpx.MockTransport(lambda request: httpx.Response(500, content=b"error")) ) with pytest.raises(FetchError, match="HTTP 500"): fetch_url("https://example.org/error", client=client) client.close() client = httpx.Client( transport=httpx.MockTransport(lambda request: (_ for _ in ()).throw(httpx.TimeoutException("timeout"))) ) with pytest.raises(FetchTimeoutError): fetch_url("https://example.org/slow", client=client) client.close() def test_fetcher_rejects_redirect_without_location(monkeypatch): _patch_dns(monkeypatch) client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(302))) with pytest.raises(FetchError, match="Location"): fetch_url("https://example.org/start", client=client) client.close()