fix: validate connected peer for rotating CDNs

tests/unit/test_fetcher.py

@@ -76,28 +76,83 @@ def test_fetcher_respects_rate_limit_and_retry_after(monkeypatch):
 
 def test_fetcher_rejects_dns_rebinding(monkeypatch):
     _patch_dns(monkeypatch)
+    calls = 0
 
     def validate(url: str, **kwargs):
-        if "/final" in url:
-            return ValidatedUrl(
-                url=url, hostname="example.org", port=443, addresses=("198.51.100.12",)
+        nonlocal calls
+        calls += 1
+        address = "93.184.216.34" if calls == 1 else "198.51.100.12"
+        return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=(address,))
+
+    monkeypatch.setattr("apps.sources.services.fetcher.validate_public_url", validate)
+
+    client = httpx.Client(
+        transport=httpx.MockTransport(
+            lambda request: httpx.Response(
+                200,
+                headers={"content-type": "text/html; charset=utf-8"},
+                content=b"Vacature",
             )
-        return ValidatedUrl(url=url, hostname="example.org", port=443, addresses=("93.184.216.34",))
+        )
+    )
+    with pytest.raises(FetchError, match="DNS-rebindcontrole"):
+        fetch_url("https://example.org/start", client=client)
+    client.close()
+
+
+class _PeerStream:
+    def __init__(self, address: str) -> None:
+        self.address = address
+
+    def get_extra_info(self, key: str):
+        return (self.address, 443) if key == "server_addr" else None
+
+
+def test_fetcher_accepts_public_cdn_rotation_when_connected_peer_is_public(monkeypatch):
+    calls = 0
+
+    def validate(url: str, **kwargs):
+        nonlocal calls
+        calls += 1
+        address = "150.171.109.35" if calls == 1 else "150.171.109.36"
+        return ValidatedUrl(url=url, hostname="cdn.example.org", port=443, addresses=(address,))
 
     monkeypatch.setattr("apps.sources.services.fetcher.validate_public_url", validate)
 
     def handler(request: httpx.Request) -> httpx.Response:
-        if request.url.path == "/start":
-            return httpx.Response(302, headers={"location": "/final"})
+        status_code = 302 if request.url.path == "/start" else 200
+        headers = {"location": "/jobs"} if status_code == 302 else {"content-type": "text/html"}
         return httpx.Response(
-            200,
-            headers={"content-type": "text/html; charset=utf-8"},
-            content=b"Vacature",
+            status_code,
+            headers=headers,
+            content=b"" if status_code == 302 else b"Vacature",
+            extensions={"network_stream": _PeerStream("150.171.109.36")},
         )
 
     client = httpx.Client(transport=httpx.MockTransport(handler))
-    with pytest.raises(FetchError, match="DNS-rebindcontrole"):
-        fetch_url("https://example.org/start", client=client)
+
+    document = fetch_url("https://cdn.example.org/start", client=client)
+
+    assert document.status_code == 200
+    assert document.final_url == "https://cdn.example.org/jobs"
+    client.close()
+
+
+def test_fetcher_rejects_private_connected_peer(monkeypatch):
+    _patch_dns(monkeypatch)
+    client = httpx.Client(
+        transport=httpx.MockTransport(
+            lambda request: httpx.Response(
+                200,
+                headers={"content-type": "text/html"},
+                content=b"Vacature",
+                extensions={"network_stream": _PeerStream("10.0.0.5")},
+            )
+        )
+    )
+
+    with pytest.raises(FetchError, match="Niet-publiek verbonden IP-adres"):
+        fetch_url("https://example.org/jobs", client=client)
     client.close()