This commit is contained in:
@@ -1,10 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime, timezone as utc
|
||||
from email.utils import parsedate_to_datetime
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import timezone as utc
|
||||
from email.utils import parsedate_to_datetime
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
@@ -13,7 +14,7 @@ from django.conf import settings
|
||||
from apps.sources.models import Source
|
||||
|
||||
from .policy import assess_url
|
||||
from .url_security import validate_public_url
|
||||
from .url_security import ValidatedUrl, validate_public_url
|
||||
|
||||
ALLOWED_CONTENT_TYPES = (
|
||||
"text/html",
|
||||
@@ -124,12 +125,19 @@ def fetch_url(
|
||||
headers=headers,
|
||||
)
|
||||
current_url = url
|
||||
previous_validation: ValidatedUrl | None = None
|
||||
try:
|
||||
for _ in range(settings.FETCHER_MAX_REDIRECTS + 1):
|
||||
validation = validate_public_url(
|
||||
current_url,
|
||||
allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
|
||||
)
|
||||
if (
|
||||
previous_validation is not None
|
||||
and validation.hostname == previous_validation.hostname
|
||||
and not set(validation.addresses).intersection(previous_validation.addresses)
|
||||
):
|
||||
raise FetchError("DNS-rebindcontrole faalde bij redirect naar dezelfde host.")
|
||||
response = http_client.get(current_url, headers=headers)
|
||||
post_validation = validate_public_url(
|
||||
str(response.url) if response.url else current_url,
|
||||
@@ -145,6 +153,7 @@ def fetch_url(
|
||||
next_decision = assess_url(current_url, source=source)
|
||||
if not next_decision.allowed:
|
||||
raise PolicyBlockedError(next_decision.reason)
|
||||
previous_validation = validation
|
||||
continue
|
||||
if response.status_code == 304:
|
||||
return FetchedDocument(url, current_url, 304, dict(response.headers), b"")
|
||||
|
||||
Reference in New Issue
Block a user