Fix release blockers and deployment build
deploy / deploy (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-21 21:22:29 +02:00
parent b8091e59bd
commit a4eced8be5
64 changed files with 1011 additions and 675 deletions
+12 -3
View File
@@ -1,10 +1,11 @@
from __future__ import annotations
import hashlib
from datetime import datetime, timezone as utc
from email.utils import parsedate_to_datetime
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
from datetime import timezone as utc
from email.utils import parsedate_to_datetime
from urllib.parse import urljoin
import httpx
@@ -13,7 +14,7 @@ from django.conf import settings
from apps.sources.models import Source
from .policy import assess_url
from .url_security import validate_public_url
from .url_security import ValidatedUrl, validate_public_url
ALLOWED_CONTENT_TYPES = (
"text/html",
@@ -124,12 +125,19 @@ def fetch_url(
headers=headers,
)
current_url = url
previous_validation: ValidatedUrl | None = None
try:
for _ in range(settings.FETCHER_MAX_REDIRECTS + 1):
validation = validate_public_url(
current_url,
allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
)
if (
previous_validation is not None
and validation.hostname == previous_validation.hostname
and not set(validation.addresses).intersection(previous_validation.addresses)
):
raise FetchError("DNS-rebindcontrole faalde bij redirect naar dezelfde host.")
response = http_client.get(current_url, headers=headers)
post_validation = validate_public_url(
str(response.url) if response.url else current_url,
@@ -145,6 +153,7 @@ def fetch_url(
next_decision = assess_url(current_url, source=source)
if not next_decision.allowed:
raise PolicyBlockedError(next_decision.reason)
previous_validation = validation
continue
if response.status_code == 304:
return FetchedDocument(url, current_url, 304, dict(response.headers), b"")