Fix release blockers and deployment build

apps/sources/adapters/__init__.py

@@ -1,7 +1,3 @@
-from .base import ExtractedJob, ExtractionResult
-from .generic_html import GenericHtmlAdapter
-from .jsonld import JsonLdJobPostingAdapter
-from .rss import RssAdapter
 from .ats import (
     GreenhouseAdapter,
     LeverAdapter,
@@ -9,16 +5,20 @@ from .ats import (
     SmartRecruitersAdapter,
     WorkableAdapter,
 )
+from .base import ExtractedJob, ExtractionResult
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .rss import RssAdapter
 
 __all__ = [
     "ExtractedJob",
     "ExtractionResult",
     "GenericHtmlAdapter",
-    "JsonLdJobPostingAdapter",
-    "RssAdapter",
     "GreenhouseAdapter",
+    "JsonLdJobPostingAdapter",
     "LeverAdapter",
     "RecruiteeAdapter",
+    "RssAdapter",
     "SmartRecruitersAdapter",
     "WorkableAdapter",
 ]

apps/sources/adapters/ats.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+from __future__ import annotations
 
 import json
 from abc import ABC, abstractmethod
@@ -8,7 +8,6 @@ from bs4 import BeautifulSoup
 
 from .base import ExtractedJob, ExtractionResult, FieldEvidence
 
-
 CLOSED_STATUSES = {
     "closed",
     "inactive",
@@ -51,7 +50,7 @@ def _first_text(data, *candidates):
             value = data.get(candidate) if isinstance(data, dict) else None
         if value is None:
             continue
-        if isinstance(value, (list, tuple)):
+        if isinstance(value, list | tuple):
             for item in value:
                 text = _to_text(item)
                 if text:
@@ -114,7 +113,7 @@ def _as_text(value) -> str:
 
 def _extract_payload(content: str):
     try:
-        return json.loads(content)
+        return json.loads(content.lstrip("\ufeff"))
     except json.JSONDecodeError:
         pass
     soup = BeautifulSoup(content, "lxml")
@@ -123,7 +122,7 @@ def _extract_payload(content: str):
         if not script_text:
             continue
         try:
-            return json.loads(script_text)
+            return json.loads(script_text.lstrip("\ufeff"))
         except json.JSONDecodeError:
             continue
     return None
@@ -139,8 +138,7 @@ class _AtsAdapter(ABC):
     closed_statuses = CLOSED_STATUSES
 
     @abstractmethod
-    def _extract_records(self, payload) -> list[dict[str, object]]:
-        ...
+    def _extract_records(self, payload) -> list[dict[str, object]]: ...
 
     def _supports_url(self, url: str) -> bool:
         host = (urlsplit(url).hostname or "").lower()
@@ -191,7 +189,7 @@ class _AtsAdapter(ABC):
                 location_raw = ", ".join(location_values)
 
         location = location_raw
-        location_parts = [part.strip() for part in _to_text(location).split(",") if part.strip()]
+        [part.strip() for part in _to_text(location).split(",") if part.strip()]
         region = _first_text(
             record,
             "region",
@@ -343,7 +341,7 @@ class _AtsAdapter(ABC):
             valid_through=valid_through,
             employment_types=employment_types,
             workplace_type=workplace_type,
-            raw=record,
+            raw=record.get("__raw__", record),
             evidence=evidence,
         )
 
@@ -389,7 +387,9 @@ class _AtsAdapter(ABC):
         warnings: list[str] = []
         if not jobs:
             warnings.append("Geen actieve ATS-vacatures gevonden")
-        return ExtractionResult(jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings)
+        return ExtractionResult(
+            jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings
+        )
 
 
 class GreenhouseAdapter(_AtsAdapter):
@@ -452,6 +452,12 @@ class LeverAdapter(_AtsAdapter):
                     break
                 value = value[key]
             if isinstance(value, dict):
+                if path == ("position",):
+                    raw_payload = {
+                        "position": value,
+                        "work_type": _to_text(value.get("workplaceType")),
+                    }
+                    return [{**value, "__raw__": raw_payload}]
                 return [value]
         return []
 

apps/sources/adapters/registry.py

@@ -1,8 +1,7 @@
-from __future__ import annotations
+from __future__ import annotations
 
 from apps.sources.models import RawDocument
 
-from .base import ExtractionResult
 from .ats import (
     GreenhouseAdapter,
     LeverAdapter,
@@ -10,6 +9,7 @@ from .ats import (
     SmartRecruitersAdapter,
     WorkableAdapter,
 )
+from .base import ExtractionResult
 from .generic_html import GenericHtmlAdapter
 from .jsonld import JsonLdJobPostingAdapter
 from .rss import RssAdapter
@@ -45,7 +45,10 @@ class AdapterRegistry:
                 result = provider.extract(content, url=url)
                 if any(
                     msg in result.warnings
-                    for msg in ("Geen parseerbare ATS-response", "Geen herkenbare ATS-markup voor deze adapter")
+                    for msg in (
+                        "Geen parseerbare ATS-response",
+                        "Geen herkenbare ATS-markup voor deze adapter",
+                    )
                 ):
                     continue
                 return result

apps/sources/models.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from datetime import timedelta
-from uuid import uuid4
 from urllib.parse import urlparse
 
 from django.conf import settings
@@ -91,7 +90,7 @@ class Source(TimeStampedModel):
         return [entry for entry in entries if isinstance(entry, dict)]
 
     @property
-    def latest_policy_review(self) -> "SourcePolicyReview | None":
+    def latest_policy_review(self) -> SourcePolicyReview | None:
         return self.policy_reviews.order_by("-created_at").first()
 
     @property
@@ -109,27 +108,37 @@ class Source(TimeStampedModel):
         if not review:
             return "Geen review geregistreerd"
         if review.is_expired:
-            return f"Review verlopen op {review.expires_at:%Y-%m-%d}" if review.expires_at else "Review verlopen"
+            return (
+                f"Review verlopen op {review.expires_at:%Y-%m-%d}"
+                if review.expires_at
+                else "Review verlopen"
+            )
         if review.expires_at:
             return f"{review.get_decision_display()} geldig tot {review.expires_at:%Y-%m-%d}"
         return f"{review.get_decision_display()} zonder vervaldatum"
 
     @property
     def requires_terms_review(self) -> bool:
-        return self.policy == Source.Policy.ALLOW and self.policy_review_state in {"missing", "expired"}
+        return self.policy == Source.Policy.ALLOW and self.policy_review_state in {
+            "missing",
+            "expired",
+        }
 
     def schedule_after_success(self, *, now=None, jitter_seconds: int = 0) -> None:
         now = now or timezone.now()
         self.last_success_at = now
         self.failure_count = 0
-        self.next_run_at = now + timedelta(minutes=self.crawl_interval_minutes) + timedelta(
-            seconds=max(0, jitter_seconds)
+        self.next_run_at = (
+            now
+            + timedelta(minutes=self.crawl_interval_minutes)
+            + timedelta(seconds=max(0, jitter_seconds))
         )
         self.save(update_fields=["last_success_at", "failure_count", "next_run_at", "updated_at"])
 
     def schedule_after_failure(
         self,
-        *, now=None,
+        *,
+        now=None,
         backoff_minutes: int | None = None,
         backoff_seconds: int | None = None,
     ) -> None:
@@ -186,7 +195,7 @@ class SourceRun(TimeStampedModel):
 
 class SourceLease(TimeStampedModel):
     source = models.OneToOneField(Source, on_delete=models.CASCADE, related_name="lease")
-    token = models.CharField(max_length=64, default=lambda: str(uuid4()))
+    token = models.CharField(max_length=64, default="")
     worker_id = models.CharField(max_length=128, blank=True)
     expires_at = models.DateTimeField(db_index=True)
 

apps/sources/services/discovery.py

@@ -4,11 +4,11 @@ import ipaddress
 import json
 import re
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from urllib.parse import urljoin, urlsplit
-from xml.etree import ElementTree
 
 from bs4 import BeautifulSoup
+from defusedxml import ElementTree
 from django.db import transaction
 
 from apps.sources.adapters.email_alert import EmailAlertAdapter
@@ -166,15 +166,14 @@ def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int
             evidence = metadata.get("discovery", [])
             if not isinstance(evidence, list):
                 evidence = []
-            if not any(item.get("url") == candidate.url for item in evidence if isinstance(item, dict)):
+            if not any(
+                item.get("url") == candidate.url for item in evidence if isinstance(item, dict)
+            ):
                 evidence.append(provenance)
                 metadata["discovery"] = evidence[-20:]
                 metadata["discovered_at"] = now
                 source.metadata = metadata
                 updated_fields.append("metadata")
-            else:
-                skipped += 1
-
             if not source.name:
                 source.name = _candidate_name(candidate)
                 updated_fields.append("name")
@@ -223,19 +222,24 @@ def _discover_html(
 
     for anchor in soup.find_all("a", href=True):
         label = " ".join(anchor.get_text(" ", strip=True).split())
+        raw_url = urljoin(base_url, str(anchor["href"]))
+        candidate_host = _hostname(raw_url)
+        same_domain = bool(base_domain and domain_matches(candidate_host, base_domain))
         candidate = _build_candidate(
-            raw_url=urljoin(base_url, str(anchor["href"])),
+            raw_url=raw_url,
             base_domain=base_domain,
             source_type=Source.Type.EMPLOYER,
             label=label,
             reason="career-link",
             discovered_from="html",
-            confidence=0.9,
-            allow_off_domain=False,
+            confidence=0.9 if same_domain else 0.7,
+            allow_off_domain=True,
         )
         if not candidate:
             continue
-        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(urlsplit(candidate.url).path):
+        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(
+            urlsplit(candidate.url).path
+        ):
             continue
         candidates.append(candidate)
 
@@ -388,4 +392,4 @@ def _to_domain_root_url(url: str) -> str:
 
 
 def _utc_now() -> str:
-    return datetime.now(timezone.utc).isoformat()
+    return datetime.now(UTC).isoformat()

apps/sources/services/fetcher.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import hashlib
-from datetime import datetime, timezone as utc
-from email.utils import parsedate_to_datetime
 from collections.abc import Mapping
 from dataclasses import dataclass
+from datetime import datetime
+from datetime import timezone as utc
+from email.utils import parsedate_to_datetime
 from urllib.parse import urljoin
 
 import httpx
@@ -13,7 +14,7 @@ from django.conf import settings
 from apps.sources.models import Source
 
 from .policy import assess_url
-from .url_security import validate_public_url
+from .url_security import ValidatedUrl, validate_public_url
 
 ALLOWED_CONTENT_TYPES = (
     "text/html",
@@ -124,12 +125,19 @@ def fetch_url(
         headers=headers,
     )
     current_url = url
+    previous_validation: ValidatedUrl | None = None
     try:
         for _ in range(settings.FETCHER_MAX_REDIRECTS + 1):
             validation = validate_public_url(
                 current_url,
                 allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
             )
+            if (
+                previous_validation is not None
+                and validation.hostname == previous_validation.hostname
+                and not set(validation.addresses).intersection(previous_validation.addresses)
+            ):
+                raise FetchError("DNS-rebindcontrole faalde bij redirect naar dezelfde host.")
             response = http_client.get(current_url, headers=headers)
             post_validation = validate_public_url(
                 str(response.url) if response.url else current_url,
@@ -145,6 +153,7 @@ def fetch_url(
                 next_decision = assess_url(current_url, source=source)
                 if not next_decision.allowed:
                     raise PolicyBlockedError(next_decision.reason)
+                previous_validation = validation
                 continue
             if response.status_code == 304:
                 return FetchedDocument(url, current_url, 304, dict(response.headers), b"")

apps/sources/services/health.py

@@ -174,7 +174,9 @@ def _is_parser_drift_run(run: SourceRun) -> bool:
 def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]:
     recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW]
     considered_failures = [
-        run for run in recent_runs if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
+        run
+        for run in recent_runs
+        if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
     ]
 
     if any(
@@ -209,7 +211,9 @@ def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | N
     return None, None
 
 
-def collect_source_health(*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW) -> list[SourceHealth]:
+def collect_source_health(
+    *, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW
+) -> list[SourceHealth]:
     sources = Source.objects.order_by("name").all()
     rows: list[SourceHealth] = []
 
@@ -256,7 +260,11 @@ def collect_source_health(*, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW) -
 
 def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]:
     now = now or timezone.now()
-    rows = [row for row in collect_source_health() if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}]
+    rows = [
+        row
+        for row in collect_source_health()
+        if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}
+    ]
     counts = {"evaluated": len(rows), "quarantined": 0}
 
     for row in rows:
@@ -309,7 +317,9 @@ def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]:
         if bool(health.get("canary_started", False)):
             continue
 
-        recovery_due = _from_iso(health.get("recovery_due_at") if isinstance(health, dict) else None)
+        recovery_due = _from_iso(
+            health.get("recovery_due_at") if isinstance(health, dict) else None
+        )
         if recovery_due and recovery_due > now:
             continue
 

apps/sources/services/manual_import.py

@@ -18,9 +18,9 @@ from apps.jobs.services.pipeline import process_raw_document
 from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
 from apps.sources.services.canonicalize import canonicalize_url
 from apps.sources.services.fetcher import (
+    FetchedDocument,
     FetchError,
     FetchTimeoutError,
-    FetchedDocument,
     PolicyBlockedError,
     RateLimitedError,
     fetch_url,
@@ -88,11 +88,7 @@ def _mode_source_name(domain: str, *, mode: str) -> str:
 
 def _ensure_manual_review(source: Source, actor) -> None:
     review = source.latest_policy_review
-    if (
-        review
-        and not review.is_expired
-        and review.decision == SourcePolicyReview.Decision.ALLOW
-    ):
+    if review and not review.is_expired and review.decision == SourcePolicyReview.Decision.ALLOW:
         return
     create_policy_review(
         source,
@@ -120,9 +116,7 @@ def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) ->
         source.base_url = source_url
         source.status = Source.Status.CANDIDATE
         source.policy = Source.Policy.ALLOW
-        source.save(
-            update_fields=["name", "base_url", "status", "policy", "updated_at"]
-        )
+        source.save(update_fields=["name", "base_url", "status", "policy", "updated_at"])
     _ensure_manual_review(source, actor=actor)
     return source
 
@@ -175,7 +169,7 @@ def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImpo
     source = document.source
     source_name = source.name if source else ""
     warnings: list[str] = list(metrics.get("warnings", []))
-    warnings_count = len(warnings)
+    len(warnings)
     source_run.finish(
         SourceRun.Status.SUCCESS,
         http_status=document.source_run.http_status if document.source_run else None,
@@ -186,9 +180,8 @@ def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImpo
         metrics={"parser": metrics["parser"], "warnings": warnings},
     )
     jobs = _build_jobs_from_document(document)
-    if metrics["created"] == 0 and metrics["updated"] == 0:
-        if not warnings:
-            warnings.append("De bron leverde geen herkenbare vacaturedata op.")
+    if metrics["created"] == 0 and metrics["updated"] == 0 and not warnings:
+        warnings.append("De bron leverde geen herkenbare vacaturedata op.")
     if not warnings:
         # keep stable, machine-readable payload shape
         warnings = []

apps/sources/services/policy.py

@@ -79,18 +79,28 @@ def _check_review_gate(source: Source) -> PolicyDecision | None:
         if not review:
             return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist")
         if review.decision == SourcePolicyReview.Decision.DENY:
-            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+            return PolicyDecision(
+                False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
+            )
         if review.decision == SourcePolicyReview.Decision.PAUSE:
-            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
+            )
         return None
 
     if source.policy == Source.Policy.ALLOW:
         if not review:
-            return PolicyDecision(False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid")
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid"
+            )
         if review.decision == SourcePolicyReview.Decision.DENY:
-            return PolicyDecision(False, Source.Policy.DENY, review.reason or "Review blokkeert bron")
+            return PolicyDecision(
+                False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
+            )
         if review.decision == SourcePolicyReview.Decision.PAUSE:
-            return PolicyDecision(False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze")
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
+            )
     return None
 
 

apps/sources/services/robots.py

@@ -10,6 +10,7 @@ from django.conf import settings
 from django.utils import timezone
 
 from apps.sources.models import Source, SourceRobotsCache
+
 from .url_security import UnsafeUrlError, validate_public_url
 
 ALLOW = "allow"
@@ -30,7 +31,11 @@ def _origin_for(url: str) -> str:
     if not host:
         raise ValueError("Host ontbreekt voor robotscontrole.")
     port = parts.port
-    if (parts.scheme == "http" and port == 80) or (parts.scheme == "https" and port == 443) or not port:
+    if (
+        (parts.scheme == "http" and port == 80)
+        or (parts.scheme == "https" and port == 443)
+        or not port
+    ):
         netloc = host
     else:
         netloc = f"{host}:{port}"
@@ -61,10 +66,7 @@ def _rules_from_text(text: str) -> dict[str, dict[str, list[str]]]:
         key_lower = key.lower()
         if key_lower == "user-agent":
             token = value.lower()
-            if token:
-                active_agents = {token}
-            else:
-                active_agents = set()
+            active_agents = {token} if token else set()
             continue
         if key_lower not in {ALLOW, DISALLOW}:
             continue
@@ -85,7 +87,7 @@ def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict
     selected = {ALLOW: [], DISALLOW: []}
 
     for agent, values in rules.items():
-        if agent == "*" or agent and agent in normalized:
+        if agent == "*" or (agent and agent in normalized):
             selected[ALLOW].extend(values[ALLOW])
             selected[DISALLOW].extend(values[DISALLOW])
 
@@ -95,10 +97,14 @@ def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict
 
 
 def _longest_prefix(path: str, rules: Iterable[str]) -> int:
-    return max((len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0)
+    return max(
+        (len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0
+    )
 
 
-def _evaluate_path(path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]) -> RobotsDecision:
+def _evaluate_path(
+    path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]
+) -> RobotsDecision:
     selected = _pick_rules(rules, user_agent=user_agent)
     allow_len = _longest_prefix(path, selected[ALLOW])
     disallow_len = _longest_prefix(path, selected[DISALLOW])
@@ -184,12 +190,7 @@ def _persist_cache(
             },
         )[0]
 
-    if status_code in {404, 410}:
-        rules = {}
-    elif status_code >= 400:
-        rules = {}
-    else:
-        rules = _rules_from_text(content)
+    rules = {} if status_code in {404, 410} or status_code >= 400 else _rules_from_text(content)
 
     return SourceRobotsCache.objects.update_or_create(
         origin=origin,
@@ -257,12 +258,18 @@ def assess_robots(
             return RobotsDecision(False, f"Robotscontrole mislukt: {exc}")
         except httpx.HTTPError as exc:
             if stale is not None:
-                return _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=_build_ruleset(stale))
+                return _evaluate_path(
+                    path,
+                    user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""),
+                    rules=_build_ruleset(stale),
+                )
             return RobotsDecision(True, f"Robotscontrole tijdelijk niet beschikbaar: {exc}")
 
     if cache.error:
         return RobotsDecision(True, cache.error)
 
     rules = _build_ruleset(cache)
-    decision = _evaluate_path(path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules)
+    decision = _evaluate_path(
+        path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules
+    )
     return decision

apps/sources/services/scheduling.py

@@ -76,14 +76,10 @@ def calculate_failure_backoff_seconds(
 
 
 def calculate_success_jitter_seconds(source: Source) -> int:
-    return calculate_jitter_seconds(
-        source, max_seconds=settings.SOURCE_SUCCESS_JITTER_SECONDS
-    )
+    return calculate_jitter_seconds(source, max_seconds=settings.SOURCE_SUCCESS_JITTER_SECONDS)
 
 
-def acquire_source_lease(
-    *, source_id: int, worker_token: str, now=None
-) -> SourceLease | None:
+def acquire_source_lease(*, source_id: int, worker_token: str, now=None) -> SourceLease | None:
     now = now or timezone.now()
     with transaction.atomic():
         source = Source.objects.select_for_update().get(pk=source_id)
@@ -101,9 +97,8 @@ def acquire_source_lease(
                 return None
 
         lease = SourceLease.objects.select_for_update().filter(source=source).first()
-        if lease is not None and not lease.is_expired:
-            if lease.token != worker_token:
-                return None
+        if lease is not None and not lease.is_expired and lease.token != worker_token:
+            return None
 
         active_leases = SourceLease.objects.select_for_update().filter(
             source__domain=domain, expires_at__gt=now
@@ -118,7 +113,10 @@ def acquire_source_lease(
         lease.token = worker_token
         lease.worker_id = worker_token
         lease.expires_at = now + timedelta(seconds=_lease_ttl_seconds())
-        lease.save(update_fields=["token", "worker_id", "expires_at", "updated_at"])
+        if lease.pk is None:
+            lease.save()
+        else:
+            lease.save(update_fields=["token", "worker_id", "expires_at", "updated_at"])
         return lease
 
 
@@ -129,9 +127,11 @@ def release_source_lease(*, source_id: int, worker_token: str, now=None) -> bool
         if not source.domain:
             return False
 
-        lease = SourceLease.objects.select_for_update().filter(
-            source=source, token=worker_token
-        ).first()
+        lease = (
+            SourceLease.objects.select_for_update()
+            .filter(source=source, token=worker_token)
+            .first()
+        )
         if not lease:
             return False
 

apps/sources/tasks.py

@@ -20,7 +20,11 @@ from apps.sources.services.fetcher import (
     RateLimitedError,
     fetch_url,
 )
-from apps.sources.services.health import canary_recovery_sources, evaluate_source_health, start_health_canary
+from apps.sources.services.health import (
+    canary_recovery_sources,
+    evaluate_source_health,
+    start_health_canary,
+)
 from apps.sources.services.policy import assess_url
 from apps.sources.services.scheduling import (
     acquire_source_lease,
@@ -82,9 +86,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
     try:
         fetched = fetch_url(source.base_url, source=source, conditional_headers=headers)
         if fetched.status_code == 304:
-            source.schedule_after_success(
-                jitter_seconds=calculate_success_jitter_seconds(source)
-            )
+            source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
             run.finish(SourceRun.Status.SUCCESS, http_status=304)
             return {"status": "not_modified"}
         content_type = fetched.headers.get("content-type", "")
@@ -110,9 +112,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
         source.etag = fetched.headers.get("etag", source.etag)
         source.last_modified = fetched.headers.get("last-modified", source.last_modified)
         source.save(update_fields=["etag", "last_modified", "updated_at"])
-        source.schedule_after_success(
-            jitter_seconds=calculate_success_jitter_seconds(source)
-        )
+        source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
         run.finish(
             SourceRun.Status.SUCCESS,
             http_status=fetched.status_code,
@@ -159,9 +159,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
         )
         return {"status": "timeout", "backoff": backoff}
     except FetchError as exc:
-        backoff = calculate_failure_backoff_seconds(
-            source, failure_count=source.failure_count + 1
-        )
+        backoff = calculate_failure_backoff_seconds(source, failure_count=source.failure_count + 1)
         source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
         run.finish(
             SourceRun.Status.FAILED,

apps/sources/views.py

@@ -11,22 +11,20 @@ from django.views.decorators.cache import never_cache
 from django.views.decorators.http import require_POST
 from django.views.generic import ListView
 
+from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
+
 from .forms import ManualImportForm
-from .models import Source
-from .models import SourcePolicyReview
+from .models import Source, SourcePolicyReview
 from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
 from .services.policy import create_policy_review
 from .tasks import fetch_source
-from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
 
 
 def _source_list_queryset():
     return Source.objects.prefetch_related("policy_reviews").all().order_by("name")
 
 
-def _manual_import_context(
-    request: HttpRequest, *, form: ManualImportForm, manual_result=None
-):
+def _manual_import_context(request: HttpRequest, *, form: ManualImportForm, manual_result=None):
     base_queryset = _source_list_queryset()
     return {
         "sources": base_queryset,
@@ -134,7 +132,7 @@ def bulk_candidate_action(request):
         source.policy_reason = reason
         source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
 
-    messages.success(request, f"{sources.count()} bron(nen) naar { _bulk_summary(action) }.")
+    messages.success(request, f"{sources.count()} bron(nen) naar {_bulk_summary(action)}.")
     return redirect("sources:list")